From 63d45f15df3484ad08c4b4d92a180e93c8ddd2d3 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Sep 2023 18:11:44 +0200 Subject: [PATCH 01/98] Add option to configure TLS connections to Cassandra in schema-migrations --- libs/cassandra-util/cassandra-util.cabal | 1 + libs/cassandra-util/default.nix | 2 + libs/cassandra-util/src/Cassandra/Schema.hs | 73 ++++++++++++++------- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/libs/cassandra-util/cassandra-util.cabal b/libs/cassandra-util/cassandra-util.cabal index 2612b0ee9b..004b8802df 100644 --- a/libs/cassandra-util/cassandra-util.cabal +++ b/libs/cassandra-util/cassandra-util.cabal @@ -77,6 +77,7 @@ library , cql-io >=0.14 , cql-io-tinylog , exceptions >=0.6 + , HsOpenSSL , imports , lens >=4.4 , lens-aeson >=1.0 diff --git a/libs/cassandra-util/default.nix b/libs/cassandra-util/default.nix index 5e634fad61..c7b1451a36 100644 --- a/libs/cassandra-util/default.nix +++ b/libs/cassandra-util/default.nix @@ -11,6 +11,7 @@ , cql-io-tinylog , exceptions , gitignoreSource +, HsOpenSSL , imports , lens , lens-aeson @@ -36,6 +37,7 @@ mkDerivation { cql-io cql-io-tinylog exceptions + HsOpenSSL imports lens lens-aeson diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index 79bbf51b9a..457c192a01 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -42,7 +42,7 @@ where import Cassandra (Client, Consistency (All, One), Keyspace (Keyspace), PrepQuery, QueryParams (QueryParams), QueryString (QueryString), R, S, Version (V4), W, params, query, query1, retry, runClient, write, x1, x5) import Cassandra qualified as CQL (init) -import Cassandra.Settings (Policy, defSettings, initialContactsPlain, setConnectTimeout, setContacts, setLogger, setMaxConnections, setPolicy, setPoolStripes, setPortNumber, setProtocolVersion, setResponseTimeout, setSendTimeout) +import Cassandra.Settings (Policy, defSettings, initialContactsPlain, setConnectTimeout, setContacts, setLogger, setMaxConnections, setPolicy, setPoolStripes, setPortNumber, setProtocolVersion, setResponseTimeout, setSSLContext, setSendTimeout) import Control.Monad.Catch import Control.Retry import Data.Aeson @@ -58,6 +58,7 @@ import Database.CQL.IO (HostResponse, Policy (Policy, acceptable, current, displ import Database.CQL.IO.Tinylog qualified as CT import Database.CQL.Protocol (Query (Query), Request (RqQuery)) import Imports hiding (All, fromString, init, intercalate, log) +import OpenSSL.Session qualified as OpenSSL import Options.Applicative hiding (info) -- FUTUREWORK: We could use the System.Logger.Class here in the future, but we don't have a ReaderT IO here (yet) import System.Logger qualified as Log @@ -73,7 +74,9 @@ data MigrationOpts = MigrationOpts migPort :: Word16, migKeyspace :: Text, migRepl :: ReplicationStrategy, - migReset :: Bool + migReset :: Bool, + migUseTLS :: Bool, + migTLSCert :: Maybe FilePath } deriving (Eq, Show, Generic) @@ -165,27 +168,29 @@ useKeyspace (Keyspace k) = void . getResult =<< qry migrateSchema :: Log.Logger -> MigrationOpts -> [Migration] -> IO () migrateSchema l o ms = do + mbSSLContext <- createSSLContext hosts <- initialContactsPlain $ pack (migHost o) - p <- - CQL.init - $ setLogger (CT.mkLogger l) - . setContacts (NonEmpty.head hosts) (NonEmpty.tail hosts) - . setPortNumber (fromIntegral $ migPort o) - . setMaxConnections 1 - . setPoolStripes 1 - -- 'migrationPolicy' ensures we only talk to one host for all queries - -- required for correct functioning of 'waitForSchemaConsistency' - . setPolicy migrationPolicy - -- use higher timeouts on schema migrations to reduce the probability - -- of a timeout happening during 'migAction' or 'metaInsert', - -- as that can lead to a state where schema migrations cannot be re-run - -- without manual action. - -- (due to e.g. "cannot create table X, already exists" errors) - . setConnectTimeout 20 - . setSendTimeout 20 - . setResponseTimeout 50 - . setProtocolVersion V4 - $ defSettings + let basicCQLSettings = + setLogger (CT.mkLogger l) + . setContacts (NonEmpty.head hosts) (NonEmpty.tail hosts) + . setPortNumber (fromIntegral $ migPort o) + . setMaxConnections 1 + . setPoolStripes 1 + -- 'migrationPolicy' ensures we only talk to one host for all queries + -- required for correct functioning of 'waitForSchemaConsistency' + . setPolicy migrationPolicy + -- use higher timeouts on schema migrations to reduce the probability + -- of a timeout happening during 'migAction' or 'metaInsert', + -- as that can lead to a state where schema migrations cannot be re-run + -- without manual action. + -- (due to e.g. "cannot create table X, already exists" errors) + . setConnectTimeout 20 + . setSendTimeout 20 + . setResponseTimeout 50 + . setProtocolVersion V4 + $ defSettings + cqlSettings = maybe basicCQLSettings (\sslCtx -> setSSLContext sslCtx basicCQLSettings) mbSSLContext + p <- CQL.init cqlSettings runClient p $ do let keyspace = Keyspace . migKeyspace $ o when (migReset o) $ do @@ -218,6 +223,20 @@ migrateSchema l o ms = do metaCreate = "create columnfamily if not exists meta (id int, version int, descr text, date timestamp, primary key (id, version))" metaInsert :: QueryString W (Int32, Text, UTCTime) () metaInsert = "insert into meta (id, version, descr, date) values (1,?,?,?)" + createSSLContext :: IO (Maybe OpenSSL.SSLContext) + createSSLContext + | o.migUseTLS = do + sslContext <- OpenSSL.context + maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) o.migTLSCert + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + | otherwise = pure Nothing -- | Retrieve and compare local and peer system schema versions. -- if they don't match, retry once per second for 30 seconds @@ -311,3 +330,13 @@ migrationOptsParser = ( long "reset" <> help "Reset the keyspace before running migrations" ) + <*> switch + ( long "use-tls" + <> help "Use TLS to connect to Cassandra" + ) + <*> option + auto + ( long "tls-certificate-file" + <> value Nothing + <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ) From 313ad58b163420470296cb8a5976d456d27aaafd Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 29 Sep 2023 11:53:02 +0200 Subject: [PATCH 02/98] WIP: Cassandra SSL in spar --- libs/types-common/src/Util/Options.hs | 4 ++- services/spar/default.nix | 1 + services/spar/spar.cabal | 1 + services/spar/src/Spar/Run.hs | 48 ++++++++++++++++++--------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/libs/types-common/src/Util/Options.hs b/libs/types-common/src/Util/Options.hs index d579c7fbb4..f2d50d4bf4 100644 --- a/libs/types-common/src/Util/Options.hs +++ b/libs/types-common/src/Util/Options.hs @@ -91,7 +91,9 @@ data CassandraOpts = CassandraOpts -- -- This option is most likely only necessary during a cassandra DC migration -- FUTUREWORK: remove this option again, or support a datacentre migration feature - _filterNodesByDatacentre :: !(Maybe Text) + _filterNodesByDatacentre :: !(Maybe Text), + _useTLS :: Bool, + _tlsCert :: Maybe FilePath } deriving (Show, Generic) diff --git a/services/spar/default.nix b/services/spar/default.nix index daebe1a84f..cc7f1b98e0 100644 --- a/services/spar/default.nix +++ b/services/spar/default.nix @@ -103,6 +103,7 @@ mkDerivation { extended galley-types hscim + HsOpenSSL hspec http-types imports diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index 548c1f2cef..702e876bdf 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -164,6 +164,7 @@ library , extended , galley-types , hscim + , HsOpenSSL , hspec , http-types , imports diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index e8bd47f0a4..7abfe0183d 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -45,6 +45,7 @@ import qualified Network.Wai as Wai import qualified Network.Wai.Handler.Warp as Warp import Network.Wai.Utilities.Request (lookupRequestId) import qualified Network.Wai.Utilities.Server as WU +import qualified OpenSSL.Session as OpenSSL import qualified SAML2.WebSSO as SAML import Spar.API (SparAPI, app) import Spar.App @@ -55,7 +56,7 @@ import Spar.Orphans () import System.Logger (Logger, msg, val, (.=), (~~)) import qualified System.Logger as Log import qualified System.Logger.Extended as Log -import Util.Options (endpoint, filterNodesByDatacentre, host, keyspace, port) +import Util.Options (CassandraOpts, endpoint, filterNodesByDatacentre, host, keyspace, port, tlsCert, useTLS) import Wire.API.Routes.Version.Wai import Wire.Sem.Logger.TinyLog @@ -65,27 +66,44 @@ import Wire.Sem.Logger.TinyLog initCassandra :: Opts -> Logger -> IO ClientState initCassandra opts lgr = do let cassOpts = cassandra opts + mbSSLContext <- createSSLContext cassOpts connectString <- maybe (Cas.initialContactsPlain (cassOpts ^. endpoint . host)) (Cas.initialContactsDisco "cassandra_spar" . cs) (discoUrl opts) - cas <- - Cas.init $ - Cas.defSettings - & Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.spar") lgr)) - & Cas.setContacts (NE.head connectString) (NE.tail connectString) - & Cas.setPortNumber (fromIntegral $ cassOpts ^. endpoint . port) - & Cas.setKeyspace (Keyspace $ cassOpts ^. keyspace) - & Cas.setMaxConnections 4 - & Cas.setMaxStreams 128 - & Cas.setPoolStripes 4 - & Cas.setSendTimeout 3 - & Cas.setResponseTimeout 10 - & Cas.setProtocolVersion V4 - & Cas.setPolicy (Cas.dcFilterPolicyIfConfigured lgr (cassOpts ^. filterNodesByDatacentre)) + let basicCASSettings = + Cas.defSettings + & Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.spar") lgr)) + & Cas.setContacts (NE.head connectString) (NE.tail connectString) + & Cas.setPortNumber (fromIntegral $ cassOpts ^. endpoint . port) + & Cas.setKeyspace (Keyspace $ cassOpts ^. keyspace) + & Cas.setMaxConnections 4 + & Cas.setMaxStreams 128 + & Cas.setPoolStripes 4 + & Cas.setSendTimeout 3 + & Cas.setResponseTimeout 10 + & Cas.setProtocolVersion V4 + & Cas.setPolicy (Cas.dcFilterPolicyIfConfigured lgr (cassOpts ^. filterNodesByDatacentre)) + casSettings = maybe basicCASSettings (\sslCtx -> Cas.setSSLContext sslCtx basicCASSettings) mbSSLContext + cas <- Cas.init casSettings runClient cas $ Cas.versionCheck Data.schemaVersion pure cas + where + createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) + createSSLContext cassOpts + | cassOpts ^. useTLS = do + sslContext <- OpenSSL.context + maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + | otherwise = pure Nothing ---------------------------------------------------------------------- -- servant / wai / warp From 094c33d8cb4c274e7179ecc7a439a63b13c33e6a Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 29 Sep 2023 14:16:10 +0200 Subject: [PATCH 03/98] Cassandra SSL in brig --- services/brig/src/Brig/App.hs | 44 ++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index 6f0f9e85e2..67ca94071b 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -141,6 +141,7 @@ import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.OpenSSL import OpenSSL.EVP.Digest (Digest, getDigestByName) import OpenSSL.Session (SSLOption (..)) +import OpenSSL.Session qualified as OpenSSL import OpenSSL.Session qualified as SSL import Polysemy import Polysemy.Final @@ -430,21 +431,38 @@ initCassandra o g = do (Cas.initialContactsPlain (Opt.cassandra o ^. endpoint . host)) (Cas.initialContactsDisco "cassandra_brig" . unpack) (Opt.discoUrl o) - p <- - Cas.init - $ Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.brig") g)) - . Cas.setContacts (NE.head c) (NE.tail c) - . Cas.setPortNumber (fromIntegral (Opt.cassandra o ^. endpoint . port)) - . Cas.setKeyspace (Keyspace (Opt.cassandra o ^. keyspace)) - . Cas.setMaxConnections 4 - . Cas.setPoolStripes 4 - . Cas.setSendTimeout 3 - . Cas.setResponseTimeout 10 - . Cas.setProtocolVersion Cas.V4 - . Cas.setPolicy (Cas.dcFilterPolicyIfConfigured g (Opt.cassandra o ^. filterNodesByDatacentre)) - $ Cas.defSettings + mbSSLContext <- createSSLContext (Opt.cassandra o) + let basicCasSettings = + Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.brig") g)) + . Cas.setContacts (NE.head c) (NE.tail c) + . Cas.setPortNumber (fromIntegral (Opt.cassandra o ^. endpoint . port)) + . Cas.setKeyspace (Keyspace (Opt.cassandra o ^. keyspace)) + . Cas.setMaxConnections 4 + . Cas.setPoolStripes 4 + . Cas.setSendTimeout 3 + . Cas.setResponseTimeout 10 + . Cas.setProtocolVersion Cas.V4 + . Cas.setPolicy (Cas.dcFilterPolicyIfConfigured g (Opt.cassandra o ^. filterNodesByDatacentre)) + $ Cas.defSettings + casSettings = maybe basicCasSettings (\sslCtx -> Cas.setSSLContext sslCtx basicCasSettings) mbSSLContext + p <- Cas.init casSettings runClient p $ versionCheck schemaVersion pure p + where + createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) + createSSLContext cassOpts + | cassOpts ^. useTLS = do + sslContext <- OpenSSL.context + maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + | otherwise = pure Nothing initCredentials :: (FromJSON a) => FilePathSecrets -> IO a initCredentials secretFile = do From b04c80c09e061580344cc1cf9188ed3ee0b5d09b Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 29 Sep 2023 15:04:33 +0200 Subject: [PATCH 04/98] Cassandra SSL for galley --- services/galley/src/Galley/App.hs | 45 ++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs index 8eded00734..c13adecfbc 100644 --- a/services/galley/src/Galley/App.hs +++ b/services/galley/src/Galley/App.hs @@ -94,6 +94,7 @@ import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.OpenSSL import Network.Wai.Utilities.JSONResponse import OpenSSL.Session as Ssl +import OpenSSL.Session qualified as OpenSSL import Polysemy import Polysemy.Error import Polysemy.Input @@ -179,19 +180,37 @@ initCassandra o l = do (C.initialContactsPlain (o ^. cassandra . endpoint . host)) (C.initialContactsDisco "cassandra_galley" . unpack) (o ^. discoUrl) - C.init - . C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.galley") l)) - . C.setContacts (NE.head c) (NE.tail c) - . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) - . C.setKeyspace (Keyspace $ o ^. cassandra . keyspace) - . C.setMaxConnections 4 - . C.setMaxStreams 128 - . C.setPoolStripes 4 - . C.setSendTimeout 3 - . C.setResponseTimeout 10 - . C.setProtocolVersion C.V4 - . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) - $ C.defSettings + mbSSLContext <- createSSLContext (o ^. cassandra) + let basicCasSettings = + C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.galley") l)) + . C.setContacts (NE.head c) (NE.tail c) + . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) + . C.setKeyspace (Keyspace $ o ^. cassandra . keyspace) + . C.setMaxConnections 4 + . C.setMaxStreams 128 + . C.setPoolStripes 4 + . C.setSendTimeout 3 + . C.setResponseTimeout 10 + . C.setProtocolVersion C.V4 + . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) + $ C.defSettings + casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext + C.init casSettings + where + createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) + createSSLContext cassOpts + | cassOpts ^. useTLS = do + sslContext <- OpenSSL.context + maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + | otherwise = pure Nothing initHttpManager :: Opts -> IO Manager initHttpManager o = do From 494e1f37f8d80658d8927270b5eb5898d0dc5fbb Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 4 Oct 2023 17:35:03 +0200 Subject: [PATCH 05/98] Add some TODOs --- services/brig/src/Brig/Index/Migrations.hs | 1 + services/galley/migrate-data/src/Galley/DataMigration.hs | 1 + services/gundeck/src/Gundeck/Env.hs | 1 + services/spar/migrate-data/src/Spar/DataMigration/Run.hs | 1 + 4 files changed, 4 insertions(+) diff --git a/services/brig/src/Brig/Index/Migrations.hs b/services/brig/src/Brig/Index/Migrations.hs index 27d7559fed..c0d91b3c4d 100644 --- a/services/brig/src/Brig/Index/Migrations.hs +++ b/services/brig/src/Brig/Index/Migrations.hs @@ -86,6 +86,7 @@ mkEnv l es cas galleyEndpoint = do <*> pure mgr <*> pure galleyEndpoint where + -- TODO: Add TLS support initCassandra = C.init $ C.setLogger (C.mkLogger l) diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 1ab4bc54bb..71029d88ae 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -69,6 +69,7 @@ mkEnv l cas = <$> initCassandra <*> initLogger where + -- TODO: Add TLS support initCassandra = C.init $ C.setLogger (C.mkLogger l) diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index df8850991d..101e337517 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -90,6 +90,7 @@ createEnv m o = do (rAddThread, rAdd) <- createRedisPool l additionalRedis "additional-write-redis" pure ([rAddThread], Just rAdd) + -- TODO: Add TLS support p <- C.init $ C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.gundeck") l)) diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index 4b13b42578..ac422c9e50 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -64,6 +64,7 @@ mkEnv settings = do . Log.setLogLevel (if s ^. setDebug == Debug then Log.Debug else Log.Info) $ Log.defSettings + -- TODO: Add TLS support initCassandra cas l = C.init . C.setLogger (C.mkLogger l) From 9d536125e58c306052be523a82764c93a2b92053 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 5 Oct 2023 15:36:11 +0200 Subject: [PATCH 06/98] Configure C* TLS in more places integration tests work locally --- Makefile | 18 ++++---- hack/cassandra.cert.pem | 30 +++++++++++++ integration/default.nix | 2 + integration/integration.cabal | 1 + integration/test/Testlib/Env.hs | 37 ++++++++++++++-- integration/test/Testlib/Types.hs | 20 ++++++++- libs/cassandra-util/src/Cassandra/Schema.hs | 11 +++-- libs/cassandra-util/src/Cassandra/Util.hs | 38 ++++++++++++---- services/brig/brig.integration.yaml | 2 + services/brig/src/Brig/Index/Eval.hs | 36 ++++++++++++---- services/brig/src/Brig/Index/Migrations.hs | 37 ++++++++++++---- services/brig/src/Brig/Index/Options.hs | 14 +++++- services/brig/test/integration/Run.hs | 3 +- services/galley/galley.integration.yaml | 2 + services/galley/test/integration/Run.hs | 3 +- services/gundeck/default.nix | 1 + services/gundeck/gundeck.cabal | 1 + services/gundeck/gundeck.integration.yaml | 2 + services/gundeck/src/Gundeck/Env.hs | 48 ++++++++++++++------- services/gundeck/test/integration/Main.hs | 3 +- services/integration.yaml | 1 + services/spar/spar.integration.yaml | 2 + 22 files changed, 248 insertions(+), 64 deletions(-) create mode 100644 hack/cassandra.cert.pem diff --git a/Makefile b/Makefile index 03b2615509..070d3f0df5 100644 --- a/Makefile +++ b/Makefile @@ -302,15 +302,15 @@ db-reset: c # Migrate all keyspaces and reset the ES index .PHONY: db-migrate db-migrate: c - ./dist/brig-schema --keyspace brig_test --replication-factor 1 > /dev/null - ./dist/galley-schema --keyspace galley_test --replication-factor 1 > /dev/null - ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 > /dev/null - ./dist/spar-schema --keyspace spar_test --replication-factor 1 > /dev/null - ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 > /dev/null - ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 > /dev/null - ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 > /dev/null - ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 > /dev/null - ./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 > /dev/null + ./dist/brig-schema --keyspace brig_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/galley-schema --keyspace galley_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/spar-schema --keyspace spar_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null ./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 > /dev/null ./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 > /dev/null ./integration/scripts/integration-dynamic-backends-brig-index.sh --elasticsearch-server http://localhost:9200 > /dev/null diff --git a/hack/cassandra.cert.pem b/hack/cassandra.cert.pem new file mode 100644 index 0000000000..b91d091423 --- /dev/null +++ b/hack/cassandra.cert.pem @@ -0,0 +1,30 @@ +Bag Attributes + friendlyName: node0 + localKeyID: 54 69 6D 65 20 31 36 39 34 37 30 30 39 39 39 39 36 32 +subject=C = None, L = None, O = None, OU = None, CN = 127.0.0.1 +issuer=C = None, L = None, O = None, OU = None, CN = 127.0.0.1 +-----BEGIN CERTIFICATE----- +MIIEQzCCAqugAwIBAgIIQCjt9rPKRJcwDQYJKoZIhvcNAQEMBQAwUDENMAsGA1UE +BhMETm9uZTENMAsGA1UEBxMETm9uZTENMAsGA1UEChMETm9uZTENMAsGA1UECxME +Tm9uZTESMBAGA1UEAxMJMTI3LjAuMC4xMB4XDTIzMDkxNDE0MTYzOVoXDTIzMTIx +MzE0MTYzOVowUDENMAsGA1UEBhMETm9uZTENMAsGA1UEBxMETm9uZTENMAsGA1UE +ChMETm9uZTENMAsGA1UECxMETm9uZTESMBAGA1UEAxMJMTI3LjAuMC4xMIIBojAN +BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA3+TVni16xmp7COX3hLnmhHUxXZxd +H6g8PZJ3dQOlhT/8Sw570ZmATkL+LGK90uAf8vW/RcBidHRpSMWsG57g/vo5fRi4 +zhMV1lFINxzmPJYvnb/CEwdoyHesWht/2SOCvOm02pDwBye8nlftdGp7fdYq/dhk ++dh8SGxB+dkqQG7+Jkv7i6xGqIj9j94UXl3ZbDyU7VJUhFg28H4vf1HiUOcV9oQ/ +JTM2qldaM5ALh+TFvBootXTS1iO9vKfbaGmfdeHibSyY13X1vvTI2GXTGWCaHxbz +1P/PEpjPFCeW6FfqwXJrv+iyB1NNdW0jHQjJzJGBeG7JxC6gDd+3GWA/yWDDKGyp +OlzKYqnqPd0+sDIPQPo1yis/4lwXrT/Wdac3Yvdmz0d9seUZ2LwSekpRZ1Phhxsk +2CsbKOaEF3w3VshoQWLFzATLuVGI25f7EcDzC0WaugrJvGhUMvJwuBXzFToaE1UL +IoNrF1IGxDjM3Qv0F623Sa3zBnDfw9kpcYFzAgMBAAGjITAfMB0GA1UdDgQWBBR5 +C6bMYEmcEtyFuTUAN3Ap55/OyzANBgkqhkiG9w0BAQwFAAOCAYEAoLn5vVYCKzJI +HTv4edUXs6evEqowSFj4dsQjEkwN2YF7MREmV/jWrrA0pRNbThUleMFtmsb5hLvT +Qxdl1eI7ntWHjDBJSLNSz55TD5+s03DyW6giHeRTTBZkuaHcmeL6csXIdRRucRba +nHQk+VVrOtp36JilBbU/cI+L9/JWNCTpOQCQnxn58yt1YoE8xAVVlKSmMPEsbzKA +dKXhIvo8xX/p0NQJ4ClPB++txZ1D/FlbG3N0OsLRAGTlbPFMZoHKMMFhg+PZNpPQ +3cdvMGEOJrk9dIF6p3g6JJRF7sNf5Q2IT1Wyzmdx1P92krx3BMdJGVQCDYdd2ZkU +v+vvaDzD5NTFOb+B7jEd23+zvCpqdqakHPUQDXMhTgGJPkq45Dp2ddAOHXhWF9RG +KU+xxomhNRETcLqNt0FP/9iETMmtgSf3FwTLm+Qro2pKBZtkJWHtP4cLNhi2Ikhj +ctlMm8xGfjzct2tQfWw4bP91S+g1t57ZSXOtEUNKOHHvPFk66b3+ +-----END CERTIFICATE----- diff --git a/integration/default.nix b/integration/default.nix index efdcfb7e04..84d88787e4 100644 --- a/integration/default.nix +++ b/integration/default.nix @@ -29,6 +29,7 @@ , filepath , gitignoreSource , hex +, HsOpenSSL , http-client , http-types , kan-extensions @@ -105,6 +106,7 @@ mkDerivation { extra filepath hex + HsOpenSSL http-client http-types kan-extensions diff --git a/integration/integration.cabal b/integration/integration.cabal index 6bf252ef0d..6ed62b2c61 100644 --- a/integration/integration.cabal +++ b/integration/integration.cabal @@ -171,6 +171,7 @@ library , extra , filepath , hex + , HsOpenSSL , http-client , http-types , kan-extensions diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index 2d98764ecf..bf82f70ff6 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -14,7 +14,10 @@ import Data.Set (Set) import Data.Set qualified as Set import Data.Yaml qualified as Yaml import Database.CQL.IO qualified as Cassandra +import Debug.Trace import Network.HTTP.Client qualified as HTTP +import OpenSSL.Session qualified as OpenSSL +import System.Directory import System.Environment (lookupEnv) import System.Exit import System.FilePath @@ -53,12 +56,26 @@ mkGlobalEnv cfgFile = do if last ps == "services" then Just (joinPath (init ps)) else Nothing + getCassCertFilePath :: IO (Maybe FilePath) = + maybe + (pure Nothing) + ( \certFilePath -> + if isAbsolute certFilePath + then pure $ Just certFilePath + else maybe (pure Nothing) (\projectRoot -> (Just <$> (makeAbsolute) (combine projectRoot certFilePath))) devEnvProjectRoot + ) + intConfig.cassandra.cassTlsCert manager <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings - let cassSettings = + + mbCassCertFilePath <- liftIO $ getCassCertFilePath + traceM $ "mbCassCertFilePath: " ++ show mbCassCertFilePath + mbSSLContext <- liftIO $ createSSLContext mbCassCertFilePath + let basicCassSettings = Cassandra.defSettings - & Cassandra.setContacts intConfig.cassandra.host [] - & Cassandra.setPortNumber (fromIntegral intConfig.cassandra.port) + & Cassandra.setContacts intConfig.cassandra.cassHost [] + & Cassandra.setPortNumber (fromIntegral intConfig.cassandra.cassPort) + cassSettings = maybe basicCassSettings (\sslCtx -> Cassandra.setSSLContext sslCtx basicCassSettings) mbSSLContext cassClient <- Cassandra.init cassSettings let resources = backendResources (Map.elems intConfig.dynamicBackends) resourcePool <- @@ -92,6 +109,20 @@ mkGlobalEnv cfgFile = do gTempDir = tempDir, gTimeOutSeconds = timeOutSeconds } + where + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just certFilePath) = do + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext certFilePath + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = pure Nothing mkEnv :: GlobalEnv -> Codensity IO Env mkEnv ge = do diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs index b4f0711753..0c38f69646 100644 --- a/integration/test/Testlib/Types.hs +++ b/integration/test/Testlib/Types.hs @@ -17,6 +17,7 @@ import Data.ByteString qualified as BS import Data.ByteString.Char8 qualified as C8 import Data.ByteString.Lazy qualified as L import Data.CaseInsensitive qualified as CI +import Data.Char (toLower) import Data.Default import Data.Functor import Data.IORef @@ -117,7 +118,7 @@ data IntegrationConfig = IntegrationConfig backendTwo :: BackendConfig, dynamicBackends :: Map String DynamicBackendConfig, rabbitmq :: RabbitMQConfig, - cassandra :: HostPort + cassandra :: CassandraConfig } deriving (Show, Generic) @@ -169,6 +170,23 @@ data HostPort = HostPort instance FromJSON HostPort +data CassandraConfig = CassandraConfig + { cassHost :: String, + cassPort :: Word16, + cassTlsCert :: Maybe FilePath + } + deriving (Show, Generic) + +instance FromJSON CassandraConfig where + parseJSON = genericParseJSON defaultOptions {fieldLabelModifier = lowerFirst . dropPrefix} + where + lowerFirst :: String -> String + lowerFirst (x : xs) = toLower x : xs + lowerFirst [] = "" + + dropPrefix :: String -> String + dropPrefix = Prelude.drop (length "cass") + -- | Initialised once per test. data Env = Env { serviceMap :: Map String ServiceMap, diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index 457c192a01..21794195fe 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -334,9 +334,8 @@ migrationOptsParser = ( long "use-tls" <> help "Use TLS to connect to Cassandra" ) - <*> option - auto - ( long "tls-certificate-file" - <> value Nothing - <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" - ) + <*> ( (optional . strOption) + ( long "tls-certificate-file" + <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ) + ) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index 942684cb6e..bd5ad456d1 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -24,7 +24,7 @@ where import Cassandra (ClientState, init) import Cassandra.CQL -import Cassandra.Settings (defSettings, setContacts, setKeyspace, setLogger, setPortNumber) +import Cassandra.Settings (defSettings, setContacts, setKeyspace, setLogger, setPortNumber, setSSLContext) import Data.Aeson import Data.Fixed import Data.Text (unpack) @@ -32,17 +32,37 @@ import Data.Time (UTCTime, nominalDiffTimeToSeconds) import Data.Time.Clock (secondsToNominalDiffTime) import Data.Time.Clock.POSIX import Database.CQL.IO.Tinylog qualified as CT +import Debug.Trace import Imports hiding (init) +import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log -defInitCassandra :: Text -> Text -> Word16 -> Log.Logger -> IO ClientState -defInitCassandra ks h p lg = - init - $ setLogger (CT.mkLogger lg) - . setPortNumber (fromIntegral p) - . setContacts (unpack h) [] - . setKeyspace (Keyspace ks) - $ defSettings +defInitCassandra :: Text -> Text -> Word16 -> Maybe FilePath -> Log.Logger -> IO ClientState +defInitCassandra ks h p mbCertPath lg = do + mbSSLContext <- createSSLContext mbCertPath + let basicCasSettings = + setLogger (CT.mkLogger lg) + . setPortNumber (fromIntegral p) + . setContacts (unpack h) [] + . setKeyspace (Keyspace ks) + $ defSettings + casSettings = maybe basicCasSettings (\sslCtx -> setSSLContext sslCtx basicCasSettings) mbSSLContext + init casSettings + where + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just tlsCertPath) = do + traceM $ "cassandra-util: " ++ show tlsCertPath + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext tlsCertPath + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = pure Nothing -- | Read cassandra's writetimes https://docs.datastax.com/en/dse/5.1/cql/cql/cql_using/useWritetime.html -- as UTCTime values without any loss of precision diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index 6114e56fa7..d02505cec9 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -8,6 +8,8 @@ cassandra: port: 9042 keyspace: brig_test # filterNodesByDatacentre: datacenter1 + useTLS: true + tlsCert: ../../hack/cassandra.cert.pem elasticsearch: url: http://127.0.0.1:9200 diff --git a/services/brig/src/Brig/Index/Eval.hs b/services/brig/src/Brig/Index/Eval.hs index 3b6e220043..ed38b89cdb 100644 --- a/services/brig/src/Brig/Index/Eval.hs +++ b/services/brig/src/Brig/Index/Eval.hs @@ -34,8 +34,10 @@ import Data.Aeson (FromJSON) import Data.Aeson qualified as Aeson import Data.Metrics qualified as Metrics import Database.Bloodhound qualified as ES +import Debug.Trace import Imports import Network.HTTP.Client as HTTP +import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log import System.Logger.Class (Logger, MonadLogger (..)) @@ -101,14 +103,32 @@ runCommand l = \case <*> pure mgr initES esURI mgr = ES.mkBHEnv (toESServer esURI) mgr - initDb cas = - C.init - $ C.setLogger (C.mkLogger l) - . C.setContacts (view cHost cas) [] - . C.setPortNumber (fromIntegral (view cPort cas)) - . C.setKeyspace (view cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings + initDb cas = do + mbSSLContext <- createSSLContext (cas ^. cTlsCert) + let basicCasSettings = + C.setLogger (C.mkLogger l) + . C.setContacts (view cHost cas) [] + . C.setPortNumber (fromIntegral (view cPort cas)) + . C.setKeyspace (view cKeyspace cas) + . C.setProtocolVersion C.V4 + $ C.defSettings + casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext + C.init casSettings + + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just tlsCertPath) = do + traceM $ "brig-index eval: " ++ show tlsCertPath + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext tlsCertPath + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = pure Nothing waitForTaskToComplete :: forall a m. (ES.MonadBH m, MonadThrow m, FromJSON a) => Int -> ES.TaskNodeId -> m () waitForTaskToComplete timeoutSeconds taskNodeId = do diff --git a/services/brig/src/Brig/Index/Migrations.hs b/services/brig/src/Brig/Index/Migrations.hs index c0d91b3c4d..0c358d0079 100644 --- a/services/brig/src/Brig/Index/Migrations.hs +++ b/services/brig/src/Brig/Index/Migrations.hs @@ -31,8 +31,10 @@ import Data.Aeson (Value, object, (.=)) import Data.Metrics qualified as Metrics import Data.Text qualified as Text import Database.Bloodhound qualified as ES +import Debug.Trace import Imports import Network.HTTP.Client qualified as HTTP +import OpenSSL.Session qualified as OpenSSL import System.Logger.Class (Logger) import System.Logger.Class qualified as Log import System.Logger.Extended (runWithLogger) @@ -86,15 +88,34 @@ mkEnv l es cas galleyEndpoint = do <*> pure mgr <*> pure galleyEndpoint where - -- TODO: Add TLS support initCassandra = - C.init - $ C.setLogger (C.mkLogger l) - . C.setContacts (view Opts.cHost cas) [] - . C.setPortNumber (fromIntegral (view Opts.cPort cas)) - . C.setKeyspace (view Opts.cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings + do + mbSSLContext <- createSSLContext (cas ^. Opts.cTlsCert) + let basicCasSettings = + C.setLogger (C.mkLogger l) + . C.setContacts (view Opts.cHost cas) [] + . C.setPortNumber (fromIntegral (view Opts.cPort cas)) + . C.setKeyspace (view Opts.cKeyspace cas) + . C.setProtocolVersion C.V4 + $ C.defSettings + casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext + C.init casSettings + + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just tlsCertPath) = do + traceM $ "brig-index: " ++ show tlsCertPath + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext tlsCertPath + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = pure Nothing + initLogger = pure l createMigrationsIndexIfNotPresent :: (MonadThrow m, ES.MonadBH m, Log.MonadLogger m) => m () diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index 6a08700e09..702bce01c9 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -31,6 +31,7 @@ module Brig.Index.Options CassandraSettings, cHost, cPort, + cTlsCert, cKeyspace, localElasticSettings, localCassandraSettings, @@ -82,7 +83,8 @@ data ElasticSettings = ElasticSettings data CassandraSettings = CassandraSettings { _cHost :: String, _cPort :: Word16, - _cKeyspace :: C.Keyspace + _cKeyspace :: C.Keyspace, + _cTlsCert :: Maybe FilePath } deriving (Show) @@ -125,7 +127,8 @@ localCassandraSettings = CassandraSettings { _cHost = "localhost", _cPort = 9042, - _cKeyspace = C.Keyspace "brig_test" + _cKeyspace = C.Keyspace "brig_test", + _cTlsCert = pure "hack/cassandra.cert.pem" } elasticServerParser :: Parser (URIRef Absolute) @@ -247,6 +250,13 @@ cassandraSettingsParser = <> showDefault ) ) + <*> ( (optional . strOption) + ( long "tls-certificate-file" + <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + <> value (fromMaybe "" (localCassandraSettings ^. cTlsCert)) + <> showDefault + ) + ) reindexToAnotherIndexSettingsParser :: Parser ReindexFromAnotherIndexSettings reindexToAnotherIndexSettingsParser = diff --git a/services/brig/test/integration/Run.hs b/services/brig/test/integration/Run.hs index 324dc92a6d..97bb68da5f 100644 --- a/services/brig/test/integration/Run.hs +++ b/services/brig/test/integration/Run.hs @@ -136,9 +136,10 @@ runTests iConf brigOpts otherArgs = do casHost = (\v -> Opts.cassandra v ^. endpoint . host) brigOpts casPort = (\v -> Opts.cassandra v ^. endpoint . port) brigOpts casKey = (\v -> Opts.cassandra v ^. keyspace) brigOpts + casTlsCert = (\v -> Opts.cassandra v ^. tlsCert) brigOpts awsOpts = Opts.aws brigOpts lg <- Logger.new Logger.defSettings -- TODO: use mkLogger'? - db <- defInitCassandra casKey casHost casPort lg + db <- defInitCassandra casKey casHost casPort casTlsCert lg mg <- newManager tlsManagerSettings let fedBrigClient = FedClient @'Brig mg (brig iConf) emailAWSOpts <- parseEmailAWSOpts diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml index e47801460b..c3b436459d 100644 --- a/services/galley/galley.integration.yaml +++ b/services/galley/galley.integration.yaml @@ -8,6 +8,8 @@ cassandra: port: 9042 keyspace: galley_test # filterNodesByDatacentre: datacenter1 + useTLS: true + tlsCert: ../../hack/cassandra.cert.pem brig: host: 0.0.0.0 diff --git a/services/galley/test/integration/Run.hs b/services/galley/test/integration/Run.hs index 7c33dadd62..4aca34ce51 100644 --- a/services/galley/test/integration/Run.hs +++ b/services/galley/test/integration/Run.hs @@ -127,8 +127,9 @@ main = withOpenSSL $ runTests go let ch = fromJust gConf ^. cassandra . endpoint . host let cp = fromJust gConf ^. cassandra . endpoint . port let ck = fromJust gConf ^. cassandra . keyspace + let cTlsCert = fromJust gConf ^. cassandra . tlsCert lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp lg + db <- defInitCassandra ck ch cp cTlsCert lg teamEventWatcher <- sequence $ SQS.watchSQSQueue <$> ((^. Aws.awsEnv) <$> awsEnv) <*> q pure $ TestSetup (fromJust gConf) (fromJust iConf) m g b c awsEnv convMaxSize db (FedClient m galleyEndpoint) teamEventWatcher queueName' = fmap (view queueName) . view journal diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix index 3dad13b422..78b071be02 100644 --- a/services/gundeck/default.nix +++ b/services/gundeck/default.nix @@ -107,6 +107,7 @@ mkDerivation { extra gundeck-types hedis + HsOpenSSL http-client http-client-tls http-types diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index ce1b2c82ac..669cfdb396 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -130,6 +130,7 @@ library , extra >=1.1 , gundeck-types >=1.0 , hedis >=0.14.0 + , HsOpenSSL , http-client >=0.7 , http-client-tls >=0.3 , http-types >=0.8 diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml index 7ceadf3ad8..0f8677f98b 100644 --- a/services/gundeck/gundeck.integration.yaml +++ b/services/gundeck/gundeck.integration.yaml @@ -12,6 +12,8 @@ cassandra: port: 9042 keyspace: gundeck_test # filterNodesByDatacentre: datacenter1 + useTLS: true + tlsCert: ../../hack/cassandra.cert.pem redis: host: 127.0.0.1 diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index 101e337517..ec5500ece4 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -43,6 +43,7 @@ import Gundeck.ThreadBudget import Imports import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.TLS (tlsManagerSettings) +import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log import System.Logger.Extended qualified as Logger import Util.Options @@ -90,21 +91,23 @@ createEnv m o = do (rAddThread, rAdd) <- createRedisPool l additionalRedis "additional-write-redis" pure ([rAddThread], Just rAdd) - -- TODO: Add TLS support - p <- - C.init - $ C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.gundeck") l)) - . C.setContacts (NE.head c) (NE.tail c) - . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) - . C.setKeyspace (Keyspace (o ^. cassandra . keyspace)) - . C.setMaxConnections 4 - . C.setMaxStreams 128 - . C.setPoolStripes 4 - . C.setSendTimeout 3 - . C.setResponseTimeout 10 - . C.setProtocolVersion C.V4 - . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) - $ C.defSettings + mbSSLContext <- createSSLContext (o ^. cassandra) + let basicCasSettings = + C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.gundeck") l)) + . C.setContacts (NE.head c) (NE.tail c) + . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) + . C.setKeyspace (Keyspace (o ^. cassandra . keyspace)) + . C.setMaxConnections 4 + . C.setMaxStreams 128 + . C.setPoolStripes 4 + . C.setSendTimeout 3 + . C.setResponseTimeout 10 + . C.setProtocolVersion C.V4 + . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) + $ C.defSettings + casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext + + p <- C.init casSettings a <- Aws.mkEnv l o n io <- mkAutoUpdate @@ -113,6 +116,21 @@ createEnv m o = do } mtbs <- mkThreadBudgetState `mapM` (o ^. settings . maxConcurrentNativePushes) pure $! (rThread : rAdditionalThreads,) $! Env (RequestId "N/A") m o l n p r rAdditional a io mtbs + where + createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) + createSSLContext cassOpts + | cassOpts ^. useTLS = do + sslContext <- OpenSSL.context + maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + | otherwise = pure Nothing reqIdMsg :: RequestId -> Logger.Msg -> Logger.Msg reqIdMsg = ("request" Logger..=) . unRequestId diff --git a/services/gundeck/test/integration/Main.hs b/services/gundeck/test/integration/Main.hs index 9ab372ede3..6b476be6ea 100644 --- a/services/gundeck/test/integration/Main.hs +++ b/services/gundeck/test/integration/Main.hs @@ -115,8 +115,9 @@ main = withOpenSSL $ runTests go ch = gConf ^. cassandra . endpoint . host cp = gConf ^. cassandra . endpoint . port ck = gConf ^. cassandra . keyspace + cTlsCert = gConf ^. cassandra . tlsCert lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp lg + db <- defInitCassandra ck ch cp cTlsCert lg pure $ TestSetup m g c c2 b db lg gConf (redis2 iConf) releaseOpts _ = pure () mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p diff --git a/services/integration.yaml b/services/integration.yaml index 65543e45f1..ae760e477e 100644 --- a/services/integration.yaml +++ b/services/integration.yaml @@ -142,3 +142,4 @@ rabbitmq: cassandra: host: 127.0.0.1 port: 9042 + tlsCert: hack/cassandra.cert.pem diff --git a/services/spar/spar.integration.yaml b/services/spar/spar.integration.yaml index 6a1eb2f398..f7a166a73f 100644 --- a/services/spar/spar.integration.yaml +++ b/services/spar/spar.integration.yaml @@ -28,6 +28,8 @@ cassandra: port: 9042 keyspace: spar_test filterNodesByDatacentre: datacenter1 + useTLS: true + tlsCert: ../../hack/cassandra.cert.pem # Wire/AWS specific, optional # discoUrl: "https://" From be94e109113e9ebb491e1d4be2c4ddfe33b9671e Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 5 Oct 2023 16:56:23 +0200 Subject: [PATCH 07/98] Add option to enable TLS to more C* inits --- services/galley/galley.cabal | 1 + .../migrate-data/src/Galley/DataMigration.hs | 44 ++++++++++++++----- .../src/Spar/DataMigration/Options.hs | 6 +++ .../src/Spar/DataMigration/Run.hs | 36 +++++++++++---- .../src/Spar/DataMigration/Types.hs | 3 +- services/spar/spar.cabal | 1 + 6 files changed, 71 insertions(+), 20 deletions(-) diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index 1c9908ea67..a2f86c0655 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -575,6 +575,7 @@ executable galley-migrate-data , exceptions , extended , galley-types + , HsOpenSSL , imports , lens , optparse-applicative diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 71029d88ae..05df46c2eb 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -24,6 +24,7 @@ import Data.Text qualified as Text import Data.Time (UTCTime, getCurrentTime) import Galley.DataMigration.Types import Imports +import OpenSSL.Session qualified as OpenSSL import Options.Applicative (Parser) import Options.Applicative qualified as Opts import System.Logger.Class (Logger) @@ -32,7 +33,8 @@ import System.Logger.Class qualified as Log data CassandraSettings = CassandraSettings { cHost :: String, cPort :: Word16, - cKeyspace :: C.Keyspace + cKeyspace :: C.Keyspace, + cTlsCert :: Maybe FilePath } cassandraSettingsParser :: Parser CassandraSettings @@ -53,6 +55,11 @@ cassandraSettingsParser = <> Opts.value "galley_test" ) ) + <*> ( (Opts.optional . Opts.strOption) + ( Opts.long "tls-certificate-file" + <> Opts.help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ) + ) migrate :: Logger -> CassandraSettings -> [Migration] -> IO () migrate l cas ms = do @@ -69,17 +76,34 @@ mkEnv l cas = <$> initCassandra <*> initLogger where - -- TODO: Add TLS support - initCassandra = - C.init - $ C.setLogger (C.mkLogger l) - . C.setContacts (cHost cas) [] - . C.setPortNumber (fromIntegral (cPort cas)) - . C.setKeyspace (cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings + initCassandra = do + mbSSLContext <- createSSLContext (cTlsCert cas) + let basicCasSettings = + C.setLogger (C.mkLogger l) + . C.setContacts (cHost cas) [] + . C.setPortNumber (fromIntegral (cPort cas)) + . C.setKeyspace (cKeyspace cas) + . C.setProtocolVersion C.V4 + $ C.defSettings + casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext + + C.init casSettings initLogger = pure l + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just tlsCertPath) = do + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext tlsCertPath + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = pure Nothing + -- | Runs only the migrations which need to run runMigrations :: [Migration] -> MigrationActionT IO () runMigrations migrations = do diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs index fdb70667eb..677abbf098 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs @@ -71,3 +71,9 @@ cassandraSettingsParser ks = <> showDefault ) ) + <*> ( (optional . strOption) + ( long "tls-certificate-file" + <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + <> showDefault + ) + ) diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index ac422c9e50..6c42839941 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -26,6 +26,7 @@ import Control.Monad.Catch (finally) import qualified Data.Text as Text import Data.Time (UTCTime, getCurrentTime) import Imports +import qualified OpenSSL.Session as OpenSSL import qualified Options.Applicative as Opts import Spar.DataMigration.Options (settingsParser) import Spar.DataMigration.Types @@ -64,15 +65,32 @@ mkEnv settings = do . Log.setLogLevel (if s ^. setDebug == Debug then Log.Debug else Log.Info) $ Log.defSettings - -- TODO: Add TLS support - initCassandra cas l = - C.init - . C.setLogger (C.mkLogger l) - . C.setContacts (cas ^. cHosts) [] - . C.setPortNumber (fromIntegral $ cas ^. cPort) - . C.setKeyspace (cas ^. cKeyspace) - . C.setProtocolVersion C.V4 - $ C.defSettings + + initCassandra cas l = do + mbSSLContext <- createSSLContext (cas ^. tlsCert) + let basicCasSettings = + C.setLogger (C.mkLogger l) + . C.setContacts (cas ^. cHosts) [] + . C.setPortNumber (fromIntegral $ cas ^. cPort) + . C.setKeyspace (cas ^. cKeyspace) + . C.setProtocolVersion C.V4 + $ C.defSettings + casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext + C.init casSettings + + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just tlsCertPath) = do + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext tlsCertPath + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = False, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = pure Nothing cleanup :: (MonadIO m) => Env -> m () cleanup env = do diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs index 751b3d20df..8e6da39530 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs @@ -62,7 +62,8 @@ data MigratorSettings = MigratorSettings data CassandraSettings = CassandraSettings { _cHosts :: !String, _cPort :: !Word16, - _cKeyspace :: !C.Keyspace + _cKeyspace :: !C.Keyspace, + _tlsCert :: Maybe FilePath } deriving (Show) diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index 702e876bdf..72b9b0d777 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -460,6 +460,7 @@ executable spar-migrate-data , conduit , containers , exceptions + , HsOpenSSL , imports , lens , optparse-applicative From 88d79c6d78de3ecc8a7095a6741ba587126035c7 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 5 Oct 2023 17:25:46 +0200 Subject: [PATCH 08/98] Set vpFailIfNoPeerCert --- integration/test/Testlib/Env.hs | 2 +- libs/cassandra-util/src/Cassandra/Schema.hs | 2 +- libs/cassandra-util/src/Cassandra/Util.hs | 2 +- services/brig/src/Brig/App.hs | 2 +- services/brig/src/Brig/Index/Eval.hs | 2 +- services/brig/src/Brig/Index/Migrations.hs | 2 +- services/federator/src/Federator/Monitor/Internal.hs | 2 +- services/galley/migrate-data/src/Galley/DataMigration.hs | 2 +- services/galley/src/Galley/App.hs | 2 +- services/gundeck/src/Gundeck/Env.hs | 2 +- services/spar/migrate-data/src/Spar/DataMigration/Run.hs | 2 +- services/spar/src/Spar/Run.hs | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index bf82f70ff6..c804af6114 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -117,7 +117,7 @@ mkGlobalEnv cfgFile = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index 21794195fe..21af85acfc 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -231,7 +231,7 @@ migrateSchema l o ms = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index bd5ad456d1..f58e5494d9 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -57,7 +57,7 @@ defInitCassandra ks h p mbCertPath lg = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index 67ca94071b..be8fb4ac0d 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -457,7 +457,7 @@ initCassandra o g = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/brig/src/Brig/Index/Eval.hs b/services/brig/src/Brig/Index/Eval.hs index ed38b89cdb..bd4c5af9b0 100644 --- a/services/brig/src/Brig/Index/Eval.hs +++ b/services/brig/src/Brig/Index/Eval.hs @@ -123,7 +123,7 @@ runCommand l = \case OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/brig/src/Brig/Index/Migrations.hs b/services/brig/src/Brig/Index/Migrations.hs index 0c358d0079..30dbeed84e 100644 --- a/services/brig/src/Brig/Index/Migrations.hs +++ b/services/brig/src/Brig/Index/Migrations.hs @@ -109,7 +109,7 @@ mkEnv l es cas galleyEndpoint = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/federator/src/Federator/Monitor/Internal.hs b/services/federator/src/Federator/Monitor/Internal.hs index 1b6b74f84d..9b8c15d63a 100644 --- a/services/federator/src/Federator/Monitor/Internal.hs +++ b/services/federator/src/Federator/Monitor/Internal.hs @@ -373,7 +373,7 @@ mkSSLContextWithoutCert settings = do SSL.contextSetVerificationMode ctx $ SSL.VerifyPeer { -- vpFailIfNoPeerCert and vpClientOnce are only relevant for servers - SSL.vpFailIfNoPeerCert = False, + SSL.vpFailIfNoPeerCert = True, SSL.vpClientOnce = False, SSL.vpCallback = Nothing } diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 05df46c2eb..33a16b3db8 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -97,7 +97,7 @@ mkEnv l cas = OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs index c13adecfbc..a80a0044c8 100644 --- a/services/galley/src/Galley/App.hs +++ b/services/galley/src/Galley/App.hs @@ -205,7 +205,7 @@ initCassandra o l = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index ec5500ece4..c4056fcc89 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -125,7 +125,7 @@ createEnv m o = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index 6c42839941..da331713a7 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -85,7 +85,7 @@ mkEnv settings = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index 7abfe0183d..3805000936 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -98,7 +98,7 @@ initCassandra opts lgr = do OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = False, + { vpFailIfNoPeerCert = True, vpClientOnce = True, vpCallback = Nothing } From 64ad952c6a21781756c42df0f2ec45ad80f2c72d Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 6 Oct 2023 17:06:57 +0200 Subject: [PATCH 09/98] WIP: Scratch out TLS/C* in service Helm charts --- charts/brig/templates/cassandra-secret.yaml | 15 +++++++++++++++ charts/brig/templates/configmap.yaml | 7 +++++++ charts/brig/templates/deployment.yaml | 8 ++++++++ charts/brig/values.yaml | 3 +++ charts/galley/templates/cassandra-secret.yaml | 15 +++++++++++++++ charts/galley/templates/configmap.yaml | 7 +++++++ charts/galley/templates/deployment.yaml | 8 ++++++++ charts/galley/values.yaml | 3 +++ charts/gundeck/templates/cassandra-secret.yaml | 15 +++++++++++++++ charts/gundeck/templates/configmap.yaml | 7 +++++++ charts/gundeck/templates/deployment.yaml | 8 ++++++++ charts/gundeck/values.yaml | 3 +++ charts/spar/templates/cassandra-secret.yaml | 15 +++++++++++++++ charts/spar/templates/configmap.yaml | 7 +++++++ charts/spar/templates/deployment.yaml | 8 ++++++++ charts/spar/values.yaml | 3 +++ 16 files changed, 132 insertions(+) create mode 100644 charts/brig/templates/cassandra-secret.yaml create mode 100644 charts/galley/templates/cassandra-secret.yaml create mode 100644 charts/gundeck/templates/cassandra-secret.yaml create mode 100644 charts/spar/templates/cassandra-secret.yaml diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..1b82446d7a --- /dev/null +++ b/charts/brig/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +apiVersion: v1 +kind: Secret +metadata: + name: brig-cassandra + labels: + app: brig + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + {{- with .Values.cassandra }} + ca.pem: {{ .ca | b64enc | quote }} +{{- end }} diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 2ed6eb6833..534b6b2c4d 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -28,6 +28,13 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + useTls: true + {{- if (hasKey .cassandra.tls.ca) }} + tlsCert: /etc/wire/brig/cassandra/ca.pem + {{- else }} + useTls: false + {{- end }} elasticsearch: url: http://{{ .elasticsearch.host }}:{{ .elasticsearch.port }} diff --git a/charts/brig/templates/deployment.yaml b/charts/brig/templates/deployment.yaml index 29f8ebc003..7892a543b5 100644 --- a/charts/brig/templates/deployment.yaml +++ b/charts/brig/templates/deployment.yaml @@ -102,6 +102,14 @@ spec: - name: "geoip" mountPath: "/usr/share/GeoIP" {{- end }} + {{- if and + (hasKey .Values.config.cassandra "tls") + (.Values.config.cassandra.tls.enabled) + (hasKey .Values.config.cassandra.tls "ca") + -}} + - name: "brig-cassandra" + mountPath: "/etc/wire/brig/cassandra" + {{- end }} env: - name: LOG_LEVEL value: {{ .Values.config.logLevel }} diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index b1109ba493..e58f6cdfb2 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -20,6 +20,9 @@ config: logNetStrings: false cassandra: host: aws-cassandra +# tls: +# enabled: false +# ca: CA in PEM format (can be self-signed) elasticsearch: host: elasticsearch-client port: 9200 diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..3a33981938 --- /dev/null +++ b/charts/galley/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +apiVersion: v1 +kind: Secret +metadata: + name: galley-cassandra + labels: + app: galley + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + {{- with .Values.cassandra }} + ca.pem: {{ .ca | b64enc | quote }} +{{- end }} diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 690bfd993c..bf77fcbca1 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -21,6 +21,13 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + useTls: true + {{- if (hasKey .cassandra.tls.ca) }} + tlsCert: /etc/wire/galley/cassandra/ca.pem + {{- else }} + useTls: false + {{- end }} brig: host: brig diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml index a9f2f50fb9..cdc86df9e1 100644 --- a/charts/galley/templates/deployment.yaml +++ b/charts/galley/templates/deployment.yaml @@ -36,6 +36,14 @@ spec: - name: "galley-secrets" secret: secretName: "galley" + {{- if and + (hasKey .Values.config.cassandra "tls") + (.Values.config.cassandra.tls.enabled) + (hasKey .Values.config.cassandra.tls "ca") + -}} + - name: "galley-cassandra" + mountPath: "/etc/wire/galley/cassandra" + {{- end }} containers: - name: galley image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/galley/values.yaml b/charts/galley/values.yaml index 0de07d8b4c..62c6151f85 100644 --- a/charts/galley/values.yaml +++ b/charts/galley/values.yaml @@ -22,6 +22,9 @@ config: cassandra: host: aws-cassandra replicaCount: 3 +# tls: +# enabled: false +# ca: CA in PEM format (can be self-signed) enableFederation: false # keep enableFederation default in sync with brig and cargohold chart's config.enableFederation as well as wire-server chart's tags.federation # Not used if enableFederation is false rabbitmq: diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..b4452818ca --- /dev/null +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +apiVersion: v1 +kind: Secret +metadata: + name: gundeck-cassandra + labels: + app: brig + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + {{- with .Values.cassandra }} + ca.pem: {{ .ca | b64enc | quote }} +{{- end }} diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index 527f521c26..c8e422cb89 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -25,6 +25,13 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + useTls: true + {{- if (hasKey .cassandra.tls.ca) }} + tlsCert: /etc/wire/gundeck/cassandra/ca.pem + {{- else }} + useTls: false + {{- end }} redis: host: {{ .redis.host }} diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml index 27255185da..37eeab6617 100644 --- a/charts/gundeck/templates/deployment.yaml +++ b/charts/gundeck/templates/deployment.yaml @@ -43,6 +43,14 @@ spec: volumeMounts: - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" + {{- if and + (hasKey .Values.config.cassandra "tls") + (.Values.config.cassandra.tls.enabled) + (hasKey .Values.config.cassandra.tls "ca") + -}} + - name: "gundeck-cassandra" + mountPath: "/etc/wire/gundeck/cassandra" + {{- end }} env: {{- if hasKey .Values.secrets "awsKeyId" }} - name: AWS_ACCESS_KEY_ID diff --git a/charts/gundeck/values.yaml b/charts/gundeck/values.yaml index 2841636144..c0b15ac26b 100644 --- a/charts/gundeck/values.yaml +++ b/charts/gundeck/values.yaml @@ -20,6 +20,9 @@ config: logNetStrings: false cassandra: host: aws-cassandra +# tls: +# enabled: false +# ca: CA in PEM format (can be self-signed) redis: host: redis-ephemeral-master port: 6379 diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..60dd2d3fdc --- /dev/null +++ b/charts/spar/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +apiVersion: v1 +kind: Secret +metadata: + name: spar-cassandra + labels: + app: spar + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + {{- with .Values.cassandra }} + ca.pem: {{ .ca | b64enc | quote }} +{{- end }} diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index 98711a4679..bb45a0f52f 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -25,6 +25,13 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + useTls: true + {{- if (hasKey .cassandra.tls.ca) }} + tlsCert: /etc/wire/spar/cassandra/ca.pem + {{- else }} + useTls: false + {{- end }} maxttlAuthreq: {{ .maxttlAuthreq }} maxttlAuthresp: {{ .maxttlAuthresp }} diff --git a/charts/spar/templates/deployment.yaml b/charts/spar/templates/deployment.yaml index 6d65b5d151..09bafe5b74 100644 --- a/charts/spar/templates/deployment.yaml +++ b/charts/spar/templates/deployment.yaml @@ -41,6 +41,14 @@ spec: volumeMounts: - name: "spar-config" mountPath: "/etc/wire/spar/conf" + {{- if and + (hasKey .Values.config.cassandra "tls") + (.Values.config.cassandra.tls.enabled) + (hasKey .Values.config.cassandra.tls "ca") + -}} + - name: "spar-cassandra" + mountPath: "/etc/wire/spar/cassandra" + {{- end }} env: {{- with .Values.config.proxy }} {{- if .httpProxy }} diff --git a/charts/spar/values.yaml b/charts/spar/values.yaml index 073fd5b0ee..e5a2d394d9 100644 --- a/charts/spar/values.yaml +++ b/charts/spar/values.yaml @@ -17,6 +17,9 @@ service: config: cassandra: host: aws-cassandra +# tls: +# enabled: false +# ca: CA in PEM format (can be self-signed) richInfoLimit: 5000 maxScimTokens: 0 logLevel: Info From b76a345d712f62bc1a97fcd9a30f96a421a4a7c7 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 13 Oct 2023 17:45:47 +0200 Subject: [PATCH 10/98] Some Helm fixes --- charts/brig/templates/cassandra-secret.yaml | 5 +++-- charts/brig/templates/configmap.yaml | 3 ++- charts/galley/templates/cassandra-secret.yaml | 5 +++-- charts/galley/templates/configmap.yaml | 3 ++- charts/gundeck/templates/cassandra-secret.yaml | 5 +++-- charts/gundeck/templates/configmap.yaml | 3 ++- charts/spar/templates/cassandra-secret.yaml | 5 +++-- charts/spar/templates/configmap.yaml | 3 ++- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index 1b82446d7a..217e2702ba 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -1,4 +1,5 @@ -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +{{- with .Values.cassandra }} +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: @@ -10,6 +11,6 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - {{- with .Values.cassandra }} ca.pem: {{ .ca | b64enc | quote }} {{- end }} +{{- end }} diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 534b6b2c4d..d02aec4240 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -30,11 +30,12 @@ data: {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} useTls: true - {{- if (hasKey .cassandra.tls.ca) }} + {{- if (hasKey .cassandra.tls "ca") }} tlsCert: /etc/wire/brig/cassandra/ca.pem {{- else }} useTls: false {{- end }} + {{- end }} elasticsearch: url: http://{{ .elasticsearch.host }}:{{ .elasticsearch.port }} diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml index 3a33981938..4ae6538ba3 100644 --- a/charts/galley/templates/cassandra-secret.yaml +++ b/charts/galley/templates/cassandra-secret.yaml @@ -1,4 +1,5 @@ -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +{{- with .Values.cassandra }} +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: @@ -10,6 +11,6 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - {{- with .Values.cassandra }} ca.pem: {{ .ca | b64enc | quote }} {{- end }} +{{- end }} diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index bf77fcbca1..e64abd5f9b 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -23,11 +23,12 @@ data: {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} useTls: true - {{- if (hasKey .cassandra.tls.ca) }} + {{- if (hasKey .cassandra.tls "ca" ) }} tlsCert: /etc/wire/galley/cassandra/ca.pem {{- else }} useTls: false {{- end }} + {{- end }} brig: host: brig diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml index b4452818ca..cdfb3c11b3 100644 --- a/charts/gundeck/templates/cassandra-secret.yaml +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -1,4 +1,5 @@ -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +{{- with .Values.cassandra }} +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: @@ -10,6 +11,6 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - {{- with .Values.cassandra }} ca.pem: {{ .ca | b64enc | quote }} {{- end }} +{{- end }} diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index c8e422cb89..cccae5784e 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -27,11 +27,12 @@ data: {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} useTls: true - {{- if (hasKey .cassandra.tls.ca) }} + {{- if (hasKey .cassandra.tls "ca" ) }} tlsCert: /etc/wire/gundeck/cassandra/ca.pem {{- else }} useTls: false {{- end }} + {{- end }} redis: host: {{ .redis.host }} diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml index 60dd2d3fdc..973c4e92b6 100644 --- a/charts/spar/templates/cassandra-secret.yaml +++ b/charts/spar/templates/cassandra-secret.yaml @@ -1,4 +1,5 @@ -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls.ca) }} +{{- with .Values.cassandra }} +{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: @@ -10,6 +11,6 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - {{- with .Values.cassandra }} ca.pem: {{ .ca | b64enc | quote }} {{- end }} +{{- end }} diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index bb45a0f52f..235fba83bd 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -27,11 +27,12 @@ data: {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} useTls: true - {{- if (hasKey .cassandra.tls.ca) }} + {{- if (hasKey .cassandra.tls "ca") }} tlsCert: /etc/wire/spar/cassandra/ca.pem {{- else }} useTls: false {{- end }} + {{- end }} maxttlAuthreq: {{ .maxttlAuthreq }} maxttlAuthresp: {{ .maxttlAuthresp }} From e777a8ed7397173298b90d335a60c40fd5f57f5c Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 6 Nov 2023 15:29:21 +0100 Subject: [PATCH 11/98] Fix useTLS in Helm charts Wrongly written and the no-tls case was broken. --- charts/brig/templates/configmap.yaml | 6 +++--- charts/galley/templates/configmap.yaml | 6 +++--- charts/gundeck/templates/configmap.yaml | 6 +++--- charts/spar/templates/configmap.yaml | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index d02aec4240..e1c7c210bf 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -29,12 +29,12 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} - useTls: true + useTLS: true {{- if (hasKey .cassandra.tls "ca") }} tlsCert: /etc/wire/brig/cassandra/ca.pem - {{- else }} - useTls: false {{- end }} + {{- else }} + useTLS: false {{- end }} elasticsearch: diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index e64abd5f9b..2cd6d758b8 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -22,12 +22,12 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} - useTls: true + useTLS: true {{- if (hasKey .cassandra.tls "ca" ) }} tlsCert: /etc/wire/galley/cassandra/ca.pem - {{- else }} - useTls: false {{- end }} + {{- else }} + useTLS: false {{- end }} brig: diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index cccae5784e..239d90d7a0 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -26,12 +26,12 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} - useTls: true + useTLS: true {{- if (hasKey .cassandra.tls "ca" ) }} tlsCert: /etc/wire/gundeck/cassandra/ca.pem - {{- else }} - useTls: false {{- end }} + {{- else }} + useTLS: false {{- end }} redis: diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index 235fba83bd..f526c1eab0 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -26,12 +26,12 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} - useTls: true + useTLS: true {{- if (hasKey .cassandra.tls "ca") }} tlsCert: /etc/wire/spar/cassandra/ca.pem - {{- else }} - useTls: false {{- end }} + {{- else }} + useTLS: false {{- end }} maxttlAuthreq: {{ .maxttlAuthreq }} From 86fcb920f6f1ca518ccfe8455fe86f769ac60e37 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 6 Nov 2023 16:20:25 +0100 Subject: [PATCH 12/98] Default to no cert in brig-index --- services/brig/src/Brig/Index/Options.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index 702bce01c9..8fb4805e03 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -128,7 +128,7 @@ localCassandraSettings = { _cHost = "localhost", _cPort = 9042, _cKeyspace = C.Keyspace "brig_test", - _cTlsCert = pure "hack/cassandra.cert.pem" + _cTlsCert = Nothing } elasticServerParser :: Parser (URIRef Absolute) From 6d63b8450d276be11188eed9ceaab267faf08982 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 7 Nov 2023 14:47:55 +0100 Subject: [PATCH 13/98] Fix: If cert is not set, use Nothing not empty string Otherwise, a file with an empty string filename is looked up, which makes no sense. --- services/brig/src/Brig/Index/Options.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index 8fb4805e03..0f64e9509a 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -253,7 +253,6 @@ cassandraSettingsParser = <*> ( (optional . strOption) ( long "tls-certificate-file" <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" - <> value (fromMaybe "" (localCassandraSettings ^. cTlsCert)) <> showDefault ) ) From 39e4051471bc73de28cf1674f08736c92d5dec22 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 9 Nov 2023 18:41:39 +0100 Subject: [PATCH 14/98] WIP: Teach cassandra-migrations TLS --- .../templates/_helpers.tpl | 56 ++++++++++++++ .../templates/cassandra-certs.yaml | 75 +++++++++++++++++++ .../templates/galley-migrate-data.yaml | 15 ++++ .../templates/migrate-schema.yaml | 71 +++++++++++++++++- .../templates/spar-migrate-data.yaml | 16 ++++ charts/cassandra-migrations/values.yaml | 15 +++- 6 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 charts/cassandra-migrations/templates/cassandra-certs.yaml diff --git a/charts/cassandra-migrations/templates/_helpers.tpl b/charts/cassandra-migrations/templates/_helpers.tpl index 551b901999..7f4c8dd1e3 100644 --- a/charts/cassandra-migrations/templates/_helpers.tpl +++ b/charts/cassandra-migrations/templates/_helpers.tpl @@ -107,6 +107,62 @@ Thus the order of priority is: {{- end -}} {{- end -}} +{{- define "useTlsCertGalley" -}} +{{ $cassandraGalley := default dict .Values.cassandraGalley }} +{{- or .Values.cassandra.tlsCert $cassandraGalley.tlsCert -}} +{{- end -}} + +{{- define "tlsCertGalley" -}} +{{ $cassandraGalley := default dict .Values.cassandraGalley }} +{{- if .Values.cassandra.tlsCert -}} +{{ .Values.cassandra.tlsCert }} +{{- else -}} +{{ $cassandraGalley.tlsCert }} +{{- end -}} +{{- end -}} + +{{- define "useTlsCertBrig" -}} +{{ $cassandraBrig := default dict .Values.cassandraBrig }} +{{- or .Values.cassandra.tlsCert $cassandraBrig.tlsCert -}} +{{- end -}} + +{{- define "tlsCertBrig" -}} +{{ $cassandraBrig := default dict .Values.cassandraBrig }} +{{- if .Values.cassandra.tlsCert -}} +{{ .Values.cassandra.tlsCert }} +{{- else -}} +{{ $cassandraBrig.tlsCert }} +{{- end -}} +{{- end -}} + +{{- define "useTlsCertSpar" -}} +{{ $cassandraSpar := default dict .Values.cassandraSpar }} +{{- or .Values.cassandra.tlsCert $cassandraSpar.tlsCert -}} +{{- end -}} + +{{- define "tlsCertSpar" -}} +{{ $cassandraSpar := default dict .Values.cassandraSpar }} +{{- if .Values.cassandra.tlsCert -}} +{{ .Values.cassandra.tlsCert }} +{{- else -}} +{{ $cassandraSpar.tlsCert }} +{{- end -}} +{{- end -}} + +{{- define "useTlsCertGundeck" -}} +{{ $cassandraGundeck := default dict .Values.cassandraGundeck }} +{{- or .Values.cassandra.tlsCert $cassandraGundeck.tlsCert -}} +{{- end -}} + +{{- define "tlsCertGundeck" -}} +{{ $cassandraGundeck := default dict .Values.cassandraGundeck }} +{{- if .Values.cassandra.tlsCert -}} +{{ .Values.cassandra.tlsCert }} +{{- else -}} +{{ $cassandraGundeck.tlsCert }} +{{- end -}} +{{- end -}} + {{/* Allow KubeVersion to be overridden. */}} {{- define "kubeVersion" -}} {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} diff --git a/charts/cassandra-migrations/templates/cassandra-certs.yaml b/charts/cassandra-migrations/templates/cassandra-certs.yaml new file mode 100644 index 0000000000..ca47c6a8bc --- /dev/null +++ b/charts/cassandra-migrations/templates/cassandra-certs.yaml @@ -0,0 +1,75 @@ +{{- if (include "useTlsCertBrig" .) }} +apiVersion: v1 +kind: Secret +metadata: + name: brig-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCertBrig" . | b64enc | quote }} +{{- end}} +{{- if (include "useTlsCertGalley" .) }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: galley-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCertGalley" . | b64enc | quote }} +{{- end}} +{{- if (include "useTlsCertGundeck" .) }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: gundeck-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCertGundeck" . | b64enc | quote }} +{{- end}} +{{- if (include "useTlsCertSpar" .) }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: spar-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCertSpar" . | b64enc | quote }} +{{- end}} diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index dee40d0b24..9f8864c159 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -42,4 +42,19 @@ spec: - "9042" - --cassandra-keyspace - galley + {{- if (include "useTlsCertGalley" .) }} + - --use-tls + - --tls-certificate-file /certs/galley/ca.pem + {{- end }} + {{- if (include "useTlsCertGalley" .) }} + volumeMounts: + - name: galley-cassandra-cert + mountPath: "/certs/galley" + {{- end }} + {{- if (include "useTlsCertGalley" .) }} + volumes: + - name: galley-cassandra-cert + secret: + secretName: galley-cassandra-cert + {{- end }} {{- end }} diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index 5129fc4baf..e9830f038f 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -9,7 +9,7 @@ metadata: heritage: {{ .Release.Service }} annotations: "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "0" + "helm.sh/hook-weight": "1" "helm.sh/hook-delete-policy": "before-hook-creation" spec: template: @@ -19,9 +19,30 @@ spec: release: {{ .Release.Name }} spec: restartPolicy: OnFailure - # specifying cassandra-migrations as initContainers executes them sequentially, rather than in parallel + # specifying cassandra-migrations as initContainers executes them sequentially, rather than in parallel # to avoid 'Column family ID mismatch' / schema disagreements # see https://stackoverflow.com/questions/29030661/creating-new-table-with-cqlsh-on-existing-keyspace-column-family-id-mismatch#40325651 for details. + volumes: + {{- if (include "useTlsCertGundeck" .) }} + - name: gundeck-cassandra-cert + secret: + secretName: gundeck-cassandra-cert + {{- end }} + {{- if (include "useTlsCertBrig" .) }} + - name: brig-cassandra-cert + secret: + secretName: brig-cassandra-cert + {{- end }} + {{- if (include "useTlsCertGalley" .) }} + - name: galley-cassandra-cert + secret: + secretName: galley-cassandra-cert + {{- end }} + {{- if (include "useTlsCertSpar" .) }} + - name: spar-cassandra-cert + secret: + secretName: spar-cassandra-cert + {{- end }} initContainers: {{- if .Values.enableGundeckMigrations }} - name: gundeck-schema @@ -41,6 +62,17 @@ spec: - gundeck - {{ template "cassandraGundeckReplicationType" . }} - "{{ template "cassandraGundeckReplicationArg" . }}" + {{- if (include "useTlsCertGundeck" .) }} + - --use-tls + - --tls-certificate-file + - /certs/gundeck/ca.pem + {{- end }} + + {{- if (include "useTlsCertGundeck" .) }} + volumeMounts: + - name: gundeck-cassandra-cert + mountPath: "/certs/gundeck" + {{- end }} {{- end }} {{- if .Values.enableBrigMigrations }} @@ -61,6 +93,17 @@ spec: - brig - {{ template "cassandraBrigReplicationType" . }} - "{{ template "cassandraBrigReplicationArg" . }}" + {{- if (include "useTlsCertBrig" .) }} + - --use-tls + - --tls-certificate-file + - /certs/brig/ca.pem + {{- end }} + + {{- if (include "useTlsCertBrig" .) }} + volumeMounts: + - name: brig-cassandra-cert + mountPath: "/certs/brig" + {{- end }} {{- end }} {{- if .Values.enableGalleyMigrations }} @@ -81,6 +124,17 @@ spec: - galley - {{ template "cassandraGalleyReplicationType" . }} - "{{ template "cassandraGalleyReplicationArg" . }}" + {{- if (include "useTlsCertGalley" .) }} + - --use-tls + - --tls-certificate-file + - /certs/galley/ca.pem + {{- end }} + + {{- if (include "useTlsCertGalley" .) }} + volumeMounts: + - name: galley-cassandra-cert + mountPath: "/certs/galley" + {{- end }} {{- end }} {{- if .Values.enableSparMigrations }} @@ -101,7 +155,18 @@ spec: - spar - {{ template "cassandraSparReplicationType" . }} - "{{ template "cassandraSparReplicationArg" . }}" - {{- end }} + {{- if (include "useTlsCertSpar" .) }} + - --use-tls + - --tls-certificate-file + - /certs/spar/ca.pem + {{- end }} + + {{- if (include "useTlsCertSpar" .) }} + volumeMounts: + - name: spar-cassandra-cert + mountPath: "/certs/spar" + {{- end }} + {{- end }} containers: - name: job-done diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index 1b9c48e066..bb248b4c14 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -43,4 +43,20 @@ spec: - "9042" - --cassandra-keyspace-brig - brig + # TODO: This is odd because we also need to talk to brig db + {{- if (include "useTlsCertSpar" .) }} + - --use-tls + - --tls-certificate-file /certs/spar/ca.pem + {{- end }} + {{- if (include "useTlsCertSpar" .) }} + volumeMounts: + - name: spar-cassandra-cert + mountPath: "/certs/spar" + {{- end }} + {{- if (include "useTlsCertSpar" .) }} + volumes: + - name: spar-cassandra-cert + secret: + secretName: spar-cassandra-cert + {{- end }} {{- end }} diff --git a/charts/cassandra-migrations/values.yaml b/charts/cassandra-migrations/values.yaml index bf2a31d1b6..554db4958a 100644 --- a/charts/cassandra-migrations/values.yaml +++ b/charts/cassandra-migrations/values.yaml @@ -47,7 +47,20 @@ images: # cassandraGundeck: # host: cassandra-ephemeral-galley # replicationMap: eu-west-1:3 - +# +# To enable TLS/SSL connections provide the certificate as PEM string: +# +# cassandra: +# host: cassandra-external +# replicationFactor: 3 +# tlsCert: +# +# This also works for dedicated service setups. E.g. +# +# cassandraGalley: +# host: cassandra-ephemeral-galley +# replicationMap: eu-west-1:3 +# tlsCert: # Overriding the following is only useful during datacenter migration time periods, # where some other job already migrates schemas. From a1d5b895fc4b0adc6639cde3caca399e3d9aed25 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 10 Nov 2023 07:58:45 +0100 Subject: [PATCH 15/98] GHC option -threaded for brig-schema and galley-schema This is required by HsOpenSSL; otherwise SSL connections to Cassandra cannot be setup. --- services/brig/brig.cabal | 2 +- services/galley/galley.cabal | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index 3df1d72450..29007477ef 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -534,7 +534,7 @@ executable brig-schema import: common-all main-is: Main.hs hs-source-dirs: schema - ghc-options: -funbox-strict-fields -Wredundant-constraints + ghc-options: -funbox-strict-fields -Wredundant-constraints -threaded default-extensions: TemplateHaskell build-depends: , base diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index a2f86c0655..36ec4c267d 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -592,6 +592,7 @@ executable galley-schema import: common-all main-is: Main.hs hs-source-dirs: schema + ghc-options: -threaded default-extensions: TemplateHaskell build-depends: , galley From 0156d046ee5a77ac36759c90efd60351bfe6d02e Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 10 Nov 2023 15:35:34 +0100 Subject: [PATCH 16/98] k8ssandra-test: Add client encryption options Used to configure TLS/SSL between clients and Cassandra. --- .../templates/client-encryption-stores.yaml | 13 +++++++++++++ .../templates/k8ssandra-cluster.yaml | 11 +++++++++++ charts/k8ssandra-test-cluster/values.yaml | 10 ++++++++++ 3 files changed, 34 insertions(+) create mode 100644 charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml diff --git a/charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml b/charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml new file mode 100644 index 0000000000..bfa4a24f92 --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml @@ -0,0 +1,13 @@ +{{- if .Values.client_encryption_options.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: client-encryption-stores + namespace: {{ .Release.Namespace }} +type: Opaque +data: + keystore: {{ .Values.client_encryption_options.keystore | quote }} + "keystore-password": {{ .Values.client_encryption_options.keystorePassword | b64enc | quote }} + truststore: {{ .Values.client_encryption_options.truststore | quote }} + "truststore-password": {{ .Values.client_encryption_options.truststorePassword | b64enc | quote }} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml b/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml index 50560a52d5..506a363adb 100644 --- a/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml +++ b/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml @@ -26,6 +26,10 @@ spec: gc_g1_max_gc_pause_ms: 300 gc_g1_initiating_heap_occupancy_percent: 55 gc_g1_parallel_threads: 16 + cassandraYaml: + client_encryption_options: + enabled: {{ .Values.client_encryption_options.enabled }} + optional: {{ .Values.client_encryption_options.optional }} datacenters: - metadata: name: datacenter-1 @@ -38,6 +42,13 @@ spec: resources: requests: storage: {{ .Values.storageSize }} + {{- if .Values.client_encryption_options.enabled }} + clientEncryptionStores: + keystoreSecretRef: + name: client-encryption-stores + truststoreSecretRef: + name: client-encryption-stores + {{- end }} reaper: autoScheduling: enabled: true diff --git a/charts/k8ssandra-test-cluster/values.yaml b/charts/k8ssandra-test-cluster/values.yaml index 3aabc8db1a..18a7e3a40e 100644 --- a/charts/k8ssandra-test-cluster/values.yaml +++ b/charts/k8ssandra-test-cluster/values.yaml @@ -11,3 +11,13 @@ storageClassName: hcloud-volumes-encrypted # storage, it's fine to request 10GB. The memory units are described here: # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory storageSize: 10G + +# These options relate to the client_encryption_options described in: +# https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options +client_encryption_options: + enabled: false + optional: true + keystore: "" + keystorePassword: password + truststore: "" + truststorePassword: password From 3d4251c9011b3a10913ab5625c902892d643d73f Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 14 Nov 2023 18:57:47 +0100 Subject: [PATCH 17/98] More Helming... --- charts/brig/templates/_helpers.tpl | 9 +++++++++ charts/brig/templates/cassandra-secret.yaml | 2 +- charts/brig/templates/configmap.yaml | 4 ++-- charts/brig/templates/deployment.yaml | 11 ++++++----- charts/galley/templates/_helpers.tpl | 9 +++++++++ charts/galley/templates/cassandra-secret.yaml | 2 +- charts/galley/templates/configmap.yaml | 4 ++-- charts/galley/templates/deployment.yaml | 10 +++++----- charts/gundeck/templates/_helpers.tpl | 9 +++++++++ charts/gundeck/templates/cassandra-secret.yaml | 2 +- charts/gundeck/templates/configmap.yaml | 4 ++-- charts/gundeck/templates/deployment.yaml | 11 ++++++----- charts/spar/templates/_helpers.tpl | 10 +++++++++- charts/spar/templates/cassandra-secret.yaml | 2 +- charts/spar/templates/configmap.yaml | 4 ++-- charts/spar/templates/deployment.yaml | 11 ++++++----- 16 files changed, 71 insertions(+), 33 deletions(-) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index 762fb52c2f..380e622055 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -7,3 +7,12 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} +{{- end -}} + +{{- define "useCassandraCA" -}} +{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} +{{- end -}} diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index 217e2702ba..656f4405e9 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -1,5 +1,5 @@ +{{- if (include "useCassandraCA" .Values.config) }} {{- with .Values.cassandra }} -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index e1c7c210bf..044fba8f2b 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -28,9 +28,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + {{- if (include "useCassandraTLS" .)}} useTLS: true - {{- if (hasKey .cassandra.tls "ca") }} + {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/brig/cassandra/ca.pem {{- end }} {{- else }} diff --git a/charts/brig/templates/deployment.yaml b/charts/brig/templates/deployment.yaml index 7892a543b5..b27b1a5959 100644 --- a/charts/brig/templates/deployment.yaml +++ b/charts/brig/templates/deployment.yaml @@ -46,6 +46,11 @@ spec: - name: "geoip" emptyDir: {} {{- end }} + {{- if (include "useCassandraCA" .Values.config) }} + - name: "brig-cassandra" + secret: + secretName: "brig-cassandra" + {{- end}} {{- if .Values.config.geoip.enabled }} # Brig needs GeoIP database to be downloaded before it can start. initContainers: @@ -102,11 +107,7 @@ spec: - name: "geoip" mountPath: "/usr/share/GeoIP" {{- end }} - {{- if and - (hasKey .Values.config.cassandra "tls") - (.Values.config.cassandra.tls.enabled) - (hasKey .Values.config.cassandra.tls "ca") - -}} + {{- if (include "useCassandraCA" .Values.config) }} - name: "brig-cassandra" mountPath: "/etc/wire/brig/cassandra" {{- end }} diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index 762fb52c2f..380e622055 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -7,3 +7,12 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} +{{- end -}} + +{{- define "useCassandraCA" -}} +{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} +{{- end -}} diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml index 4ae6538ba3..8d694e2db2 100644 --- a/charts/galley/templates/cassandra-secret.yaml +++ b/charts/galley/templates/cassandra-secret.yaml @@ -1,5 +1,5 @@ +{{- if (include "useCassandraCA" .Values.config)}} {{- with .Values.cassandra }} -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 2cd6d758b8..72b3714b94 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -21,9 +21,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + {{- if (include "useCassandraTLS" .) }} useTLS: true - {{- if (hasKey .cassandra.tls "ca" ) }} + {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/galley/cassandra/ca.pem {{- end }} {{- else }} diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml index cdc86df9e1..a1b07c7866 100644 --- a/charts/galley/templates/deployment.yaml +++ b/charts/galley/templates/deployment.yaml @@ -36,11 +36,7 @@ spec: - name: "galley-secrets" secret: secretName: "galley" - {{- if and - (hasKey .Values.config.cassandra "tls") - (.Values.config.cassandra.tls.enabled) - (hasKey .Values.config.cassandra.tls "ca") - -}} + {{- if (include "useCassandraCA" .Values.config) }} - name: "galley-cassandra" mountPath: "/etc/wire/galley/cassandra" {{- end }} @@ -57,6 +53,10 @@ spec: mountPath: "/etc/wire/galley/conf" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" + {{- if (include "useCassandraCA" .Values.config)}} + - name: "galley-cassandra" + mountPath: "/etc/wire/galley/cassandra" + {{- end }} env: {{- if hasKey .Values.secrets "awsKeyId" }} - name: AWS_ACCESS_KEY_ID diff --git a/charts/gundeck/templates/_helpers.tpl b/charts/gundeck/templates/_helpers.tpl index 762fb52c2f..380e622055 100644 --- a/charts/gundeck/templates/_helpers.tpl +++ b/charts/gundeck/templates/_helpers.tpl @@ -7,3 +7,12 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} +{{- end -}} + +{{- define "useCassandraCA" -}} +{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} +{{- end -}} diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml index cdfb3c11b3..be38727ca6 100644 --- a/charts/gundeck/templates/cassandra-secret.yaml +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -1,5 +1,5 @@ +{{- if (include "useCassandraCA" .Values.config)}} {{- with .Values.cassandra }} -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index 239d90d7a0..25c19addb7 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -25,9 +25,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + {{- if (include "useCassandraTLS" .) }} useTLS: true - {{- if (hasKey .cassandra.tls "ca" ) }} + {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/gundeck/cassandra/ca.pem {{- end }} {{- else }} diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml index 37eeab6617..bedd0e5962 100644 --- a/charts/gundeck/templates/deployment.yaml +++ b/charts/gundeck/templates/deployment.yaml @@ -32,6 +32,11 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "gundeck-cassandra" + secret: + secretName: "gundeck-cassandra" + {{- end}} containers: - name: gundeck image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -43,11 +48,7 @@ spec: volumeMounts: - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" - {{- if and - (hasKey .Values.config.cassandra "tls") - (.Values.config.cassandra.tls.enabled) - (hasKey .Values.config.cassandra.tls "ca") - -}} + {{- if (include "useCassandraCA" .Values.config) }} - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} diff --git a/charts/spar/templates/_helpers.tpl b/charts/spar/templates/_helpers.tpl index 762fb52c2f..d1fb1f875e 100644 --- a/charts/spar/templates/_helpers.tpl +++ b/charts/spar/templates/_helpers.tpl @@ -1,4 +1,3 @@ - {{/* Allow KubeVersion to be overridden. */}} {{- define "kubeVersion" -}} {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} @@ -7,3 +6,12 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} +{{- end -}} + +{{- define "useCassandraCA" -}} +{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} +{{- end -}} diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml index 973c4e92b6..36684bb705 100644 --- a/charts/spar/templates/cassandra-secret.yaml +++ b/charts/spar/templates/cassandra-secret.yaml @@ -1,5 +1,5 @@ +{{- if (include "useCassandraCA" .Values.config) }} {{- with .Values.cassandra }} -{{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) (hasKey .Values.cassandra.tls "ca") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index f526c1eab0..c35b4add1e 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -25,9 +25,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if and (hasKey .cassandra "tls") (.cassandra.tls.enabled) }} + {{- if (include "useCassandraTLS" .) }} useTLS: true - {{- if (hasKey .cassandra.tls "ca") }} + {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/spar/cassandra/ca.pem {{- end }} {{- else }} diff --git a/charts/spar/templates/deployment.yaml b/charts/spar/templates/deployment.yaml index 09bafe5b74..26a1145af5 100644 --- a/charts/spar/templates/deployment.yaml +++ b/charts/spar/templates/deployment.yaml @@ -30,6 +30,11 @@ spec: - name: "spar-config" configMap: name: "spar" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "spar-cassandra" + secret: + secretName: "spar-cassandra" + {{- end}} containers: - name: spar image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -41,11 +46,7 @@ spec: volumeMounts: - name: "spar-config" mountPath: "/etc/wire/spar/conf" - {{- if and - (hasKey .Values.config.cassandra "tls") - (.Values.config.cassandra.tls.enabled) - (hasKey .Values.config.cassandra.tls "ca") - -}} + {{- if (include "useCassandraCA" .Values.config) }} - name: "spar-cassandra" mountPath: "/etc/wire/spar/cassandra" {{- end }} From a01386a6959558bb8f6c45327a3090332842f611 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 14 Nov 2023 19:30:43 +0100 Subject: [PATCH 18/98] Helming SSL support for the check-cluster-job --- .../templates/cassandra-client-ca.yaml | 10 ++++++++++ .../templates/check-cluster-job.yaml | 14 ++++++++++++++ charts/k8ssandra-test-cluster/values.yaml | 3 +++ 3 files changed, 27 insertions(+) create mode 100644 charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml diff --git a/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml b/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml new file mode 100644 index 0000000000..adc19a5fa9 --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml @@ -0,0 +1,10 @@ +{{- if .Values.client_encryption_options.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: cassandra-client-ca + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca: {{ .Values.client_encryption_options.ca | quote }} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml index b91292e276..fba1100d24 100644 --- a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml +++ b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml @@ -12,7 +12,21 @@ spec: containers: - name: cassandra image: cassandra:3.11 + {{- if not .Values.client_encryption_options.enabled }} command: ["cqlsh", "k8ssandra-cluster-datacenter-1-service"] + {{- else }} + command: ["cqlsh", "k8ssandra-cluster-datacenter-1-service", "--ssl"] + env: + - name: SSL_CERTFILE + value: "/certs/ca.pem" + volumeMounts: + - name: cassandra-cert + mountPath: "/certs/ca.pem" + volumes: + - name: cassandra-cert + secret: + secretName: cassandra-client-ca + {{- end }} restartPolicy: OnFailure # Default is 6 retries. 8 is a bit arbitrary, but should be sufficient for # low resource environments (e.g. Wire-in-a-box.) diff --git a/charts/k8ssandra-test-cluster/values.yaml b/charts/k8ssandra-test-cluster/values.yaml index 18a7e3a40e..b9d9e9ca49 100644 --- a/charts/k8ssandra-test-cluster/values.yaml +++ b/charts/k8ssandra-test-cluster/values.yaml @@ -21,3 +21,6 @@ client_encryption_options: keystorePassword: password truststore: "" truststorePassword: password + + # TODO: This could be deduced from the keystore. + ca: "" From 1ec0d60133df612442ed85b276ba4299ec58ad10 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 15 Nov 2023 08:53:30 +0100 Subject: [PATCH 19/98] check-cluster-job: Fix cqlsh command line args --- charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml index fba1100d24..ffe8bfc248 100644 --- a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml +++ b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml @@ -15,7 +15,7 @@ spec: {{- if not .Values.client_encryption_options.enabled }} command: ["cqlsh", "k8ssandra-cluster-datacenter-1-service"] {{- else }} - command: ["cqlsh", "k8ssandra-cluster-datacenter-1-service", "--ssl"] + command: ["cqlsh", "--ssl", "k8ssandra-cluster-datacenter-1-service"] env: - name: SSL_CERTFILE value: "/certs/ca.pem" From 449f06c5652f64d6c06db6296c6779e41de16a93 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 15 Nov 2023 08:56:50 +0100 Subject: [PATCH 20/98] Fix cassandra-secret yaml files (wrong context) --- charts/brig/templates/cassandra-secret.yaml | 4 +--- charts/galley/templates/cassandra-secret.yaml | 4 +--- charts/gundeck/templates/cassandra-secret.yaml | 4 +--- charts/spar/templates/cassandra-secret.yaml | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index 656f4405e9..5b480bec3f 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -1,5 +1,4 @@ {{- if (include "useCassandraCA" .Values.config) }} -{{- with .Values.cassandra }} apiVersion: v1 kind: Secret metadata: @@ -11,6 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .ca | b64enc | quote }} -{{- end }} + ca.pem: {{ .Values.config.cassandra.tls.ca | quote }} {{- end }} diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml index 8d694e2db2..eae8b753b1 100644 --- a/charts/galley/templates/cassandra-secret.yaml +++ b/charts/galley/templates/cassandra-secret.yaml @@ -1,5 +1,4 @@ {{- if (include "useCassandraCA" .Values.config)}} -{{- with .Values.cassandra }} apiVersion: v1 kind: Secret metadata: @@ -11,6 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .ca | b64enc | quote }} -{{- end }} + ca.pem: {{ .Values.config.cassandra.tls.ca | quote }} {{- end }} diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml index be38727ca6..b1e7bdd608 100644 --- a/charts/gundeck/templates/cassandra-secret.yaml +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -1,5 +1,4 @@ {{- if (include "useCassandraCA" .Values.config)}} -{{- with .Values.cassandra }} apiVersion: v1 kind: Secret metadata: @@ -11,6 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .ca | b64enc | quote }} -{{- end }} + ca.pem: {{ .Values.config.cassandra.tls.ca | quote }} {{- end }} diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml index 36684bb705..18db296404 100644 --- a/charts/spar/templates/cassandra-secret.yaml +++ b/charts/spar/templates/cassandra-secret.yaml @@ -1,5 +1,4 @@ {{- if (include "useCassandraCA" .Values.config) }} -{{- with .Values.cassandra }} apiVersion: v1 kind: Secret metadata: @@ -11,6 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .ca | b64enc | quote }} -{{- end }} + ca.pem: {{ .Values.config.cassandra.tls.ca | quote }} {{- end }} From 85bf11d2470284c061886e8ee4f2bdb362e86ea2 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 15 Nov 2023 10:38:21 +0100 Subject: [PATCH 21/98] brig: Add some debug logs for TLS cert file path --- services/brig/src/Brig/App.hs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index be8fb4ac0d..89c851c017 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -431,7 +431,7 @@ initCassandra o g = do (Cas.initialContactsPlain (Opt.cassandra o ^. endpoint . host)) (Cas.initialContactsDisco "cassandra_brig" . unpack) (Opt.discoUrl o) - mbSSLContext <- createSSLContext (Opt.cassandra o) + mbSSLContext <- createSSLContext (Opt.cassandra o) g let basicCasSettings = Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.brig") g)) . Cas.setContacts (NE.head c) (NE.tail c) @@ -449,11 +449,21 @@ initCassandra o g = do runClient p $ versionCheck schemaVersion pure p where - createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) - createSSLContext cassOpts + -- TODO: Re-consider logging + createSSLContext :: CassandraOpts -> Logger -> IO (Maybe OpenSSL.SSLContext) + createSSLContext cassOpts logger | cassOpts ^. useTLS = do sslContext <- OpenSSL.context - maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) + let mbTlsCertPath = cassOpts ^. tlsCert + void . liftIO $ Log.debug logger (Log.msg ("TLS cert file path: " <> show mbTlsCertPath)) + maybe + (pure ()) + ( \certFile -> do + fileExists <- doesFileExist certFile + void . liftIO $ Log.debug logger (Log.msg ("TLS cert file exists: " <> show fileExists)) + OpenSSL.contextSetCAFile sslContext certFile + ) + mbTlsCertPath OpenSSL.contextSetVerificationMode sslContext OpenSSL.VerifyPeer From 47e7efa0b05c52f77bcc9a53d29ba2931b5ed8b0 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 15 Nov 2023 18:47:34 +0100 Subject: [PATCH 22/98] Make galley-migrate-data -threaded This is required by HsOpenSSL. --- services/galley/galley.cabal | 1 + 1 file changed, 1 insertion(+) diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index 36ec4c267d..2837c35e65 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -556,6 +556,7 @@ executable galley-integration executable galley-migrate-data import: common-all main-is: ../main.hs + ghc-options: -threaded -- cabal-fmt: expand migrate-data/src other-modules: From 480a5db2f8bb3c19e6c6fdfc92e744340abce77a Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 15 Nov 2023 18:54:40 +0100 Subject: [PATCH 23/98] Happy Helming --- .../templates/cassandra-certs.yaml | 4 ++-- .../templates/galley-migrate-data.yaml | 6 ++++-- .../templates/spar-migrate-data.yaml | 6 ++++-- .../elasticsearch-index/templates/_helpers.tpl | 9 +++++++++ .../templates/cassandra-secret.yaml | 14 ++++++++++++++ .../templates/migrate-data.yaml | 17 +++++++++++++++++ charts/elasticsearch-index/values.yaml | 3 +++ charts/galley/templates/deployment.yaml | 3 ++- 8 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 charts/elasticsearch-index/templates/cassandra-secret.yaml diff --git a/charts/cassandra-migrations/templates/cassandra-certs.yaml b/charts/cassandra-migrations/templates/cassandra-certs.yaml index ca47c6a8bc..e3cdcb0dd3 100644 --- a/charts/cassandra-migrations/templates/cassandra-certs.yaml +++ b/charts/cassandra-migrations/templates/cassandra-certs.yaml @@ -28,7 +28,7 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" annotations: - "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-weight": "0" "helm.sh/hook-delete-policy": hook-succeeded,hook-failed type: Opaque @@ -66,7 +66,7 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" annotations: - "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-weight": "0" "helm.sh/hook-delete-policy": hook-succeeded,hook-failed type: Opaque diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index 9f8864c159..34ff4b693f 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -43,8 +43,10 @@ spec: - --cassandra-keyspace - galley {{- if (include "useTlsCertGalley" .) }} - - --use-tls - - --tls-certificate-file /certs/galley/ca.pem + # TODO: This option does not exist, yet + # - --use-tls + - --tls-certificate-file + - /certs/galley/ca.pem {{- end }} {{- if (include "useTlsCertGalley" .) }} volumeMounts: diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index bb248b4c14..0e2af4a73f 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -45,8 +45,10 @@ spec: - brig # TODO: This is odd because we also need to talk to brig db {{- if (include "useTlsCertSpar" .) }} - - --use-tls - - --tls-certificate-file /certs/spar/ca.pem + # TODO: This option does not exist, yet + # - --use-tls + - --tls-certificate-file + - /certs/spar/ca.pem {{- end }} {{- if (include "useTlsCertSpar" .) }} volumeMounts: diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl index 762fb52c2f..380e622055 100644 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ b/charts/elasticsearch-index/templates/_helpers.tpl @@ -7,3 +7,12 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} +{{- end -}} + +{{- define "useCassandraCA" -}} +{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} +{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} +{{- end -}} diff --git a/charts/elasticsearch-index/templates/cassandra-secret.yaml b/charts/elasticsearch-index/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..a096d6dd90 --- /dev/null +++ b/charts/elasticsearch-index/templates/cassandra-secret.yaml @@ -0,0 +1,14 @@ +{{- if (include "useCassandraCA" .Values) }} +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-index-migrate-cassandra-client-ca + labels: + app: elasticsearch-index-migrate-data + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.cassandra.tls.ca | quote }} +{{- end }} diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index 3ef47bcf5e..b8031c7b93 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -43,3 +43,20 @@ spec: - "{{ required "missing elasticsearch-index.galley.host!" .Values.galley.host }}" - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" + {{- if (include "useTlsCertGundeck" .) }} + # - TODO: This option does not yet exists in brig-index (Brig.Index.Options) + # use-tls + - --tls-certificate-file + - /certs/ca.pem + {{- end }} + {{- if (include "useCassandraCA" .Values) }} + volumeMounts: + - name: elasticsearch-index-migrate-cassandra-client-ca + mountPath: "/certs" + {{- end }} + {{- if (include "useCassandraCA" .Values) }} + volumes: + - name: elasticsearch-index-migrate-cassandra-client-ca + secret: + secretName: elasticsearch-index-migrate-cassandra-client-ca + {{- end}} diff --git a/charts/elasticsearch-index/values.yaml b/charts/elasticsearch-index/values.yaml index 4cbd2e5110..7d9dfd864d 100644 --- a/charts/elasticsearch-index/values.yaml +++ b/charts/elasticsearch-index/values.yaml @@ -8,6 +8,9 @@ cassandra: # host: port: 9042 keyspace: brig +# tls: +# enabled: false +# ca: CA in PEM format (can be self-signed) galley: host: galley port: 8080 diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml index a1b07c7866..7829dc45a8 100644 --- a/charts/galley/templates/deployment.yaml +++ b/charts/galley/templates/deployment.yaml @@ -38,7 +38,8 @@ spec: secretName: "galley" {{- if (include "useCassandraCA" .Values.config) }} - name: "galley-cassandra" - mountPath: "/etc/wire/galley/cassandra" + secret: + secretName: galley-cassandra {{- end }} containers: - name: galley From 4d63cd3fdcc70de875e903240fe19e709486dfe6 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 16 Nov 2023 17:19:04 +0100 Subject: [PATCH 24/98] Formatting --- services/galley/galley.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index 2837c35e65..90839461da 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -556,7 +556,7 @@ executable galley-integration executable galley-migrate-data import: common-all main-is: ../main.hs - ghc-options: -threaded + ghc-options: -threaded -- cabal-fmt: expand migrate-data/src other-modules: From 3c0e036e30d551fc0282feadde85d813a2f1fc1c Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 16 Nov 2023 17:19:20 +0100 Subject: [PATCH 25/98] spar-data-migrate: Get the TLS cert file per target database --- services/spar/migrate-data/src/Spar/DataMigration/Options.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs index 677abbf098..84d6445f9e 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs @@ -72,8 +72,8 @@ cassandraSettingsParser ks = ) ) <*> ( (optional . strOption) - ( long "tls-certificate-file" - <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ( long ("tls-certificate-file-" ++ ks) + <> help ("Location of a PEM encoded list of CA certificates to be used when verifying" ++ ks ++ "'s Cassandra server's certificate") <> showDefault ) ) From 2f429c06b3d7335a4f9215dc14577de07978d803 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 16 Nov 2023 18:10:29 +0100 Subject: [PATCH 26/98] Happy helming: Give spar-migrate-data access to certs for both cassandra connections --- .../templates/cassandra-certs.yaml | 2 +- .../templates/spar-migrate-data.yaml | 22 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/charts/cassandra-migrations/templates/cassandra-certs.yaml b/charts/cassandra-migrations/templates/cassandra-certs.yaml index e3cdcb0dd3..6bdefed3e5 100644 --- a/charts/cassandra-migrations/templates/cassandra-certs.yaml +++ b/charts/cassandra-migrations/templates/cassandra-certs.yaml @@ -9,7 +9,7 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" annotations: - "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-weight": "0" "helm.sh/hook-delete-policy": hook-succeeded,hook-failed type: Opaque diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index 0e2af4a73f..3644eb17e7 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -43,20 +43,34 @@ spec: - "9042" - --cassandra-keyspace-brig - brig - # TODO: This is odd because we also need to talk to brig db + {{- if (include "useTlsCertBrig" .) }} + # TODO: This option does not exist, yet + # - --use-tls + - --tls-certificate-file-brig + - /certs/brig/ca.pem + {{- end }} {{- if (include "useTlsCertSpar" .) }} # TODO: This option does not exist, yet # - --use-tls - - --tls-certificate-file + - --tls-certificate-file-spar - /certs/spar/ca.pem {{- end }} - {{- if (include "useTlsCertSpar" .) }} volumeMounts: + {{- if (include "useTlsCertBrig" .) }} + - name: brig-cassandra-cert + mountPath: "/certs/brig" + {{- end }} + {{- if (include "useTlsCertSpar" .) }} - name: spar-cassandra-cert mountPath: "/certs/spar" {{- end }} - {{- if (include "useTlsCertSpar" .) }} volumes: + {{- if (include "useTlsCertBrig" .) }} + - name: brig-cassandra-cert + secret: + secretName: brig-cassandra-cert + {{- end }} + {{- if (include "useTlsCertSpar" .) }} - name: spar-cassandra-cert secret: secretName: spar-cassandra-cert From 471dc4804e3a4d64fb4893feeb870397e0d3a34a Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 17 Nov 2023 16:19:08 +0100 Subject: [PATCH 27/98] Happy helming: Fix check-cluster-job --- .../k8ssandra-test-cluster/templates/cassandra-client-ca.yaml | 2 +- charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml b/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml index adc19a5fa9..e8d5ef5cea 100644 --- a/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml +++ b/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml @@ -6,5 +6,5 @@ metadata: namespace: {{ .Release.Namespace }} type: Opaque data: - ca: {{ .Values.client_encryption_options.ca | quote }} + ca.pem: {{ .Values.client_encryption_options.ca | quote }} {{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml index ffe8bfc248..f7ccdf264d 100644 --- a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml +++ b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml @@ -21,7 +21,7 @@ spec: value: "/certs/ca.pem" volumeMounts: - name: cassandra-cert - mountPath: "/certs/ca.pem" + mountPath: "/certs" volumes: - name: cassandra-cert secret: From 9fb06a9483dcef328d7e107e8ce79840174e5fd9 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 11:47:22 +0100 Subject: [PATCH 28/98] Delete trace logs --- libs/cassandra-util/src/Cassandra/Util.hs | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index f58e5494d9..3f35e42243 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -32,7 +32,6 @@ import Data.Time (UTCTime, nominalDiffTimeToSeconds) import Data.Time.Clock (secondsToNominalDiffTime) import Data.Time.Clock.POSIX import Database.CQL.IO.Tinylog qualified as CT -import Debug.Trace import Imports hiding (init) import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log @@ -51,7 +50,6 @@ defInitCassandra ks h p mbCertPath lg = do where createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) createSSLContext (Just tlsCertPath) = do - traceM $ "cassandra-util: " ++ show tlsCertPath sslContext <- OpenSSL.context OpenSSL.contextSetCAFile sslContext tlsCertPath OpenSSL.contextSetVerificationMode From 3cbf8b2fe97a36c362a512cfeda84023d0a769e1 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 16:37:00 +0100 Subject: [PATCH 29/98] Provide one function to create C* connections for services This reduces duplication a lot! --- libs/cassandra-util/src/Cassandra/Util.hs | 70 ++++++++++++++++++++++- services/brig/src/Brig/App.hs | 67 +++++----------------- services/galley/src/Galley/App.hs | 54 ++++------------- services/gundeck/gundeck.cabal | 1 - services/gundeck/src/Gundeck/Env.hs | 54 +++++------------ services/spar/spar.cabal | 1 - services/spar/src/Spar/Run.hs | 60 +++++-------------- 7 files changed, 119 insertions(+), 188 deletions(-) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index 3f35e42243..e29b46ab16 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -17,20 +17,23 @@ module Cassandra.Util ( defInitCassandra, + initCassandraForService, Writetime (..), writetimeToInt64, ) where -import Cassandra (ClientState, init) import Cassandra.CQL -import Cassandra.Settings (defSettings, setContacts, setKeyspace, setLogger, setPortNumber, setSSLContext) +import Cassandra.Schema +import Cassandra.Settings (dcFilterPolicyIfConfigured, initialContactsDisco, initialContactsPlain, mkLogger) import Data.Aeson import Data.Fixed -import Data.Text (unpack) +import Data.List.NonEmpty qualified as NE +import Data.Text (pack, unpack) import Data.Time (UTCTime, nominalDiffTimeToSeconds) import Data.Time.Clock (secondsToNominalDiffTime) import Data.Time.Clock.POSIX +import Database.CQL.IO import Database.CQL.IO.Tinylog qualified as CT import Imports hiding (init) import OpenSSL.Session qualified as OpenSSL @@ -62,6 +65,67 @@ defInitCassandra ks h p mbCertPath lg = do pure $ Just sslContext createSSLContext Nothing = pure Nothing +-- | Create Cassandra `ClientState` ("connection") for a service +-- +-- Unfortunately, we have to deal with many function arguments here, because +-- @CassandraOpts@ is defined in @types-common@ which depends on +-- @cassandra-util@ (this package.) +initCassandraForService :: + Text -> + Word16 -> + String -> + Text -> + Maybe FilePath -> + Maybe Text -> + Maybe Text -> + Maybe Int32 -> + Log.Logger -> + IO ClientState +initCassandraForService host port serviceName keyspace mbTlsCertPath filterNodesByDatacentre discoUrl mbSchemaVersion logger = do + c <- + maybe + (initialContactsPlain host) + (initialContactsDisco ("cassandra_" ++ serviceName) . unpack) + discoUrl + mbSSLContext <- createSSLContext mbTlsCertPath + let basicCasSettings = + setLogger (mkLogger (Log.clone (Just (pack ("cassandra." ++ serviceName))) logger)) + . setContacts (NE.head c) (NE.tail c) + . setPortNumber (fromIntegral port) + . setKeyspace (Keyspace keyspace) + . setMaxConnections 4 + . setPoolStripes 4 + . setSendTimeout 3 + -- TODO: setMaxStreams needed? + . setResponseTimeout 10 + . setProtocolVersion V4 + . setPolicy (dcFilterPolicyIfConfigured logger filterNodesByDatacentre) + $ defSettings + casSettings = maybe basicCasSettings (\sslCtx -> setSSLContext sslCtx basicCasSettings) mbSSLContext + p <- init casSettings + maybe (pure ()) (\v -> runClient p $ (versionCheck v)) mbSchemaVersion + pure p + where + -- TODO: Re-consider logging + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just certFile) = do + void . liftIO $ Log.debug logger (Log.msg ("TLS cert file path: " <> show certFile)) + fileExists <- doesFileExist certFile + void . liftIO $ Log.debug logger (Log.msg ("TLS cert file exists: " <> show fileExists)) + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext certFile + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = True, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = do + void . liftIO $ Log.debug logger (Log.msg ("No TLS cert file path configured." :: Text)) + pure Nothing + -- | Read cassandra's writetimes https://docs.datastax.com/en/dse/5.1/cql/cql/cql_using/useWritetime.html -- as UTCTime values without any loss of precision newtype Writetime a = Writetime {writetimeToUTC :: UTCTime} diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index 89c851c017..545c5f7d00 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -106,10 +106,9 @@ import Brig.User.Search.Index (IndexEnv (..), MonadIndexIO (..), runIndexIO) import Brig.User.Template import Brig.ZAuth (MonadZAuth (..), runZAuth) import Brig.ZAuth qualified as ZAuth -import Cassandra (Keyspace (Keyspace), runClient) +import Cassandra (runClient) import Cassandra qualified as Cas -import Cassandra.Schema (versionCheck) -import Cassandra.Settings qualified as Cas +import Cassandra.Util (initCassandraForService) import Control.AutoUpdate import Control.Error import Control.Exception.Enclosed (handleAny) @@ -120,12 +119,10 @@ import Data.ByteString.Conversion import Data.Domain import Data.GeoIP2 qualified as GeoIp import Data.IP -import Data.List.NonEmpty qualified as NE import Data.Metrics (Metrics) import Data.Metrics.Middleware qualified as Metrics import Data.Misc import Data.Qualified -import Data.Text (unpack) import Data.Text qualified as Text import Data.Text.Encoding (encodeUtf8) import Data.Text.Encoding qualified as Text @@ -141,7 +138,6 @@ import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.OpenSSL import OpenSSL.EVP.Digest (Digest, getDigestByName) import OpenSSL.Session (SSLOption (..)) -import OpenSSL.Session qualified as OpenSSL import OpenSSL.Session qualified as SSL import Polysemy import Polysemy.Final @@ -425,54 +421,17 @@ initExtGetManager = do in verifyRsaFingerprint sha pinset initCassandra :: Opts -> Logger -> IO Cas.ClientState -initCassandra o g = do - c <- - maybe - (Cas.initialContactsPlain (Opt.cassandra o ^. endpoint . host)) - (Cas.initialContactsDisco "cassandra_brig" . unpack) - (Opt.discoUrl o) - mbSSLContext <- createSSLContext (Opt.cassandra o) g - let basicCasSettings = - Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.brig") g)) - . Cas.setContacts (NE.head c) (NE.tail c) - . Cas.setPortNumber (fromIntegral (Opt.cassandra o ^. endpoint . port)) - . Cas.setKeyspace (Keyspace (Opt.cassandra o ^. keyspace)) - . Cas.setMaxConnections 4 - . Cas.setPoolStripes 4 - . Cas.setSendTimeout 3 - . Cas.setResponseTimeout 10 - . Cas.setProtocolVersion Cas.V4 - . Cas.setPolicy (Cas.dcFilterPolicyIfConfigured g (Opt.cassandra o ^. filterNodesByDatacentre)) - $ Cas.defSettings - casSettings = maybe basicCasSettings (\sslCtx -> Cas.setSSLContext sslCtx basicCasSettings) mbSSLContext - p <- Cas.init casSettings - runClient p $ versionCheck schemaVersion - pure p - where - -- TODO: Re-consider logging - createSSLContext :: CassandraOpts -> Logger -> IO (Maybe OpenSSL.SSLContext) - createSSLContext cassOpts logger - | cassOpts ^. useTLS = do - sslContext <- OpenSSL.context - let mbTlsCertPath = cassOpts ^. tlsCert - void . liftIO $ Log.debug logger (Log.msg ("TLS cert file path: " <> show mbTlsCertPath)) - maybe - (pure ()) - ( \certFile -> do - fileExists <- doesFileExist certFile - void . liftIO $ Log.debug logger (Log.msg ("TLS cert file exists: " <> show fileExists)) - OpenSSL.contextSetCAFile sslContext certFile - ) - mbTlsCertPath - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - | otherwise = pure Nothing +initCassandra o g = + initCassandraForService + (Opt.cassandra o ^. endpoint . host) + (Opt.cassandra o ^. endpoint . port) + "brig" + (Opt.cassandra o ^. keyspace) + (Opt.cassandra o ^. tlsCert) + (Opt.cassandra o ^. filterNodesByDatacentre) + (Opt.discoUrl o) + (Just schemaVersion) + g initCredentials :: (FromJSON a) => FilePathSecrets -> IO a initCredentials secretFile = do diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs index a80a0044c8..8f0cf488c0 100644 --- a/services/galley/src/Galley/App.hs +++ b/services/galley/src/Galley/App.hs @@ -46,16 +46,13 @@ where import Bilge hiding (Request, header, host, options, port, statusCode, statusMessage) import Cassandra hiding (Set) -import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Util (initCassandraForService) import Control.Error hiding (err) import Control.Lens hiding ((.=)) -import Data.List.NonEmpty qualified as NE import Data.Metrics.Middleware import Data.Misc import Data.Qualified import Data.Range -import Data.Text (unpack) import Data.Time.Clock import Galley.API.Error import Galley.Aws qualified as Aws @@ -94,7 +91,6 @@ import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.OpenSSL import Network.Wai.Utilities.JSONResponse import OpenSSL.Session as Ssl -import OpenSSL.Session qualified as OpenSSL import Polysemy import Polysemy.Error import Polysemy.Input @@ -174,43 +170,17 @@ createEnv m o l = do <*> pure codeURIcfg initCassandra :: Opts -> Logger -> IO ClientState -initCassandra o l = do - c <- - maybe - (C.initialContactsPlain (o ^. cassandra . endpoint . host)) - (C.initialContactsDisco "cassandra_galley" . unpack) - (o ^. discoUrl) - mbSSLContext <- createSSLContext (o ^. cassandra) - let basicCasSettings = - C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.galley") l)) - . C.setContacts (NE.head c) (NE.tail c) - . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) - . C.setKeyspace (Keyspace $ o ^. cassandra . keyspace) - . C.setMaxConnections 4 - . C.setMaxStreams 128 - . C.setPoolStripes 4 - . C.setSendTimeout 3 - . C.setResponseTimeout 10 - . C.setProtocolVersion C.V4 - . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) - $ C.defSettings - casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext - C.init casSettings - where - createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) - createSSLContext cassOpts - | cassOpts ^. useTLS = do - sslContext <- OpenSSL.context - maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - | otherwise = pure Nothing +initCassandra o l = + initCassandraForService + (o ^. cassandra . endpoint . host) + (o ^. cassandra . endpoint . port) + "galley" + (o ^. cassandra . keyspace) + (o ^. cassandra . tlsCert) + (o ^. cassandra . filterNodesByDatacentre) + (o ^. discoUrl) + Nothing + l initHttpManager :: Opts -> IO Manager initHttpManager o = do diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index 669cfdb396..ce1b2c82ac 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -130,7 +130,6 @@ library , extra >=1.1 , gundeck-types >=1.0 , hedis >=0.14.0 - , HsOpenSSL , http-client >=0.7 , http-client-tls >=0.3 , http-types >=0.8 diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index c4056fcc89..b7e0f3ebf8 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -20,14 +20,12 @@ module Gundeck.Env where import Bilge hiding (host, port) -import Cassandra (ClientState, Keyspace (..)) -import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra (ClientState) +import Cassandra.Util (initCassandraForService) import Control.AutoUpdate import Control.Concurrent.Async (Async) import Control.Lens (makeLenses, (^.)) import Control.Retry (capDelay, exponentialBackoff) -import Data.List.NonEmpty qualified as NE import Data.Metrics.Middleware (Metrics) import Data.Misc (Milliseconds (..)) import Data.Text (unpack) @@ -43,7 +41,6 @@ import Gundeck.ThreadBudget import Imports import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.TLS (tlsManagerSettings) -import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log import System.Logger.Extended qualified as Logger import Util.Options @@ -70,11 +67,6 @@ schemaVersion = 7 createEnv :: Metrics -> Opts -> IO ([Async ()], Env) createEnv m o = do l <- Logger.mkLogger (o ^. logLevel) (o ^. logNetStrings) (o ^. logFormat) - c <- - maybe - (C.initialContactsPlain (o ^. cassandra . endpoint . host)) - (C.initialContactsDisco "cassandra_gundeck" . unpack) - (o ^. discoUrl) n <- newManager tlsManagerSettings @@ -91,23 +83,18 @@ createEnv m o = do (rAddThread, rAdd) <- createRedisPool l additionalRedis "additional-write-redis" pure ([rAddThread], Just rAdd) - mbSSLContext <- createSSLContext (o ^. cassandra) - let basicCasSettings = - C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.gundeck") l)) - . C.setContacts (NE.head c) (NE.tail c) - . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) - . C.setKeyspace (Keyspace (o ^. cassandra . keyspace)) - . C.setMaxConnections 4 - . C.setMaxStreams 128 - . C.setPoolStripes 4 - . C.setSendTimeout 3 - . C.setResponseTimeout 10 - . C.setProtocolVersion C.V4 - . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) - $ C.defSettings - casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext + p <- + initCassandraForService + (o ^. cassandra . endpoint . host) + (o ^. cassandra . endpoint . port) + "gundeck" + (o ^. cassandra . keyspace) + (o ^. cassandra . tlsCert) + (o ^. cassandra . filterNodesByDatacentre) + (o ^. discoUrl) + Nothing + l - p <- C.init casSettings a <- Aws.mkEnv l o n io <- mkAutoUpdate @@ -116,21 +103,6 @@ createEnv m o = do } mtbs <- mkThreadBudgetState `mapM` (o ^. settings . maxConcurrentNativePushes) pure $! (rThread : rAdditionalThreads,) $! Env (RequestId "N/A") m o l n p r rAdditional a io mtbs - where - createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) - createSSLContext cassOpts - | cassOpts ^. useTLS = do - sslContext <- OpenSSL.context - maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - | otherwise = pure Nothing reqIdMsg :: RequestId -> Logger.Msg -> Logger.Msg reqIdMsg = ("request" Logger..=) . unRequestId diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index 72b9b0d777..8d1c981915 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -164,7 +164,6 @@ library , extended , galley-types , hscim - , HsOpenSSL , hspec , http-types , imports diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index 3805000936..06fa22e772 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -30,11 +30,9 @@ where import qualified Bilge import Cassandra as Cas -import qualified Cassandra.Schema as Cas -import qualified Cassandra.Settings as Cas +import Cassandra.Util (initCassandraForService) import Control.Lens (to, (^.)) import Data.Id -import Data.List.NonEmpty as NE import Data.Metrics.Servant (servantPrometheusMiddleware) import Data.Proxy (Proxy (Proxy)) import qualified Data.UUID as UUID @@ -45,18 +43,17 @@ import qualified Network.Wai as Wai import qualified Network.Wai.Handler.Warp as Warp import Network.Wai.Utilities.Request (lookupRequestId) import qualified Network.Wai.Utilities.Server as WU -import qualified OpenSSL.Session as OpenSSL import qualified SAML2.WebSSO as SAML import Spar.API (SparAPI, app) import Spar.App import qualified Spar.Data as Data import Spar.Data.Instances () -import Spar.Options +import Spar.Options as Opt import Spar.Orphans () import System.Logger (Logger, msg, val, (.=), (~~)) import qualified System.Logger as Log import qualified System.Logger.Extended as Log -import Util.Options (CassandraOpts, endpoint, filterNodesByDatacentre, host, keyspace, port, tlsCert, useTLS) +import Util.Options import Wire.API.Routes.Version.Wai import Wire.Sem.Logger.TinyLog @@ -64,46 +61,17 @@ import Wire.Sem.Logger.TinyLog -- cassandra initCassandra :: Opts -> Logger -> IO ClientState -initCassandra opts lgr = do - let cassOpts = cassandra opts - mbSSLContext <- createSSLContext cassOpts - connectString <- - maybe - (Cas.initialContactsPlain (cassOpts ^. endpoint . host)) - (Cas.initialContactsDisco "cassandra_spar" . cs) - (discoUrl opts) - let basicCASSettings = - Cas.defSettings - & Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.spar") lgr)) - & Cas.setContacts (NE.head connectString) (NE.tail connectString) - & Cas.setPortNumber (fromIntegral $ cassOpts ^. endpoint . port) - & Cas.setKeyspace (Keyspace $ cassOpts ^. keyspace) - & Cas.setMaxConnections 4 - & Cas.setMaxStreams 128 - & Cas.setPoolStripes 4 - & Cas.setSendTimeout 3 - & Cas.setResponseTimeout 10 - & Cas.setProtocolVersion V4 - & Cas.setPolicy (Cas.dcFilterPolicyIfConfigured lgr (cassOpts ^. filterNodesByDatacentre)) - casSettings = maybe basicCASSettings (\sslCtx -> Cas.setSSLContext sslCtx basicCASSettings) mbSSLContext - cas <- Cas.init casSettings - runClient cas $ Cas.versionCheck Data.schemaVersion - pure cas - where - createSSLContext :: CassandraOpts -> IO (Maybe OpenSSL.SSLContext) - createSSLContext cassOpts - | cassOpts ^. useTLS = do - sslContext <- OpenSSL.context - maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) (cassOpts ^. tlsCert) - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - | otherwise = pure Nothing +initCassandra opts lgr = + initCassandraForService + (Opt.cassandra opts ^. endpoint . host) + (Opt.cassandra opts ^. endpoint . port) + "spar" + (Opt.cassandra opts ^. keyspace) + (Opt.cassandra opts ^. tlsCert) + (Opt.cassandra opts ^. filterNodesByDatacentre) + (Opt.discoUrl opts) + (Just Data.schemaVersion) + lgr ---------------------------------------------------------------------- -- servant / wai / warp From 68a84bb5cd08f6a4c44f6f0462b3d22a064d58d6 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 17:16:34 +0100 Subject: [PATCH 30/98] Use defInitCassandra to reduce duplication --- libs/cassandra-util/src/Cassandra/Util.hs | 1 + services/galley/galley.cabal | 1 - .../migrate-data/src/Galley/DataMigration.hs | 36 +++++-------------- 3 files changed, 9 insertions(+), 29 deletions(-) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index e29b46ab16..d8469ee240 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -47,6 +47,7 @@ defInitCassandra ks h p mbCertPath lg = do . setPortNumber (fromIntegral p) . setContacts (unpack h) [] . setKeyspace (Keyspace ks) + . setProtocolVersion V4 $ defSettings casSettings = maybe basicCasSettings (\sslCtx -> setSSLContext sslCtx basicCasSettings) mbSSLContext init casSettings diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index 90839461da..6d63451aed 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -576,7 +576,6 @@ executable galley-migrate-data , exceptions , extended , galley-types - , HsOpenSSL , imports , lens , optparse-applicative diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 33a16b3db8..9101ff1cd7 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -18,13 +18,12 @@ module Galley.DataMigration (cassandraSettingsParser, migrate) where import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Util (defInitCassandra) import Control.Monad.Catch (finally) import Data.Text qualified as Text import Data.Time (UTCTime, getCurrentTime) import Galley.DataMigration.Types import Imports -import OpenSSL.Session qualified as OpenSSL import Options.Applicative (Parser) import Options.Applicative qualified as Opts import System.Logger.Class (Logger) @@ -76,34 +75,15 @@ mkEnv l cas = <$> initCassandra <*> initLogger where - initCassandra = do - mbSSLContext <- createSSLContext (cTlsCert cas) - let basicCasSettings = - C.setLogger (C.mkLogger l) - . C.setContacts (cHost cas) [] - . C.setPortNumber (fromIntegral (cPort cas)) - . C.setKeyspace (cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings - casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext - - C.init casSettings + initCassandra = + defInitCassandra + ((C.unKeyspace . cKeyspace) cas) + ((Text.pack . cHost) cas) + (cPort cas) + (cTlsCert cas) + l initLogger = pure l - createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) - createSSLContext (Just tlsCertPath) = do - sslContext <- OpenSSL.context - OpenSSL.contextSetCAFile sslContext tlsCertPath - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - createSSLContext Nothing = pure Nothing - -- | Runs only the migrations which need to run runMigrations :: [Migration] -> MigrationActionT IO () runMigrations migrations = do From a22f268e3157551e0daded3fea9d6218d5b8daab Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 17:25:44 +0100 Subject: [PATCH 31/98] Use defInitCassandra; reduce duplication --- .../src/Spar/DataMigration/Run.hs | 37 +++++-------------- services/spar/spar.cabal | 1 - 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index da331713a7..d9035d2e44 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -19,14 +19,14 @@ module Spar.DataMigration.Run where +import Cassandra (ClientState) import qualified Cassandra as C -import qualified Cassandra.Settings as C +import Cassandra.Util (defInitCassandra) import Control.Lens import Control.Monad.Catch (finally) import qualified Data.Text as Text import Data.Time (UTCTime, getCurrentTime) import Imports -import qualified OpenSSL.Session as OpenSSL import qualified Options.Applicative as Opts import Spar.DataMigration.Options (settingsParser) import Spar.DataMigration.Types @@ -66,31 +66,14 @@ mkEnv settings = do (if s ^. setDebug == Debug then Log.Debug else Log.Info) $ Log.defSettings - initCassandra cas l = do - mbSSLContext <- createSSLContext (cas ^. tlsCert) - let basicCasSettings = - C.setLogger (C.mkLogger l) - . C.setContacts (cas ^. cHosts) [] - . C.setPortNumber (fromIntegral $ cas ^. cPort) - . C.setKeyspace (cas ^. cKeyspace) - . C.setProtocolVersion C.V4 - $ C.defSettings - casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext - C.init casSettings - - createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) - createSSLContext (Just tlsCertPath) = do - sslContext <- OpenSSL.context - OpenSSL.contextSetCAFile sslContext tlsCertPath - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - createSSLContext Nothing = pure Nothing + initCassandra :: CassandraSettings -> Log.Logger -> IO ClientState + initCassandra cas l = + defInitCassandra + (C.unKeyspace (cas ^. cKeyspace)) + (Text.pack (cas ^. cHosts)) + (cas ^. cPort) + (cas ^. tlsCert) + l cleanup :: (MonadIO m) => Env -> m () cleanup env = do diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index 8d1c981915..548c1f2cef 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -459,7 +459,6 @@ executable spar-migrate-data , conduit , containers , exceptions - , HsOpenSSL , imports , lens , optparse-applicative From 9a319a19bef117bb007aeb982b51b5fcc9c13de5 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 17:30:48 +0100 Subject: [PATCH 32/98] Use defInitCassandra --- services/brig/src/Brig/Index/Migrations.hs | 36 +++++----------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/services/brig/src/Brig/Index/Migrations.hs b/services/brig/src/Brig/Index/Migrations.hs index 30dbeed84e..e7f659c36a 100644 --- a/services/brig/src/Brig/Index/Migrations.hs +++ b/services/brig/src/Brig/Index/Migrations.hs @@ -24,17 +24,15 @@ import Brig.Index.Migrations.Types import Brig.Index.Options qualified as Opts import Brig.User.Search.Index qualified as Search import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Util (defInitCassandra) import Control.Lens (view, (^.)) import Control.Monad.Catch (MonadThrow, catchAll, finally, throwM) import Data.Aeson (Value, object, (.=)) import Data.Metrics qualified as Metrics import Data.Text qualified as Text import Database.Bloodhound qualified as ES -import Debug.Trace import Imports import Network.HTTP.Client qualified as HTTP -import OpenSSL.Session qualified as OpenSSL import System.Logger.Class (Logger) import System.Logger.Class qualified as Log import System.Logger.Extended (runWithLogger) @@ -89,32 +87,12 @@ mkEnv l es cas galleyEndpoint = do <*> pure galleyEndpoint where initCassandra = - do - mbSSLContext <- createSSLContext (cas ^. Opts.cTlsCert) - let basicCasSettings = - C.setLogger (C.mkLogger l) - . C.setContacts (view Opts.cHost cas) [] - . C.setPortNumber (fromIntegral (view Opts.cPort cas)) - . C.setKeyspace (view Opts.cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings - casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext - C.init casSettings - - createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) - createSSLContext (Just tlsCertPath) = do - traceM $ "brig-index: " ++ show tlsCertPath - sslContext <- OpenSSL.context - OpenSSL.contextSetCAFile sslContext tlsCertPath - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - createSSLContext Nothing = pure Nothing + defInitCassandra + (C.unKeyspace (cas ^. Opts.cKeyspace)) + (Text.pack (cas ^. Opts.cHost)) + (cas ^. Opts.cPort) + (cas ^. Opts.cTlsCert) + l initLogger = pure l From 7d0813da5aeddc597c66f9fe8cff7c0d24fd2aaf Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 17:56:31 +0100 Subject: [PATCH 33/98] use defInitCassandra to reduce duplication --- services/brig/src/Brig/Index/Eval.hs | 38 +++++++--------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/services/brig/src/Brig/Index/Eval.hs b/services/brig/src/Brig/Index/Eval.hs index bd4c5af9b0..bcd2025b7f 100644 --- a/services/brig/src/Brig/Index/Eval.hs +++ b/services/brig/src/Brig/Index/Eval.hs @@ -26,18 +26,17 @@ import Brig.Index.Migrations import Brig.Index.Options import Brig.User.Search.Index import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Util (defInitCassandra) import Control.Lens import Control.Monad.Catch import Control.Retry import Data.Aeson (FromJSON) import Data.Aeson qualified as Aeson import Data.Metrics qualified as Metrics +import Data.Text qualified as Text import Database.Bloodhound qualified as ES -import Debug.Trace import Imports import Network.HTTP.Client as HTTP -import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log import System.Logger.Class (Logger, MonadLogger (..)) @@ -103,32 +102,13 @@ runCommand l = \case <*> pure mgr initES esURI mgr = ES.mkBHEnv (toESServer esURI) mgr - initDb cas = do - mbSSLContext <- createSSLContext (cas ^. cTlsCert) - let basicCasSettings = - C.setLogger (C.mkLogger l) - . C.setContacts (view cHost cas) [] - . C.setPortNumber (fromIntegral (view cPort cas)) - . C.setKeyspace (view cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings - casSettings = maybe basicCasSettings (\sslCtx -> C.setSSLContext sslCtx basicCasSettings) mbSSLContext - C.init casSettings - - createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) - createSSLContext (Just tlsCertPath) = do - traceM $ "brig-index eval: " ++ show tlsCertPath - sslContext <- OpenSSL.context - OpenSSL.contextSetCAFile sslContext tlsCertPath - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - createSSLContext Nothing = pure Nothing + initDb cas = + defInitCassandra + (C.unKeyspace (cas ^. cKeyspace)) + (Text.pack (cas ^. cHost)) + (cas ^. cPort) + (cas ^. cTlsCert) + l waitForTaskToComplete :: forall a m. (ES.MonadBH m, MonadThrow m, FromJSON a) => Int -> ES.TaskNodeId -> m () waitForTaskToComplete timeoutSeconds taskNodeId = do From 52a1678d4ff29f43327cf599dacab22e305765ba Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 19:06:17 +0100 Subject: [PATCH 34/98] Refactor to use one way to create the SSLContext --- libs/cassandra-util/cassandra-util.cabal | 1 + .../src/Cassandra/MigrateSchema.hs | 133 ++++++++++++++++ libs/cassandra-util/src/Cassandra/Schema.hs | 144 +----------------- libs/cassandra-util/src/Cassandra/Util.hs | 48 +++--- services/brig/src/Brig/Schema/Run.hs | 1 + services/galley/src/Galley/Schema/Run.hs | 1 + services/gundeck/src/Gundeck/Schema/Run.hs | 1 + services/spar/src/Spar/Schema/Run.hs | 1 + 8 files changed, 157 insertions(+), 173 deletions(-) create mode 100644 libs/cassandra-util/src/Cassandra/MigrateSchema.hs diff --git a/libs/cassandra-util/cassandra-util.cabal b/libs/cassandra-util/cassandra-util.cabal index 004b8802df..4df1169fd6 100644 --- a/libs/cassandra-util/cassandra-util.cabal +++ b/libs/cassandra-util/cassandra-util.cabal @@ -15,6 +15,7 @@ library Cassandra Cassandra.CQL Cassandra.Exec + Cassandra.MigrateSchema Cassandra.Schema Cassandra.Settings Cassandra.Util diff --git a/libs/cassandra-util/src/Cassandra/MigrateSchema.hs b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs new file mode 100644 index 0000000000..49cc58c7fc --- /dev/null +++ b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs @@ -0,0 +1,133 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Cassandra.MigrateSchema (migrateSchema) where + +import Cassandra (Client, Consistency (All, One), Keyspace (Keyspace), PrepQuery, QueryString (QueryString), R, S, Version (V4), W, params, query, query1, retry, runClient, write, x1) +import Cassandra.Schema +import Cassandra.Settings (Policy, defSettings, initialContactsPlain, setConnectTimeout, setContacts, setLogger, setMaxConnections, setPolicy, setPoolStripes, setPortNumber, setProtocolVersion, setResponseTimeout, setSendTimeout) +import Cassandra.Util (initCassandra) +import Control.Retry +import Data.List.NonEmpty qualified as NonEmpty +import Data.Text (pack) +import Data.Text.Lazy (fromStrict) +import Data.Time.Clock +import Data.UUID (UUID) +import Database.CQL.IO (Policy (Policy, acceptable, current, display, hostCount, onEvent, select, setup), schema) +import Database.CQL.IO.Tinylog qualified as CT +import Imports hiding (All, fromString, init, intercalate, log) +import System.Logger qualified as Log + +-- FUTUREWORK: We could use the System.Logger.Class here in the future, but we don't have a ReaderT IO here (yet) +migrateSchema :: Log.Logger -> MigrationOpts -> [Migration] -> IO () +migrateSchema l o ms = do + hosts <- initialContactsPlain $ pack (migHost o) + let cqlSettings = + setLogger (CT.mkLogger l) + . setContacts (NonEmpty.head hosts) (NonEmpty.tail hosts) + . setPortNumber (fromIntegral $ migPort o) + . setMaxConnections 1 + . setPoolStripes 1 + -- 'migrationPolicy' ensures we only talk to one host for all queries + -- required for correct functioning of 'waitForSchemaConsistency' + . setPolicy migrationPolicy + -- use higher timeouts on schema migrations to reduce the probability + -- of a timeout happening during 'migAction' or 'metaInsert', + -- as that can lead to a state where schema migrations cannot be re-run + -- without manual action. + -- (due to e.g. "cannot create table X, already exists" errors) + . setConnectTimeout 20 + . setSendTimeout 20 + . setResponseTimeout 50 + . setProtocolVersion V4 + $ defSettings + p <- initCassandra cqlSettings o.migTLSCert l + runClient p $ do + let keyspace = Keyspace . migKeyspace $ o + when (migReset o) $ do + info "Dropping keyspace." + void $ schema (dropKeyspace keyspace) (params All ()) + createKeyspace keyspace (migRepl o) + useKeyspace keyspace + void $ schema metaCreate (params All ()) + migrations <- newer <$> schemaVersion + if null migrations + then info "No new migrations." + else info "New migrations found." + forM_ migrations $ \Migration {..} -> do + info $ "[" <> pack (show migVersion) <> "] " <> migText + migAction + now <- liftIO getCurrentTime + write metaInsert (params All (migVersion, migText, now)) + info "Waiting for schema version consistency across peers..." + waitForSchemaConsistency + info "... done waiting." + where + newer v = + dropWhile (maybe (const False) (>=) v . migVersion) + . sortBy (\x y -> migVersion x `compare` migVersion y) + $ ms + info = liftIO . Log.log l Log.Info . Log.msg + dropKeyspace :: Keyspace -> QueryString S () () + dropKeyspace (Keyspace k) = QueryString $ "drop keyspace if exists \"" <> fromStrict k <> "\"" + metaCreate :: QueryString S () () + metaCreate = "create columnfamily if not exists meta (id int, version int, descr text, date timestamp, primary key (id, version))" + metaInsert :: QueryString W (Int32, Text, UTCTime) () + metaInsert = "insert into meta (id, version, descr, date) values (1,?,?,?)" + +-- | Retrieve and compare local and peer system schema versions. +-- if they don't match, retry once per second for 30 seconds +waitForSchemaConsistency :: Client () +waitForSchemaConsistency = do + void $ retryWhileN 30 inDisagreement getSystemVersions + where + getSystemVersions :: Client (UUID, [UUID]) + getSystemVersions = do + -- These two sub-queries must be made to the same node. + -- (comparing local from node A and peers from node B wouldn't be correct) + -- using the custom 'migrationPolicy' when connecting to cassandra ensures this. + mbLocalVersion <- systemLocalVersion + peers <- systemPeerVersions + case mbLocalVersion of + Just localVersion -> pure $ (localVersion, peers) + Nothing -> error "No system_version in system.local (should never happen)" + inDisagreement :: (UUID, [UUID]) -> Bool + inDisagreement (localVersion, peers) = not $ all (== localVersion) peers + systemLocalVersion :: Client (Maybe UUID) + systemLocalVersion = fmap runIdentity <$> qry + where + qry = retry x1 (query1 cql (params One ())) + cql :: PrepQuery R () (Identity UUID) + cql = "select schema_version from system.local" + systemPeerVersions :: Client [UUID] + systemPeerVersions = fmap runIdentity <$> qry + where + qry = retry x1 (query cql (params One ())) + cql :: PrepQuery R () (Identity UUID) + cql = "select schema_version from system.peers" + +retryWhileN :: (MonadIO m) => Int -> (a -> Bool) -> m a -> m a +retryWhileN n f m = + retrying + (constantDelay 1000000 <> limitRetries n) + (const (pure . f)) + (const m) + +-- | The migrationPolicy selects only one and always the same host +migrationPolicy :: IO Policy +migrationPolicy = do + h <- newIORef Nothing + pure $ + Policy + { setup = setHost h, + onEvent = const $ pure (), + select = readIORef h, + acceptable = const $ pure True, + hostCount = fromIntegral . length . maybeToList <$> readIORef h, + display = ("migrationPolicy: " ++) . show <$> readIORef h, + current = maybeToList <$> readIORef h + } + where + setHost h (a : _) _ = writeIORef h (Just a) + setHost _ _ _ = pure () diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index 21af85acfc..0233342fd6 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -1,6 +1,5 @@ {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} -- for ReplicationStrategy {-# OPTIONS_GHC -Wno-partial-fields #-} @@ -34,34 +33,23 @@ module Cassandra.Schema versionCheck, createKeyspace, useKeyspace, - migrateSchema, migrationOptsParser, schema', ) where -import Cassandra (Client, Consistency (All, One), Keyspace (Keyspace), PrepQuery, QueryParams (QueryParams), QueryString (QueryString), R, S, Version (V4), W, params, query, query1, retry, runClient, write, x1, x5) -import Cassandra qualified as CQL (init) -import Cassandra.Settings (Policy, defSettings, initialContactsPlain, setConnectTimeout, setContacts, setLogger, setMaxConnections, setPolicy, setPoolStripes, setPortNumber, setProtocolVersion, setResponseTimeout, setSSLContext, setSendTimeout) +import Cassandra (Client, Consistency (All, One), Keyspace (Keyspace), QueryParams (QueryParams), QueryString (QueryString), params, query1, retry, x5) import Control.Monad.Catch -import Control.Retry import Data.Aeson -import Data.List.NonEmpty qualified as NonEmpty import Data.List.Split (splitOn) import Data.Text (intercalate, pack) import Data.Text.Lazy (fromStrict) import Data.Text.Lazy qualified as LT import Data.Text.Lazy.Builder (fromString, fromText, toLazyText) -import Data.Time.Clock -import Data.UUID (UUID) -import Database.CQL.IO (HostResponse, Policy (Policy, acceptable, current, display, hostCount, onEvent, select, setup), getResult, request, schema) -import Database.CQL.IO.Tinylog qualified as CT +import Database.CQL.IO (HostResponse, getResult, request, schema) import Database.CQL.Protocol (Query (Query), Request (RqQuery)) import Imports hiding (All, fromString, init, intercalate, log) -import OpenSSL.Session qualified as OpenSSL import Options.Applicative hiding (info) --- FUTUREWORK: We could use the System.Logger.Class here in the future, but we don't have a ReaderT IO here (yet) -import System.Logger qualified as Log data Migration = Migration { migVersion :: Int32, @@ -166,134 +154,6 @@ useKeyspace (Keyspace k) = void . getResult =<< qry prms = QueryParams One False () Nothing Nothing Nothing Nothing cql = QueryString $ "use \"" <> fromStrict k <> "\"" -migrateSchema :: Log.Logger -> MigrationOpts -> [Migration] -> IO () -migrateSchema l o ms = do - mbSSLContext <- createSSLContext - hosts <- initialContactsPlain $ pack (migHost o) - let basicCQLSettings = - setLogger (CT.mkLogger l) - . setContacts (NonEmpty.head hosts) (NonEmpty.tail hosts) - . setPortNumber (fromIntegral $ migPort o) - . setMaxConnections 1 - . setPoolStripes 1 - -- 'migrationPolicy' ensures we only talk to one host for all queries - -- required for correct functioning of 'waitForSchemaConsistency' - . setPolicy migrationPolicy - -- use higher timeouts on schema migrations to reduce the probability - -- of a timeout happening during 'migAction' or 'metaInsert', - -- as that can lead to a state where schema migrations cannot be re-run - -- without manual action. - -- (due to e.g. "cannot create table X, already exists" errors) - . setConnectTimeout 20 - . setSendTimeout 20 - . setResponseTimeout 50 - . setProtocolVersion V4 - $ defSettings - cqlSettings = maybe basicCQLSettings (\sslCtx -> setSSLContext sslCtx basicCQLSettings) mbSSLContext - p <- CQL.init cqlSettings - runClient p $ do - let keyspace = Keyspace . migKeyspace $ o - when (migReset o) $ do - info "Dropping keyspace." - void $ schema (dropKeyspace keyspace) (params All ()) - createKeyspace keyspace (migRepl o) - useKeyspace keyspace - void $ schema metaCreate (params All ()) - migrations <- newer <$> schemaVersion - if null migrations - then info "No new migrations." - else info "New migrations found." - forM_ migrations $ \Migration {..} -> do - info $ "[" <> pack (show migVersion) <> "] " <> migText - migAction - now <- liftIO getCurrentTime - write metaInsert (params All (migVersion, migText, now)) - info "Waiting for schema version consistency across peers..." - waitForSchemaConsistency - info "... done waiting." - where - newer v = - dropWhile (maybe (const False) (>=) v . migVersion) - . sortBy (\x y -> migVersion x `compare` migVersion y) - $ ms - info = liftIO . Log.log l Log.Info . Log.msg - dropKeyspace :: Keyspace -> QueryString S () () - dropKeyspace (Keyspace k) = QueryString $ "drop keyspace if exists \"" <> fromStrict k <> "\"" - metaCreate :: QueryString S () () - metaCreate = "create columnfamily if not exists meta (id int, version int, descr text, date timestamp, primary key (id, version))" - metaInsert :: QueryString W (Int32, Text, UTCTime) () - metaInsert = "insert into meta (id, version, descr, date) values (1,?,?,?)" - createSSLContext :: IO (Maybe OpenSSL.SSLContext) - createSSLContext - | o.migUseTLS = do - sslContext <- OpenSSL.context - maybe (pure ()) (OpenSSL.contextSetCAFile sslContext) o.migTLSCert - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - | otherwise = pure Nothing - --- | Retrieve and compare local and peer system schema versions. --- if they don't match, retry once per second for 30 seconds -waitForSchemaConsistency :: Client () -waitForSchemaConsistency = do - void $ retryWhileN 30 inDisagreement getSystemVersions - where - getSystemVersions :: Client (UUID, [UUID]) - getSystemVersions = do - -- These two sub-queries must be made to the same node. - -- (comparing local from node A and peers from node B wouldn't be correct) - -- using the custom 'migrationPolicy' when connecting to cassandra ensures this. - mbLocalVersion <- systemLocalVersion - peers <- systemPeerVersions - case mbLocalVersion of - Just localVersion -> pure $ (localVersion, peers) - Nothing -> error "No system_version in system.local (should never happen)" - inDisagreement :: (UUID, [UUID]) -> Bool - inDisagreement (localVersion, peers) = not $ all (== localVersion) peers - systemLocalVersion :: Client (Maybe UUID) - systemLocalVersion = fmap runIdentity <$> qry - where - qry = retry x1 (query1 cql (params One ())) - cql :: PrepQuery R () (Identity UUID) - cql = "select schema_version from system.local" - systemPeerVersions :: Client [UUID] - systemPeerVersions = fmap runIdentity <$> qry - where - qry = retry x1 (query cql (params One ())) - cql :: PrepQuery R () (Identity UUID) - cql = "select schema_version from system.peers" - -retryWhileN :: (MonadIO m) => Int -> (a -> Bool) -> m a -> m a -retryWhileN n f m = - retrying - (constantDelay 1000000 <> limitRetries n) - (const (pure . f)) - (const m) - --- | The migrationPolicy selects only one and always the same host -migrationPolicy :: IO Policy -migrationPolicy = do - h <- newIORef Nothing - pure $ - Policy - { setup = setHost h, - onEvent = const $ pure (), - select = readIORef h, - acceptable = const $ pure True, - hostCount = fromIntegral . length . maybeToList <$> readIORef h, - display = ("migrationPolicy: " ++) . show <$> readIORef h, - current = maybeToList <$> readIORef h - } - where - setHost h (a : _) _ = writeIORef h (Just a) - setHost _ _ _ = pure () - migrationOptsParser :: Parser MigrationOpts migrationOptsParser = MigrationOpts diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index d8469ee240..431f107679 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -18,6 +18,7 @@ module Cassandra.Util ( defInitCassandra, initCassandraForService, + initCassandra, Writetime (..), writetimeToInt64, ) @@ -40,31 +41,15 @@ import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log defInitCassandra :: Text -> Text -> Word16 -> Maybe FilePath -> Log.Logger -> IO ClientState -defInitCassandra ks h p mbCertPath lg = do - mbSSLContext <- createSSLContext mbCertPath +defInitCassandra ks h p mbTlsCertPath logger = do let basicCasSettings = - setLogger (CT.mkLogger lg) + setLogger (CT.mkLogger logger) . setPortNumber (fromIntegral p) . setContacts (unpack h) [] . setKeyspace (Keyspace ks) . setProtocolVersion V4 $ defSettings - casSettings = maybe basicCasSettings (\sslCtx -> setSSLContext sslCtx basicCasSettings) mbSSLContext - init casSettings - where - createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) - createSSLContext (Just tlsCertPath) = do - sslContext <- OpenSSL.context - OpenSSL.contextSetCAFile sslContext tlsCertPath - OpenSSL.contextSetVerificationMode - sslContext - OpenSSL.VerifyPeer - { vpFailIfNoPeerCert = True, - vpClientOnce = True, - vpCallback = Nothing - } - pure $ Just sslContext - createSSLContext Nothing = pure Nothing + initCassandra basicCasSettings mbTlsCertPath logger -- | Create Cassandra `ClientState` ("connection") for a service -- @@ -88,7 +73,6 @@ initCassandraForService host port serviceName keyspace mbTlsCertPath filterNodes (initialContactsPlain host) (initialContactsDisco ("cassandra_" ++ serviceName) . unpack) discoUrl - mbSSLContext <- createSSLContext mbTlsCertPath let basicCasSettings = setLogger (mkLogger (Log.clone (Just (pack ("cassandra." ++ serviceName))) logger)) . setContacts (NE.head c) (NE.tail c) @@ -102,17 +86,19 @@ initCassandraForService host port serviceName keyspace mbTlsCertPath filterNodes . setProtocolVersion V4 . setPolicy (dcFilterPolicyIfConfigured logger filterNodesByDatacentre) $ defSettings - casSettings = maybe basicCasSettings (\sslCtx -> setSSLContext sslCtx basicCasSettings) mbSSLContext - p <- init casSettings + p <- initCassandra basicCasSettings mbTlsCertPath logger maybe (pure ()) (\v -> runClient p $ (versionCheck v)) mbSchemaVersion pure p + +initCassandra :: Settings -> Maybe FilePath -> Log.Logger -> IO ClientState +initCassandra settings (Just tlsCertPath) logger = do + sslContext <- createSSLContext tlsCertPath + let settings' = setSSLContext sslContext settings + init settings' where - -- TODO: Re-consider logging - createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) - createSSLContext (Just certFile) = do + createSSLContext :: FilePath -> IO OpenSSL.SSLContext + createSSLContext certFile = do void . liftIO $ Log.debug logger (Log.msg ("TLS cert file path: " <> show certFile)) - fileExists <- doesFileExist certFile - void . liftIO $ Log.debug logger (Log.msg ("TLS cert file exists: " <> show fileExists)) sslContext <- OpenSSL.context OpenSSL.contextSetCAFile sslContext certFile OpenSSL.contextSetVerificationMode @@ -122,10 +108,10 @@ initCassandraForService host port serviceName keyspace mbTlsCertPath filterNodes vpClientOnce = True, vpCallback = Nothing } - pure $ Just sslContext - createSSLContext Nothing = do - void . liftIO $ Log.debug logger (Log.msg ("No TLS cert file path configured." :: Text)) - pure Nothing + pure sslContext +initCassandra settings Nothing logger = do + void . liftIO $ Log.debug logger (Log.msg ("No TLS cert file path configured." :: Text)) + init settings -- | Read cassandra's writetimes https://docs.datastax.com/en/dse/5.1/cql/cql/cql_using/useWritetime.html -- as UTCTime values without any loss of precision diff --git a/services/brig/src/Brig/Schema/Run.hs b/services/brig/src/Brig/Schema/Run.hs index 15f996f73b..049a51e5f5 100644 --- a/services/brig/src/Brig/Schema/Run.hs +++ b/services/brig/src/Brig/Schema/Run.hs @@ -56,6 +56,7 @@ import Brig.Schema.V78_ClientLastActive qualified as V78_ClientLastActive import Brig.Schema.V79_ConnectionRemoteIndex qualified as V79_ConnectionRemoteIndex import Brig.Schema.V80_KeyPackageCiphersuite qualified as V80_KeyPackageCiphersuite import Brig.Schema.V81_AddFederationRemoteTeams qualified as V81_AddFederationRemoteTeams +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Imports diff --git a/services/galley/src/Galley/Schema/Run.hs b/services/galley/src/Galley/Schema/Run.hs index 5291020674..91ae2c3185 100644 --- a/services/galley/src/Galley/Schema/Run.hs +++ b/services/galley/src/Galley/Schema/Run.hs @@ -17,6 +17,7 @@ module Galley.Schema.Run where +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Galley.Schema.V20 qualified as V20 diff --git a/services/gundeck/src/Gundeck/Schema/Run.hs b/services/gundeck/src/Gundeck/Schema/Run.hs index 056363c244..ccec5141e4 100644 --- a/services/gundeck/src/Gundeck/Schema/Run.hs +++ b/services/gundeck/src/Gundeck/Schema/Run.hs @@ -17,6 +17,7 @@ module Gundeck.Schema.Run where +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Gundeck.Schema.V1 qualified as V1 diff --git a/services/spar/src/Spar/Schema/Run.hs b/services/spar/src/Spar/Schema/Run.hs index 4fef2264a3..a853c1c13c 100644 --- a/services/spar/src/Spar/Schema/Run.hs +++ b/services/spar/src/Spar/Schema/Run.hs @@ -17,6 +17,7 @@ module Spar.Schema.Run where +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Imports From e8af93554b6b1ffd567fd619c2abc926c323748a Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 19:16:27 +0100 Subject: [PATCH 35/98] --use-tls is gone --- Makefile | 18 +++++++++--------- libs/cassandra-util/src/Cassandra/Schema.hs | 5 ----- libs/types-common/src/Util/Options.hs | 1 - 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 070d3f0df5..a445c276e2 100644 --- a/Makefile +++ b/Makefile @@ -302,15 +302,15 @@ db-reset: c # Migrate all keyspaces and reset the ES index .PHONY: db-migrate db-migrate: c - ./dist/brig-schema --keyspace brig_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/galley-schema --keyspace galley_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/spar-schema --keyspace spar_test --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 --use-tls --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/brig-schema --keyspace brig_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/galley-schema --keyspace galley_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/spar-schema --keyspace spar_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null ./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 > /dev/null ./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 > /dev/null ./integration/scripts/integration-dynamic-backends-brig-index.sh --elasticsearch-server http://localhost:9200 > /dev/null diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index 0233342fd6..5a6a800b41 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -63,7 +63,6 @@ data MigrationOpts = MigrationOpts migKeyspace :: Text, migRepl :: ReplicationStrategy, migReset :: Bool, - migUseTLS :: Bool, migTLSCert :: Maybe FilePath } deriving (Eq, Show, Generic) @@ -190,10 +189,6 @@ migrationOptsParser = ( long "reset" <> help "Reset the keyspace before running migrations" ) - <*> switch - ( long "use-tls" - <> help "Use TLS to connect to Cassandra" - ) <*> ( (optional . strOption) ( long "tls-certificate-file" <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" diff --git a/libs/types-common/src/Util/Options.hs b/libs/types-common/src/Util/Options.hs index f2d50d4bf4..bc0336dcd2 100644 --- a/libs/types-common/src/Util/Options.hs +++ b/libs/types-common/src/Util/Options.hs @@ -92,7 +92,6 @@ data CassandraOpts = CassandraOpts -- This option is most likely only necessary during a cassandra DC migration -- FUTUREWORK: remove this option again, or support a datacentre migration feature _filterNodesByDatacentre :: !(Maybe Text), - _useTLS :: Bool, _tlsCert :: Maybe FilePath } deriving (Show, Generic) From 4d6d8e8a810420c201c13fcda40be81600645dfd Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 19:21:30 +0100 Subject: [PATCH 36/98] Remove --use-tls from Helm charts --- .../cassandra-migrations/templates/galley-migrate-data.yaml | 2 -- charts/cassandra-migrations/templates/migrate-schema.yaml | 4 ---- charts/cassandra-migrations/templates/spar-migrate-data.yaml | 4 ---- 3 files changed, 10 deletions(-) diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index 34ff4b693f..701af45d8a 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -43,8 +43,6 @@ spec: - --cassandra-keyspace - galley {{- if (include "useTlsCertGalley" .) }} - # TODO: This option does not exist, yet - # - --use-tls - --tls-certificate-file - /certs/galley/ca.pem {{- end }} diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index e9830f038f..9f4aa704ca 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -63,7 +63,6 @@ spec: - {{ template "cassandraGundeckReplicationType" . }} - "{{ template "cassandraGundeckReplicationArg" . }}" {{- if (include "useTlsCertGundeck" .) }} - - --use-tls - --tls-certificate-file - /certs/gundeck/ca.pem {{- end }} @@ -94,7 +93,6 @@ spec: - {{ template "cassandraBrigReplicationType" . }} - "{{ template "cassandraBrigReplicationArg" . }}" {{- if (include "useTlsCertBrig" .) }} - - --use-tls - --tls-certificate-file - /certs/brig/ca.pem {{- end }} @@ -125,7 +123,6 @@ spec: - {{ template "cassandraGalleyReplicationType" . }} - "{{ template "cassandraGalleyReplicationArg" . }}" {{- if (include "useTlsCertGalley" .) }} - - --use-tls - --tls-certificate-file - /certs/galley/ca.pem {{- end }} @@ -156,7 +153,6 @@ spec: - {{ template "cassandraSparReplicationType" . }} - "{{ template "cassandraSparReplicationArg" . }}" {{- if (include "useTlsCertSpar" .) }} - - --use-tls - --tls-certificate-file - /certs/spar/ca.pem {{- end }} diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index 3644eb17e7..e8fcdf7067 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -44,14 +44,10 @@ spec: - --cassandra-keyspace-brig - brig {{- if (include "useTlsCertBrig" .) }} - # TODO: This option does not exist, yet - # - --use-tls - --tls-certificate-file-brig - /certs/brig/ca.pem {{- end }} {{- if (include "useTlsCertSpar" .) }} - # TODO: This option does not exist, yet - # - --use-tls - --tls-certificate-file-spar - /certs/spar/ca.pem {{- end }} From b4808b53af6084b0aeee8216c918a3ee7046d07b Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 19:27:13 +0100 Subject: [PATCH 37/98] Remove useTLS from test config files --- services/brig/brig.integration.yaml | 1 - services/galley/galley.integration.yaml | 1 - services/gundeck/gundeck.integration.yaml | 1 - services/spar/spar.integration.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index d02505cec9..5c8728d637 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -8,7 +8,6 @@ cassandra: port: 9042 keyspace: brig_test # filterNodesByDatacentre: datacenter1 - useTLS: true tlsCert: ../../hack/cassandra.cert.pem elasticsearch: diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml index c3b436459d..1baa3a0af0 100644 --- a/services/galley/galley.integration.yaml +++ b/services/galley/galley.integration.yaml @@ -8,7 +8,6 @@ cassandra: port: 9042 keyspace: galley_test # filterNodesByDatacentre: datacenter1 - useTLS: true tlsCert: ../../hack/cassandra.cert.pem brig: diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml index 0f8677f98b..aeb60f0e06 100644 --- a/services/gundeck/gundeck.integration.yaml +++ b/services/gundeck/gundeck.integration.yaml @@ -12,7 +12,6 @@ cassandra: port: 9042 keyspace: gundeck_test # filterNodesByDatacentre: datacenter1 - useTLS: true tlsCert: ../../hack/cassandra.cert.pem redis: diff --git a/services/spar/spar.integration.yaml b/services/spar/spar.integration.yaml index f7a166a73f..96eb9e91d5 100644 --- a/services/spar/spar.integration.yaml +++ b/services/spar/spar.integration.yaml @@ -28,7 +28,6 @@ cassandra: port: 9042 keyspace: spar_test filterNodesByDatacentre: datacenter1 - useTLS: true tlsCert: ../../hack/cassandra.cert.pem # Wire/AWS specific, optional From b7305a2427ded029c65f41e0225f0e92e31b1870 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 20 Nov 2023 19:28:13 +0100 Subject: [PATCH 38/98] Remove useTLS from Helm charts --- charts/brig/templates/_helpers.tpl | 5 ----- charts/brig/templates/configmap.yaml | 5 ----- charts/elasticsearch-index/templates/_helpers.tpl | 5 ----- charts/galley/templates/_helpers.tpl | 5 ----- charts/galley/templates/configmap.yaml | 5 ----- charts/gundeck/templates/_helpers.tpl | 5 ----- charts/gundeck/templates/configmap.yaml | 5 ----- charts/spar/templates/_helpers.tpl | 5 ----- charts/spar/templates/configmap.yaml | 5 ----- 9 files changed, 45 deletions(-) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index 380e622055..06877b74e5 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -8,11 +8,6 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraTLS" -}} -{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} -{{- end -}} - {{- define "useCassandraCA" -}} -{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} {{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} {{- end -}} diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 044fba8f2b..aa13c624a0 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -28,14 +28,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraTLS" .)}} - useTLS: true {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/brig/cassandra/ca.pem {{- end }} - {{- else }} - useTLS: false - {{- end }} elasticsearch: url: http://{{ .elasticsearch.host }}:{{ .elasticsearch.port }} diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl index 380e622055..06877b74e5 100644 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ b/charts/elasticsearch-index/templates/_helpers.tpl @@ -8,11 +8,6 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraTLS" -}} -{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} -{{- end -}} - {{- define "useCassandraCA" -}} -{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} {{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} {{- end -}} diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index 380e622055..06877b74e5 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -8,11 +8,6 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraTLS" -}} -{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} -{{- end -}} - {{- define "useCassandraCA" -}} -{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} {{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} {{- end -}} diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 72b3714b94..1de59df32b 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -21,14 +21,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraTLS" .) }} - useTLS: true {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/galley/cassandra/ca.pem {{- end }} - {{- else }} - useTLS: false - {{- end }} brig: host: brig diff --git a/charts/gundeck/templates/_helpers.tpl b/charts/gundeck/templates/_helpers.tpl index 380e622055..06877b74e5 100644 --- a/charts/gundeck/templates/_helpers.tpl +++ b/charts/gundeck/templates/_helpers.tpl @@ -8,11 +8,6 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraTLS" -}} -{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} -{{- end -}} - {{- define "useCassandraCA" -}} -{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} {{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} {{- end -}} diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index 25c19addb7..f5d02892e3 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -25,14 +25,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraTLS" .) }} - useTLS: true {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/gundeck/cassandra/ca.pem {{- end }} - {{- else }} - useTLS: false - {{- end }} redis: host: {{ .redis.host }} diff --git a/charts/spar/templates/_helpers.tpl b/charts/spar/templates/_helpers.tpl index d1fb1f875e..effe0492f2 100644 --- a/charts/spar/templates/_helpers.tpl +++ b/charts/spar/templates/_helpers.tpl @@ -7,11 +7,6 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraTLS" -}} -{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled }} -{{- end -}} - {{- define "useCassandraCA" -}} -{{/* The evaluation of Helm is odd: This cannot call useCassandraTLS without changing the evaluation order. */}} {{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} {{- end -}} diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index c35b4add1e..abea8b2f22 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -25,14 +25,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraTLS" .) }} - useTLS: true {{- if (include "useCassandraCA" .) }} tlsCert: /etc/wire/spar/cassandra/ca.pem {{- end }} - {{- else }} - useTLS: false - {{- end }} maxttlAuthreq: {{ .maxttlAuthreq }} maxttlAuthresp: {{ .maxttlAuthresp }} From 0d018e3a9c06457aee9ee49bdea13a3962e4535b Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 07:49:31 +0100 Subject: [PATCH 39/98] Re-generate nix files --- services/gundeck/default.nix | 1 - services/spar/default.nix | 1 - 2 files changed, 2 deletions(-) diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix index 78b071be02..3dad13b422 100644 --- a/services/gundeck/default.nix +++ b/services/gundeck/default.nix @@ -107,7 +107,6 @@ mkDerivation { extra gundeck-types hedis - HsOpenSSL http-client http-client-tls http-types diff --git a/services/spar/default.nix b/services/spar/default.nix index cc7f1b98e0..daebe1a84f 100644 --- a/services/spar/default.nix +++ b/services/spar/default.nix @@ -103,7 +103,6 @@ mkDerivation { extended galley-types hscim - HsOpenSSL hspec http-types imports From 5c8a9260e330ff84c870113c007b70bad68ffede Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 07:58:50 +0100 Subject: [PATCH 40/98] Cleanup --- charts/elasticsearch-index/templates/migrate-data.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index b8031c7b93..7672fdfd94 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -44,8 +44,6 @@ spec: - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" {{- if (include "useTlsCertGundeck" .) }} - # - TODO: This option does not yet exists in brig-index (Brig.Index.Options) - # use-tls - --tls-certificate-file - /certs/ca.pem {{- end }} From 8c8551b708c990dc33fe7150fe0321f58d653024 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 08:06:55 +0100 Subject: [PATCH 41/98] Cleaup: Delete obsolete option from values --- charts/brig/values.yaml | 1 - charts/galley/values.yaml | 1 - charts/gundeck/values.yaml | 1 - charts/spar/values.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index e58f6cdfb2..d6d964619c 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -21,7 +21,6 @@ config: cassandra: host: aws-cassandra # tls: -# enabled: false # ca: CA in PEM format (can be self-signed) elasticsearch: host: elasticsearch-client diff --git a/charts/galley/values.yaml b/charts/galley/values.yaml index 62c6151f85..17640d3ecb 100644 --- a/charts/galley/values.yaml +++ b/charts/galley/values.yaml @@ -23,7 +23,6 @@ config: host: aws-cassandra replicaCount: 3 # tls: -# enabled: false # ca: CA in PEM format (can be self-signed) enableFederation: false # keep enableFederation default in sync with brig and cargohold chart's config.enableFederation as well as wire-server chart's tags.federation # Not used if enableFederation is false diff --git a/charts/gundeck/values.yaml b/charts/gundeck/values.yaml index c0b15ac26b..054d454c1c 100644 --- a/charts/gundeck/values.yaml +++ b/charts/gundeck/values.yaml @@ -21,7 +21,6 @@ config: cassandra: host: aws-cassandra # tls: -# enabled: false # ca: CA in PEM format (can be self-signed) redis: host: redis-ephemeral-master diff --git a/charts/spar/values.yaml b/charts/spar/values.yaml index e5a2d394d9..823c58fc9c 100644 --- a/charts/spar/values.yaml +++ b/charts/spar/values.yaml @@ -18,7 +18,6 @@ config: cassandra: host: aws-cassandra # tls: -# enabled: false # ca: CA in PEM format (can be self-signed) richInfoLimit: 5000 maxScimTokens: 0 From d1613a6f178346c70a48c9b16772458549252c4d Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 08:09:35 +0100 Subject: [PATCH 42/98] Remove Debug.Trace --- integration/test/Testlib/Env.hs | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index c804af6114..cde73ae7bd 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -14,7 +14,6 @@ import Data.Set (Set) import Data.Set qualified as Set import Data.Yaml qualified as Yaml import Database.CQL.IO qualified as Cassandra -import Debug.Trace import Network.HTTP.Client qualified as HTTP import OpenSSL.Session qualified as OpenSSL import System.Directory @@ -69,7 +68,6 @@ mkGlobalEnv cfgFile = do manager <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings mbCassCertFilePath <- liftIO $ getCassCertFilePath - traceM $ "mbCassCertFilePath: " ++ show mbCassCertFilePath mbSSLContext <- liftIO $ createSSLContext mbCassCertFilePath let basicCassSettings = Cassandra.defSettings From c61cfe07ecd1f85684e65818cf3e95bec427b001 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 08:16:27 +0100 Subject: [PATCH 43/98] Federator's HTTP SSL does not belong to the C* story --- services/federator/src/Federator/Monitor/Internal.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/federator/src/Federator/Monitor/Internal.hs b/services/federator/src/Federator/Monitor/Internal.hs index 9b8c15d63a..1b6b74f84d 100644 --- a/services/federator/src/Federator/Monitor/Internal.hs +++ b/services/federator/src/Federator/Monitor/Internal.hs @@ -373,7 +373,7 @@ mkSSLContextWithoutCert settings = do SSL.contextSetVerificationMode ctx $ SSL.VerifyPeer { -- vpFailIfNoPeerCert and vpClientOnce are only relevant for servers - SSL.vpFailIfNoPeerCert = True, + SSL.vpFailIfNoPeerCert = False, SSL.vpClientOnce = False, SSL.vpCallback = Nothing } From edd806eb5dc97554b7ffb4902fb4ae08aaa59bd9 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 11:22:42 +0100 Subject: [PATCH 44/98] Add changelog --- changelog.d/2-features/cassandra-tls | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog.d/2-features/cassandra-tls diff --git a/changelog.d/2-features/cassandra-tls b/changelog.d/2-features/cassandra-tls new file mode 100644 index 0000000000..69a985a442 --- /dev/null +++ b/changelog.d/2-features/cassandra-tls @@ -0,0 +1,6 @@ +Allow the configuration of TLS-secured connections to Cassandra. TLS is used +when a certificate is provided. This is either done with +`--tls-certificate-file` for migrations or the configuration attribute +`cassandra.tlsCert` for services. In Helm charts, the certificate is provided as +PEM string in the attribute `cassandra.tlsCert` (analog to service +configuration.) From 6e2dd1c08ea10c2dbd84ce5f4172dc119f5f6f72 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 11:26:28 +0100 Subject: [PATCH 45/98] setMaxStreams isn't needed The services that used it, were using the default anyways. --- libs/cassandra-util/src/Cassandra/Util.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index 431f107679..abefc74e6f 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -81,7 +81,6 @@ initCassandraForService host port serviceName keyspace mbTlsCertPath filterNodes . setMaxConnections 4 . setPoolStripes 4 . setSendTimeout 3 - -- TODO: setMaxStreams needed? . setResponseTimeout 10 . setProtocolVersion V4 . setPolicy (dcFilterPolicyIfConfigured logger filterNodesByDatacentre) From ceb0e8f5e4511885d091984d3b8d3eef14bfa669 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 16:16:47 +0100 Subject: [PATCH 46/98] Use common Helm structure to set certs --- charts/brig/templates/_helpers.tpl | 2 +- charts/brig/templates/cassandra-secret.yaml | 2 +- charts/brig/values.yaml | 4 ++-- charts/galley/templates/_helpers.tpl | 2 +- charts/galley/templates/cassandra-secret.yaml | 2 +- charts/galley/values.yaml | 4 ++-- charts/gundeck/templates/_helpers.tpl | 2 +- charts/gundeck/templates/cassandra-secret.yaml | 2 +- charts/gundeck/values.yaml | 4 ++-- charts/spar/templates/_helpers.tpl | 2 +- charts/spar/templates/cassandra-secret.yaml | 2 +- charts/spar/values.yaml | 4 ++-- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index 06877b74e5..dc77c5a1d2 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -9,5 +9,5 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} +{{ and (hasKey .cassandra "tlsCert") }} {{- end -}} diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index 5b480bec3f..659de4e8e7 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -10,5 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .Values.config.cassandra.tls.ca | quote }} + ca.pem: {{ .Values.config.cassandra.tlsCert | quote }} {{- end }} diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index d6d964619c..ff4cabd0f0 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -20,8 +20,8 @@ config: logNetStrings: false cassandra: host: aws-cassandra -# tls: -# ca: CA in PEM format (can be self-signed) +# To enable TLS: +# tlsCert: Date: Tue, 21 Nov 2023 16:32:41 +0100 Subject: [PATCH 47/98] Rename tlsCert -> tlsCa This explains better what's expected. --- changelog.d/2-features/cassandra-tls | 4 +- charts/brig/templates/_helpers.tpl | 2 +- charts/brig/templates/cassandra-secret.yaml | 2 +- charts/brig/templates/configmap.yaml | 2 +- charts/brig/values.yaml | 2 +- .../templates/_helpers.tpl | 48 +++++++++---------- .../templates/cassandra-certs.yaml | 16 +++---- .../templates/galley-migrate-data.yaml | 6 +-- .../templates/migrate-schema.yaml | 24 +++++----- .../templates/spar-migrate-data.yaml | 12 ++--- charts/cassandra-migrations/values.yaml | 4 +- .../templates/migrate-data.yaml | 2 +- charts/galley/templates/_helpers.tpl | 2 +- charts/galley/templates/cassandra-secret.yaml | 2 +- charts/galley/templates/configmap.yaml | 2 +- charts/galley/values.yaml | 2 +- charts/gundeck/templates/_helpers.tpl | 2 +- .../gundeck/templates/cassandra-secret.yaml | 2 +- charts/gundeck/templates/configmap.yaml | 2 +- charts/gundeck/values.yaml | 2 +- charts/spar/templates/_helpers.tpl | 2 +- charts/spar/templates/cassandra-secret.yaml | 2 +- charts/spar/templates/configmap.yaml | 2 +- charts/spar/values.yaml | 2 +- integration/test/Testlib/Env.hs | 2 +- integration/test/Testlib/Types.hs | 2 +- .../src/Cassandra/MigrateSchema.hs | 2 +- libs/cassandra-util/src/Cassandra/Schema.hs | 2 +- libs/cassandra-util/src/Cassandra/Util.hs | 12 ++--- libs/types-common/src/Util/Options.hs | 2 +- services/brig/brig.integration.yaml | 2 +- services/brig/src/Brig/App.hs | 2 +- services/brig/src/Brig/Index/Eval.hs | 2 +- services/brig/src/Brig/Index/Migrations.hs | 2 +- services/brig/src/Brig/Index/Options.hs | 6 +-- services/brig/test/integration/Run.hs | 4 +- services/galley/galley.integration.yaml | 2 +- .../migrate-data/src/Galley/DataMigration.hs | 4 +- services/galley/src/Galley/App.hs | 2 +- services/galley/test/integration/Run.hs | 4 +- services/gundeck/gundeck.integration.yaml | 2 +- services/gundeck/src/Gundeck/Env.hs | 2 +- services/gundeck/test/integration/Main.hs | 4 +- services/integration.yaml | 2 +- .../src/Spar/DataMigration/Run.hs | 2 +- .../src/Spar/DataMigration/Types.hs | 2 +- services/spar/spar.integration.yaml | 2 +- services/spar/src/Spar/Run.hs | 2 +- 48 files changed, 109 insertions(+), 109 deletions(-) diff --git a/changelog.d/2-features/cassandra-tls b/changelog.d/2-features/cassandra-tls index 69a985a442..3e3195f1e7 100644 --- a/changelog.d/2-features/cassandra-tls +++ b/changelog.d/2-features/cassandra-tls @@ -1,6 +1,6 @@ Allow the configuration of TLS-secured connections to Cassandra. TLS is used when a certificate is provided. This is either done with `--tls-certificate-file` for migrations or the configuration attribute -`cassandra.tlsCert` for services. In Helm charts, the certificate is provided as -PEM string in the attribute `cassandra.tlsCert` (analog to service +`cassandra.tlsCa` for services. In Helm charts, the certificate is provided as +PEM string in the attribute `cassandra.tlsCa` (analog to service configuration.) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index dc77c5a1d2..d535de0756 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -9,5 +9,5 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tlsCert") }} +{{ and (hasKey .cassandra "tlsCa") }} {{- end -}} diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index 659de4e8e7..df584f99b9 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -10,5 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .Values.config.cassandra.tlsCert | quote }} + ca.pem: {{ .Values.config.cassandra.tlsCa | quote }} {{- end }} diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index aa13c624a0..450c2cef0a 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -29,7 +29,7 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if (include "useCassandraCA" .) }} - tlsCert: /etc/wire/brig/cassandra/ca.pem + tlsCa: /etc/wire/brig/cassandra/ca.pem {{- end }} elasticsearch: diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index ff4cabd0f0..52895ce93c 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -21,7 +21,7 @@ config: cassandra: host: aws-cassandra # To enable TLS: -# tlsCert: +# tlsCa: # # This also works for dedicated service setups. E.g. # # cassandraGalley: # host: cassandra-ephemeral-galley # replicationMap: eu-west-1:3 -# tlsCert: +# tlsCa: # Overriding the following is only useful during datacenter migration time periods, # where some other job already migrates schemas. diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index 7672fdfd94..c7282dffe7 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -43,7 +43,7 @@ spec: - "{{ required "missing elasticsearch-index.galley.host!" .Values.galley.host }}" - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" - {{- if (include "useTlsCertGundeck" .) }} + {{- if (include "useTlsCaGundeck" .) }} - --tls-certificate-file - /certs/ca.pem {{- end }} diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index dc77c5a1d2..d535de0756 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -9,5 +9,5 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tlsCert") }} +{{ and (hasKey .cassandra "tlsCa") }} {{- end -}} diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml index 85ac7ad474..04da327f99 100644 --- a/charts/galley/templates/cassandra-secret.yaml +++ b/charts/galley/templates/cassandra-secret.yaml @@ -10,5 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .Values.config.cassandra.tlsCert | quote }} + ca.pem: {{ .Values.config.cassandra.tlsCa | quote }} {{- end }} diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 1de59df32b..39c73d1b8a 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -22,7 +22,7 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if (include "useCassandraCA" .) }} - tlsCert: /etc/wire/galley/cassandra/ca.pem + tlsCa: /etc/wire/galley/cassandra/ca.pem {{- end }} brig: diff --git a/charts/galley/values.yaml b/charts/galley/values.yaml index 2137473f05..facae104e5 100644 --- a/charts/galley/values.yaml +++ b/charts/galley/values.yaml @@ -23,7 +23,7 @@ config: host: aws-cassandra replicaCount: 3 # To enable TLS: -# tlsCert: (Just <$> (makeAbsolute) (combine projectRoot certFilePath))) devEnvProjectRoot ) - intConfig.cassandra.cassTlsCert + intConfig.cassandra.cassTlsCa manager <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs index 0c38f69646..d08773d000 100644 --- a/integration/test/Testlib/Types.hs +++ b/integration/test/Testlib/Types.hs @@ -173,7 +173,7 @@ instance FromJSON HostPort data CassandraConfig = CassandraConfig { cassHost :: String, cassPort :: Word16, - cassTlsCert :: Maybe FilePath + cassTlsCa :: Maybe FilePath } deriving (Show, Generic) diff --git a/libs/cassandra-util/src/Cassandra/MigrateSchema.hs b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs index 49cc58c7fc..35f00a4da1 100644 --- a/libs/cassandra-util/src/Cassandra/MigrateSchema.hs +++ b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs @@ -42,7 +42,7 @@ migrateSchema l o ms = do . setResponseTimeout 50 . setProtocolVersion V4 $ defSettings - p <- initCassandra cqlSettings o.migTLSCert l + p <- initCassandra cqlSettings o.migTlsCa l runClient p $ do let keyspace = Keyspace . migKeyspace $ o when (migReset o) $ do diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index 5a6a800b41..fc2b2b4a10 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -63,7 +63,7 @@ data MigrationOpts = MigrationOpts migKeyspace :: Text, migRepl :: ReplicationStrategy, migReset :: Bool, - migTLSCert :: Maybe FilePath + migTlsCa :: Maybe FilePath } deriving (Eq, Show, Generic) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index abefc74e6f..c0dc51120b 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -41,7 +41,7 @@ import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log defInitCassandra :: Text -> Text -> Word16 -> Maybe FilePath -> Log.Logger -> IO ClientState -defInitCassandra ks h p mbTlsCertPath logger = do +defInitCassandra ks h p mbTlsCaPath logger = do let basicCasSettings = setLogger (CT.mkLogger logger) . setPortNumber (fromIntegral p) @@ -49,7 +49,7 @@ defInitCassandra ks h p mbTlsCertPath logger = do . setKeyspace (Keyspace ks) . setProtocolVersion V4 $ defSettings - initCassandra basicCasSettings mbTlsCertPath logger + initCassandra basicCasSettings mbTlsCaPath logger -- | Create Cassandra `ClientState` ("connection") for a service -- @@ -67,7 +67,7 @@ initCassandraForService :: Maybe Int32 -> Log.Logger -> IO ClientState -initCassandraForService host port serviceName keyspace mbTlsCertPath filterNodesByDatacentre discoUrl mbSchemaVersion logger = do +initCassandraForService host port serviceName keyspace mbTlsCaPath filterNodesByDatacentre discoUrl mbSchemaVersion logger = do c <- maybe (initialContactsPlain host) @@ -85,13 +85,13 @@ initCassandraForService host port serviceName keyspace mbTlsCertPath filterNodes . setProtocolVersion V4 . setPolicy (dcFilterPolicyIfConfigured logger filterNodesByDatacentre) $ defSettings - p <- initCassandra basicCasSettings mbTlsCertPath logger + p <- initCassandra basicCasSettings mbTlsCaPath logger maybe (pure ()) (\v -> runClient p $ (versionCheck v)) mbSchemaVersion pure p initCassandra :: Settings -> Maybe FilePath -> Log.Logger -> IO ClientState -initCassandra settings (Just tlsCertPath) logger = do - sslContext <- createSSLContext tlsCertPath +initCassandra settings (Just tlsCaPath) logger = do + sslContext <- createSSLContext tlsCaPath let settings' = setSSLContext sslContext settings init settings' where diff --git a/libs/types-common/src/Util/Options.hs b/libs/types-common/src/Util/Options.hs index bc0336dcd2..40ca142908 100644 --- a/libs/types-common/src/Util/Options.hs +++ b/libs/types-common/src/Util/Options.hs @@ -92,7 +92,7 @@ data CassandraOpts = CassandraOpts -- This option is most likely only necessary during a cassandra DC migration -- FUTUREWORK: remove this option again, or support a datacentre migration feature _filterNodesByDatacentre :: !(Maybe Text), - _tlsCert :: Maybe FilePath + _tlsCa :: Maybe FilePath } deriving (Show, Generic) diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index 5c8728d637..e7c6d74f30 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -8,7 +8,7 @@ cassandra: port: 9042 keyspace: brig_test # filterNodesByDatacentre: datacenter1 - tlsCert: ../../hack/cassandra.cert.pem + tlsCa: ../../hack/cassandra.cert.pem elasticsearch: url: http://127.0.0.1:9200 diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index 545c5f7d00..b33db9d42f 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -427,7 +427,7 @@ initCassandra o g = (Opt.cassandra o ^. endpoint . port) "brig" (Opt.cassandra o ^. keyspace) - (Opt.cassandra o ^. tlsCert) + (Opt.cassandra o ^. tlsCa) (Opt.cassandra o ^. filterNodesByDatacentre) (Opt.discoUrl o) (Just schemaVersion) diff --git a/services/brig/src/Brig/Index/Eval.hs b/services/brig/src/Brig/Index/Eval.hs index bcd2025b7f..5a6650ec85 100644 --- a/services/brig/src/Brig/Index/Eval.hs +++ b/services/brig/src/Brig/Index/Eval.hs @@ -107,7 +107,7 @@ runCommand l = \case (C.unKeyspace (cas ^. cKeyspace)) (Text.pack (cas ^. cHost)) (cas ^. cPort) - (cas ^. cTlsCert) + (cas ^. cTlsCa) l waitForTaskToComplete :: forall a m. (ES.MonadBH m, MonadThrow m, FromJSON a) => Int -> ES.TaskNodeId -> m () diff --git a/services/brig/src/Brig/Index/Migrations.hs b/services/brig/src/Brig/Index/Migrations.hs index e7f659c36a..c00550b4e5 100644 --- a/services/brig/src/Brig/Index/Migrations.hs +++ b/services/brig/src/Brig/Index/Migrations.hs @@ -91,7 +91,7 @@ mkEnv l es cas galleyEndpoint = do (C.unKeyspace (cas ^. Opts.cKeyspace)) (Text.pack (cas ^. Opts.cHost)) (cas ^. Opts.cPort) - (cas ^. Opts.cTlsCert) + (cas ^. Opts.cTlsCa) l initLogger = pure l diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index 0f64e9509a..3b91ae132e 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -31,7 +31,7 @@ module Brig.Index.Options CassandraSettings, cHost, cPort, - cTlsCert, + cTlsCa, cKeyspace, localElasticSettings, localCassandraSettings, @@ -84,7 +84,7 @@ data CassandraSettings = CassandraSettings { _cHost :: String, _cPort :: Word16, _cKeyspace :: C.Keyspace, - _cTlsCert :: Maybe FilePath + _cTlsCa :: Maybe FilePath } deriving (Show) @@ -128,7 +128,7 @@ localCassandraSettings = { _cHost = "localhost", _cPort = 9042, _cKeyspace = C.Keyspace "brig_test", - _cTlsCert = Nothing + _cTlsCa = Nothing } elasticServerParser :: Parser (URIRef Absolute) diff --git a/services/brig/test/integration/Run.hs b/services/brig/test/integration/Run.hs index 97bb68da5f..ad45070078 100644 --- a/services/brig/test/integration/Run.hs +++ b/services/brig/test/integration/Run.hs @@ -136,10 +136,10 @@ runTests iConf brigOpts otherArgs = do casHost = (\v -> Opts.cassandra v ^. endpoint . host) brigOpts casPort = (\v -> Opts.cassandra v ^. endpoint . port) brigOpts casKey = (\v -> Opts.cassandra v ^. keyspace) brigOpts - casTlsCert = (\v -> Opts.cassandra v ^. tlsCert) brigOpts + casTlsCa = (\v -> Opts.cassandra v ^. tlsCa) brigOpts awsOpts = Opts.aws brigOpts lg <- Logger.new Logger.defSettings -- TODO: use mkLogger'? - db <- defInitCassandra casKey casHost casPort casTlsCert lg + db <- defInitCassandra casKey casHost casPort casTlsCa lg mg <- newManager tlsManagerSettings let fedBrigClient = FedClient @'Brig mg (brig iConf) emailAWSOpts <- parseEmailAWSOpts diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml index 1baa3a0af0..558dd7924d 100644 --- a/services/galley/galley.integration.yaml +++ b/services/galley/galley.integration.yaml @@ -8,7 +8,7 @@ cassandra: port: 9042 keyspace: galley_test # filterNodesByDatacentre: datacenter1 - tlsCert: ../../hack/cassandra.cert.pem + tlsCa: ../../hack/cassandra.cert.pem brig: host: 0.0.0.0 diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 9101ff1cd7..92676dec1b 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -33,7 +33,7 @@ data CassandraSettings = CassandraSettings { cHost :: String, cPort :: Word16, cKeyspace :: C.Keyspace, - cTlsCert :: Maybe FilePath + cTlsCa :: Maybe FilePath } cassandraSettingsParser :: Parser CassandraSettings @@ -80,7 +80,7 @@ mkEnv l cas = ((C.unKeyspace . cKeyspace) cas) ((Text.pack . cHost) cas) (cPort cas) - (cTlsCert cas) + (cTlsCa cas) l initLogger = pure l diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs index 8f0cf488c0..ab81a87294 100644 --- a/services/galley/src/Galley/App.hs +++ b/services/galley/src/Galley/App.hs @@ -176,7 +176,7 @@ initCassandra o l = (o ^. cassandra . endpoint . port) "galley" (o ^. cassandra . keyspace) - (o ^. cassandra . tlsCert) + (o ^. cassandra . tlsCa) (o ^. cassandra . filterNodesByDatacentre) (o ^. discoUrl) Nothing diff --git a/services/galley/test/integration/Run.hs b/services/galley/test/integration/Run.hs index 4aca34ce51..4b2a3e6130 100644 --- a/services/galley/test/integration/Run.hs +++ b/services/galley/test/integration/Run.hs @@ -127,9 +127,9 @@ main = withOpenSSL $ runTests go let ch = fromJust gConf ^. cassandra . endpoint . host let cp = fromJust gConf ^. cassandra . endpoint . port let ck = fromJust gConf ^. cassandra . keyspace - let cTlsCert = fromJust gConf ^. cassandra . tlsCert + let cTlsCa = fromJust gConf ^. cassandra . tlsCa lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp cTlsCert lg + db <- defInitCassandra ck ch cp cTlsCa lg teamEventWatcher <- sequence $ SQS.watchSQSQueue <$> ((^. Aws.awsEnv) <$> awsEnv) <*> q pure $ TestSetup (fromJust gConf) (fromJust iConf) m g b c awsEnv convMaxSize db (FedClient m galleyEndpoint) teamEventWatcher queueName' = fmap (view queueName) . view journal diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml index aeb60f0e06..6571221484 100644 --- a/services/gundeck/gundeck.integration.yaml +++ b/services/gundeck/gundeck.integration.yaml @@ -12,7 +12,7 @@ cassandra: port: 9042 keyspace: gundeck_test # filterNodesByDatacentre: datacenter1 - tlsCert: ../../hack/cassandra.cert.pem + tlsCa: ../../hack/cassandra.cert.pem redis: host: 127.0.0.1 diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index b7e0f3ebf8..6d3f7bf6e4 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -89,7 +89,7 @@ createEnv m o = do (o ^. cassandra . endpoint . port) "gundeck" (o ^. cassandra . keyspace) - (o ^. cassandra . tlsCert) + (o ^. cassandra . tlsCa) (o ^. cassandra . filterNodesByDatacentre) (o ^. discoUrl) Nothing diff --git a/services/gundeck/test/integration/Main.hs b/services/gundeck/test/integration/Main.hs index 6b476be6ea..d25e2b6b01 100644 --- a/services/gundeck/test/integration/Main.hs +++ b/services/gundeck/test/integration/Main.hs @@ -115,9 +115,9 @@ main = withOpenSSL $ runTests go ch = gConf ^. cassandra . endpoint . host cp = gConf ^. cassandra . endpoint . port ck = gConf ^. cassandra . keyspace - cTlsCert = gConf ^. cassandra . tlsCert + cTlsCa = gConf ^. cassandra . tlsCa lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp cTlsCert lg + db <- defInitCassandra ck ch cp cTlsCa lg pure $ TestSetup m g c c2 b db lg gConf (redis2 iConf) releaseOpts _ = pure () mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p diff --git a/services/integration.yaml b/services/integration.yaml index ae760e477e..040a709cf1 100644 --- a/services/integration.yaml +++ b/services/integration.yaml @@ -142,4 +142,4 @@ rabbitmq: cassandra: host: 127.0.0.1 port: 9042 - tlsCert: hack/cassandra.cert.pem + tlsCa: hack/cassandra.cert.pem diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index d9035d2e44..3721ba6d20 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -72,7 +72,7 @@ mkEnv settings = do (C.unKeyspace (cas ^. cKeyspace)) (Text.pack (cas ^. cHosts)) (cas ^. cPort) - (cas ^. tlsCert) + (cas ^. tlsCa) l cleanup :: (MonadIO m) => Env -> m () diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs index 8e6da39530..6331b10159 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs @@ -63,7 +63,7 @@ data CassandraSettings = CassandraSettings { _cHosts :: !String, _cPort :: !Word16, _cKeyspace :: !C.Keyspace, - _tlsCert :: Maybe FilePath + _tlsCa :: Maybe FilePath } deriving (Show) diff --git a/services/spar/spar.integration.yaml b/services/spar/spar.integration.yaml index 96eb9e91d5..31edba552d 100644 --- a/services/spar/spar.integration.yaml +++ b/services/spar/spar.integration.yaml @@ -28,7 +28,7 @@ cassandra: port: 9042 keyspace: spar_test filterNodesByDatacentre: datacenter1 - tlsCert: ../../hack/cassandra.cert.pem + tlsCa: ../../hack/cassandra.cert.pem # Wire/AWS specific, optional # discoUrl: "https://" diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index 06fa22e772..29d49b72a7 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -67,7 +67,7 @@ initCassandra opts lgr = (Opt.cassandra opts ^. endpoint . port) "spar" (Opt.cassandra opts ^. keyspace) - (Opt.cassandra opts ^. tlsCert) + (Opt.cassandra opts ^. tlsCa) (Opt.cassandra opts ^. filterNodesByDatacentre) (Opt.discoUrl opts) (Just Data.schemaVersion) From 6c020aeda0538e5e2d887181ca9a6976f5039475 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 21 Nov 2023 18:53:40 +0100 Subject: [PATCH 48/98] Take the CA string unencoded This is a bit nicer than taking a base64 blob. --- charts/brig/templates/cassandra-secret.yaml | 2 +- charts/elasticsearch-index/templates/_helpers.tpl | 2 +- charts/elasticsearch-index/templates/cassandra-secret.yaml | 2 +- charts/elasticsearch-index/values.yaml | 5 ++--- charts/galley/templates/cassandra-secret.yaml | 2 +- charts/gundeck/templates/cassandra-secret.yaml | 2 +- .../templates/cassandra-client-ca.yaml | 2 +- charts/spar/templates/cassandra-secret.yaml | 2 +- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index df584f99b9..548cd11994 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -10,5 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .Values.config.cassandra.tlsCa | quote }} + ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }} {{- end }} diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl index 06877b74e5..d535de0756 100644 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ b/charts/elasticsearch-index/templates/_helpers.tpl @@ -9,5 +9,5 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tls") .cassandra.tls.enabled (hasKey .cassandra.tls "ca") }} +{{ and (hasKey .cassandra "tlsCa") }} {{- end -}} diff --git a/charts/elasticsearch-index/templates/cassandra-secret.yaml b/charts/elasticsearch-index/templates/cassandra-secret.yaml index a096d6dd90..564a94804a 100644 --- a/charts/elasticsearch-index/templates/cassandra-secret.yaml +++ b/charts/elasticsearch-index/templates/cassandra-secret.yaml @@ -10,5 +10,5 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - ca.pem: {{ .Values.cassandra.tls.ca | quote }} + ca.pem: {{ .Values.cassandra.tlsCa | b64enc | quote }} {{- end }} diff --git a/charts/elasticsearch-index/values.yaml b/charts/elasticsearch-index/values.yaml index 7d9dfd864d..81ee2ff043 100644 --- a/charts/elasticsearch-index/values.yaml +++ b/charts/elasticsearch-index/values.yaml @@ -8,9 +8,8 @@ cassandra: # host: port: 9042 keyspace: brig -# tls: -# enabled: false -# ca: CA in PEM format (can be self-signed) +# To enable TLS: +# tlsCa: Date: Wed, 22 Nov 2023 16:43:07 +0100 Subject: [PATCH 49/98] Let K8ssandra create the Java KeysStores And, only consume the resulting CA certificate (ca.crt). --- charts/brig/templates/_helpers.tpl | 10 ++- charts/brig/templates/cassandra-secret.yaml | 2 +- charts/brig/templates/configmap.yaml | 2 +- charts/brig/templates/deployment.yaml | 2 +- charts/brig/values.yaml | 5 ++ .../templates/_helpers.tpl | 76 +++++++++++++++---- .../templates/cassandra-certs.yaml | 8 +- .../templates/galley-migrate-data.yaml | 10 +-- .../templates/migrate-schema.yaml | 40 +++++----- .../templates/spar-migrate-data.yaml | 20 ++--- charts/cassandra-migrations/values.yaml | 17 +++++ .../templates/_helpers.tpl | 10 ++- .../templates/cassandra-secret.yaml | 2 +- .../templates/migrate-data.yaml | 6 +- charts/galley/templates/_helpers.tpl | 10 ++- charts/galley/templates/cassandra-secret.yaml | 2 +- charts/galley/templates/configmap.yaml | 2 +- charts/galley/templates/deployment.yaml | 2 +- charts/gundeck/templates/_helpers.tpl | 10 ++- .../gundeck/templates/cassandra-secret.yaml | 2 +- charts/gundeck/templates/configmap.yaml | 2 +- charts/gundeck/templates/deployment.yaml | 2 +- .../templates/check-cluster-job.yaml | 8 +- .../templates/client-encryption-stores.yaml | 13 ---- .../templates/jks-store-pass.yaml | 8 ++ .../templates/k8ssandra-cluster.yaml | 12 ++- .../templates/tls-ca-secret.yaml | 10 +++ .../templates/tls-certificate.yaml | 44 +++++++++++ .../templates/tls-issuer.yaml | 9 +++ charts/k8ssandra-test-cluster/values.yaml | 6 +- charts/spar/templates/_helpers.tpl | 10 ++- charts/spar/templates/cassandra-secret.yaml | 2 +- charts/spar/templates/configmap.yaml | 2 +- charts/spar/templates/deployment.yaml | 2 +- 34 files changed, 271 insertions(+), 97 deletions(-) delete mode 100644 charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml create mode 100644 charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml create mode 100644 charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml create mode 100644 charts/k8ssandra-test-cluster/templates/tls-certificate.yaml create mode 100644 charts/k8ssandra-test-cluster/templates/tls-issuer.yaml diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index d535de0756..1bb20ba67c 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -9,5 +9,13 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tlsCa") }} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "brig-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} {{- end -}} diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index 548cd11994..fcf1d704ae 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (include "useCassandraCA" .Values.config) }} +{{- if (hasKey .Values.config.cassandra "tlsCa") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 450c2cef0a..77a5eb22c4 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -29,7 +29,7 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if (include "useCassandraCA" .) }} - tlsCa: /etc/wire/brig/cassandra/ca.pem + tlsCa: /etc/wire/brig/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} elasticsearch: diff --git a/charts/brig/templates/deployment.yaml b/charts/brig/templates/deployment.yaml index b27b1a5959..e563159a9e 100644 --- a/charts/brig/templates/deployment.yaml +++ b/charts/brig/templates/deployment.yaml @@ -49,7 +49,7 @@ spec: {{- if (include "useCassandraCA" .Values.config) }} - name: "brig-cassandra" secret: - secretName: "brig-cassandra" + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} {{- end}} {{- if .Values.config.geoip.enabled }} # Brig needs GeoIP database to be downloaded before it can start. diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index 52895ce93c..a8845f82b6 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -22,6 +22,11 @@ config: host: aws-cassandra # To enable TLS: # tlsCa: +# key: + elasticsearch: host: elasticsearch-client port: 9200 diff --git a/charts/cassandra-migrations/templates/_helpers.tpl b/charts/cassandra-migrations/templates/_helpers.tpl index 79e46443d0..e0025dda4a 100644 --- a/charts/cassandra-migrations/templates/_helpers.tpl +++ b/charts/cassandra-migrations/templates/_helpers.tpl @@ -107,59 +107,103 @@ Thus the order of priority is: {{- end -}} {{- end -}} -{{- define "useTlsCaGalley" -}} +{{- define "useTlsGalley" -}} {{ $cassandraGalley := default dict .Values.cassandraGalley }} -{{- or .Values.cassandra.tlsCa $cassandraGalley.tlsCa -}} +{{- or .Values.cassandra.tlsCa $cassandraGalley.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGalley.tlsCaSecretRef -}} {{- end -}} {{- define "tlsCaGalley" -}} {{ $cassandraGalley := default dict .Values.cassandraGalley }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} +{{ .Values.cassandra.tlsCa | toYaml }} {{- else -}} -{{ $cassandraGalley.tlsCa }} +{{ $cassandraGalley.tlsCa | toYaml }} {{- end -}} {{- end -}} -{{- define "useTlsCaBrig" -}} +{{- define "tlsSecretRefGalley" -}} +{{ $cassandraGalley := default dict .Values.cassandraGalley }} +{{- if .Values.cassandra.tlsCaSecretRef -}} +{{ .Values.cassandra.tlsCaSecretRef | toYaml }} +{{- else if $cassandraGalley.tlsCaSecretRef -}} +{{ $cassandraGalley.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "galley-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + +{{- define "useTlsBrig" -}} {{ $cassandraBrig := default dict .Values.cassandraBrig }} -{{- or .Values.cassandra.tlsCa $cassandraBrig.tlsCa -}} +{{- or .Values.cassandra.tlsCa $cassandraBrig.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraBrig.tlsCaSecretRef -}} {{- end -}} {{- define "tlsCaBrig" -}} {{ $cassandraBrig := default dict .Values.cassandraBrig }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} +{{ .Values.cassandra.tlsCa | toYaml }} {{- else -}} -{{ $cassandraBrig.tlsCa }} +{{ $cassandraBrig.tlsCa | toYaml }} +{{- end -}} +{{- end -}} + +{{- define "tlsSecretRefBrig" -}} +{{ $cassandraBrig := default dict .Values.cassandraBrig }} +{{- if .Values.cassandra.tlsCaSecretRef -}} +{{ .Values.cassandra.tlsCaSecretRef | toYaml }} +{{- else if $cassandraBrig.tlsCaSecretRef -}} +{{ $cassandraBrig.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "brig-cassandra-cert" "key" "ca.pem" | toYaml -}} {{- end -}} {{- end -}} -{{- define "useTlsCaSpar" -}} +{{- define "useTlsSpar" -}} {{ $cassandraSpar := default dict .Values.cassandraSpar }} -{{- or .Values.cassandra.tlsCa $cassandraSpar.tlsCa -}} +{{- or .Values.cassandra.tlsCa $cassandraSpar.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraSpar.tlsCaSecretRef -}} {{- end -}} {{- define "tlsCaSpar" -}} {{ $cassandraSpar := default dict .Values.cassandraSpar }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} +{{ .Values.cassandra.tlsCa | toYaml }} {{- else -}} -{{ $cassandraSpar.tlsCa }} +{{ $cassandraSpar.tlsCa | toYaml }} {{- end -}} {{- end -}} -{{- define "useTlsCaGundeck" -}} +{{- define "tlsSecretRefSpar" -}} +{{ $cassandraSpar := default dict .Values.cassandraSpar }} +{{- if .Values.cassandra.tlsCaSecretRef -}} +{{ .Values.cassandra.tlsCaSecretRef | toYaml }} +{{- else if $cassandraSpar.tlsCaSecretRef -}} +{{ $cassandraSpar.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "spar-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + +{{- define "useTlsGundeck" -}} {{ $cassandraGundeck := default dict .Values.cassandraGundeck }} -{{- or .Values.cassandra.tlsCa $cassandraGundeck.tlsCa -}} +{{- or .Values.cassandra.tlsCa $cassandraGundeck.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGundeck.tlsCaSecretRef -}} {{- end -}} {{- define "tlsCaGundeck" -}} {{ $cassandraGundeck := default dict .Values.cassandraGundeck }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} +{{ .Values.cassandra.tlsCa | toYaml }} {{- else -}} -{{ $cassandraGundeck.tlsCa }} +{{ $cassandraGundeck.tlsCa | toYaml }} +{{- end -}} +{{- end -}} + +{{- define "tlsSecretRefGundeck" -}} +{{ $cassandraGundeck := default dict .Values.cassandraGundeck }} +{{- if .Values.cassandra.tlsCaSecretRef -}} +{{ .Values.cassandra.tlsCaSecretRef | toYaml }} +{{- else if $cassandraGundeck.tlsCaSecretRef -}} +{{ $cassandraGundeck.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "gundeck-cassandra-cert" "key" "ca.pem" | toYaml -}} {{- end -}} {{- end -}} diff --git a/charts/cassandra-migrations/templates/cassandra-certs.yaml b/charts/cassandra-migrations/templates/cassandra-certs.yaml index df4d8d1dd9..b4d5c54657 100644 --- a/charts/cassandra-migrations/templates/cassandra-certs.yaml +++ b/charts/cassandra-migrations/templates/cassandra-certs.yaml @@ -1,4 +1,4 @@ -{{- if (include "useTlsCaBrig" .) }} +{{- if (include "tlsCaBrig" . | fromYaml) }} apiVersion: v1 kind: Secret metadata: @@ -16,7 +16,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaBrig" . | b64enc | quote }} {{- end}} -{{- if (include "useTlsCaGalley" .) }} +{{- if (include "tlsCaGalley" . | fromYaml) }} --- apiVersion: v1 kind: Secret @@ -35,7 +35,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaGalley" . | b64enc | quote }} {{- end}} -{{- if (include "useTlsCaGundeck" .) }} +{{- if (include "tlsCaGundeck" . | fromYaml) }} --- apiVersion: v1 kind: Secret @@ -54,7 +54,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaGundeck" . | b64enc | quote }} {{- end}} -{{- if (include "useTlsCaSpar" .) }} +{{- if (include "tlsCaSpar" . | fromYaml) }} --- apiVersion: v1 kind: Secret diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index b04ad06876..63083ce0bf 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -42,19 +42,19 @@ spec: - "9042" - --cassandra-keyspace - galley - {{- if (include "useTlsCaGalley" .) }} + {{- if (include "useTlsGalley" .) }} - --tls-certificate-file - - /certs/galley/ca.pem + - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsCaGalley" .) }} + {{- if (include "useTlsGalley" .) }} volumeMounts: - name: galley-cassandra-cert mountPath: "/certs/galley" {{- end }} - {{- if (include "useTlsCaGalley" .) }} + {{- if (include "useTlsGalley" .) }} volumes: - name: galley-cassandra-cert secret: - secretName: galley-cassandra-cert + secretName: {{ (include "tlsSecretRefGalley" . | fromYaml).name }} {{- end }} {{- end }} diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index 6842a345c8..56544c56a1 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -23,25 +23,25 @@ spec: # to avoid 'Column family ID mismatch' / schema disagreements # see https://stackoverflow.com/questions/29030661/creating-new-table-with-cqlsh-on-existing-keyspace-column-family-id-mismatch#40325651 for details. volumes: - {{- if (include "useTlsCaGundeck" .) }} + {{- if (include "useTlsGundeck" .) }} - name: gundeck-cassandra-cert secret: - secretName: gundeck-cassandra-cert + secretName: {{ (include "tlsSecretRefGundeck" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsCaBrig" .) }} + {{- if (include "useTlsBrig" .) }} - name: brig-cassandra-cert secret: - secretName: brig-cassandra-cert + secretName: {{ (include "tlsSecretRefBrig" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsCaGalley" .) }} + {{- if (include "useTlsGalley" .) }} - name: galley-cassandra-cert secret: - secretName: galley-cassandra-cert + secretName: {{ (include "tlsSecretRefGalley" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsCaSpar" .) }} + {{- if (include "useTlsSpar" .) }} - name: spar-cassandra-cert secret: - secretName: spar-cassandra-cert + secretName: {{ (include "tlsSecretRefSpar" . | fromYaml).name }} {{- end }} initContainers: {{- if .Values.enableGundeckMigrations }} @@ -62,12 +62,12 @@ spec: - gundeck - {{ template "cassandraGundeckReplicationType" . }} - "{{ template "cassandraGundeckReplicationArg" . }}" - {{- if (include "useTlsCaGundeck" .) }} + {{- if (include "useTlsGundeck" .) }} - --tls-certificate-file - - /certs/gundeck/ca.pem + - /certs/gundeck/{{- (include "tlsSecretRefGundeck" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsCaGundeck" .) }} + {{- if (include "useTlsGundeck" .) }} volumeMounts: - name: gundeck-cassandra-cert mountPath: "/certs/gundeck" @@ -92,12 +92,12 @@ spec: - brig - {{ template "cassandraBrigReplicationType" . }} - "{{ template "cassandraBrigReplicationArg" . }}" - {{- if (include "useTlsCaBrig" .) }} + {{- if (include "useTlsBrig" .) }} - --tls-certificate-file - - /certs/brig/ca.pem + - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsCaBrig" .) }} + {{- if (include "useTlsBrig" .) }} volumeMounts: - name: brig-cassandra-cert mountPath: "/certs/brig" @@ -122,12 +122,12 @@ spec: - galley - {{ template "cassandraGalleyReplicationType" . }} - "{{ template "cassandraGalleyReplicationArg" . }}" - {{- if (include "useTlsCaGalley" .) }} + {{- if (include "useTlsGalley" .) }} - --tls-certificate-file - - /certs/galley/ca.pem + - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsCaGalley" .) }} + {{- if (include "useTlsGalley" .) }} volumeMounts: - name: galley-cassandra-cert mountPath: "/certs/galley" @@ -152,12 +152,12 @@ spec: - spar - {{ template "cassandraSparReplicationType" . }} - "{{ template "cassandraSparReplicationArg" . }}" - {{- if (include "useTlsCaSpar" .) }} + {{- if (include "useTlsSpar" .) }} - --tls-certificate-file - - /certs/spar/ca.pem + - /certs/spar/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsCaSpar" .) }} + {{- if (include "useTlsSpar" .) }} volumeMounts: - name: spar-cassandra-cert mountPath: "/certs/spar" diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index 19e8f1457f..239d2b94cb 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -43,32 +43,32 @@ spec: - "9042" - --cassandra-keyspace-brig - brig - {{- if (include "useTlsCaBrig" .) }} + {{- if (include "useTlsBrig" .) }} - --tls-certificate-file-brig - - /certs/brig/ca.pem + - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsCaSpar" .) }} + {{- if (include "useTlsSpar" .) }} - --tls-certificate-file-spar - - /certs/spar/ca.pem + - /certs/spar/{{- (include "tlsSecretRefSpar" . | fromYaml).key }} {{- end }} volumeMounts: - {{- if (include "useTlsCaBrig" .) }} + {{- if (include "useTlsBrig" .) }} - name: brig-cassandra-cert mountPath: "/certs/brig" {{- end }} - {{- if (include "useTlsCaSpar" .) }} + {{- if (include "useTlsSpar" .) }} - name: spar-cassandra-cert mountPath: "/certs/spar" {{- end }} volumes: - {{- if (include "useTlsCaBrig" .) }} + {{- if (include "useTlsBrig" .) }} - name: brig-cassandra-cert secret: - secretName: brig-cassandra-cert + secretName: {{ (include "tlsSecretRefBrig" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsCaSpar" .) }} + {{- if (include "useTlsSpar" .) }} - name: spar-cassandra-cert secret: - secretName: spar-cassandra-cert + secretName: {{ (include "tlsSecretRefSpar" . | fromYaml).name }} {{- end }} {{- end }} diff --git a/charts/cassandra-migrations/values.yaml b/charts/cassandra-migrations/values.yaml index f726aa269b..f80e3448fa 100644 --- a/charts/cassandra-migrations/values.yaml +++ b/charts/cassandra-migrations/values.yaml @@ -61,6 +61,15 @@ images: # host: cassandra-ephemeral-galley # replicationMap: eu-west-1:3 # tlsCa: +# +# You may also directly refer to a Secret resource: +# +# cassandra: +# host: cassandra-external +# replicationFactor: 3 +# tlsCaSecretRef: +# name: +# key: # Overriding the following is only useful during datacenter migration time periods, # where some other job already migrates schemas. @@ -77,3 +86,11 @@ podSecurityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault + +#cassandra: +# host: cassandra-external +# replicationFactor: 3 +## tlsCa: "foo" +# tlsCaSecretRef: +# name: refN +# key: refK diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl index d535de0756..9638afd547 100644 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ b/charts/elasticsearch-index/templates/_helpers.tpl @@ -9,5 +9,13 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tlsCa") }} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "elasticsearch-index-migrate-cassandra-client-ca" "key" "ca.pem" | toYaml -}} +{{- end -}} {{- end -}} diff --git a/charts/elasticsearch-index/templates/cassandra-secret.yaml b/charts/elasticsearch-index/templates/cassandra-secret.yaml index 564a94804a..b332fb0d35 100644 --- a/charts/elasticsearch-index/templates/cassandra-secret.yaml +++ b/charts/elasticsearch-index/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (include "useCassandraCA" .Values) }} +{{- if (hasKey .Values.cassandra "tlsCa") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index c7282dffe7..e119401061 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -43,9 +43,9 @@ spec: - "{{ required "missing elasticsearch-index.galley.host!" .Values.galley.host }}" - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" - {{- if (include "useTlsCaGundeck" .) }} + {{- if (include "useCassandraCA" .Values) }} - --tls-certificate-file - - /certs/ca.pem + - /certs/{{- (include "tlsSecretRef" .Values | fromYaml).key }} {{- end }} {{- if (include "useCassandraCA" .Values) }} volumeMounts: @@ -56,5 +56,5 @@ spec: volumes: - name: elasticsearch-index-migrate-cassandra-client-ca secret: - secretName: elasticsearch-index-migrate-cassandra-client-ca + secretName: {{ (include "tlsSecretRef" .Values | fromYaml).name }} {{- end}} diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index d535de0756..1db8fc993d 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -9,5 +9,13 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tlsCa") }} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "galley-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} {{- end -}} diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml index 3e72145c6b..335144e088 100644 --- a/charts/galley/templates/cassandra-secret.yaml +++ b/charts/galley/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (include "useCassandraCA" .Values.config)}} +{{- if (hasKey .Values.config.cassandra "tlsCa") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 39c73d1b8a..461c8f4324 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -22,7 +22,7 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if (include "useCassandraCA" .) }} - tlsCa: /etc/wire/galley/cassandra/ca.pem + tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} brig: diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml index 7829dc45a8..084d49c9e3 100644 --- a/charts/galley/templates/deployment.yaml +++ b/charts/galley/templates/deployment.yaml @@ -39,7 +39,7 @@ spec: {{- if (include "useCassandraCA" .Values.config) }} - name: "galley-cassandra" secret: - secretName: galley-cassandra + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} {{- end }} containers: - name: galley diff --git a/charts/gundeck/templates/_helpers.tpl b/charts/gundeck/templates/_helpers.tpl index d535de0756..87f178365c 100644 --- a/charts/gundeck/templates/_helpers.tpl +++ b/charts/gundeck/templates/_helpers.tpl @@ -9,5 +9,13 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tlsCa") }} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "gundeck-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} {{- end -}} diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml index e9a36e08ce..01b1ed0dc7 100644 --- a/charts/gundeck/templates/cassandra-secret.yaml +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (include "useCassandraCA" .Values.config)}} +{{- if (hasKey .Values.config.cassandra "tlsCa") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index 48dc6e869b..7962df079c 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -26,7 +26,7 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if (include "useCassandraCA" .) }} - tlsCa: /etc/wire/gundeck/cassandra/ca.pem + tlsCa: /etc/wire/gundeck/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} redis: diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml index bedd0e5962..b79a9e9725 100644 --- a/charts/gundeck/templates/deployment.yaml +++ b/charts/gundeck/templates/deployment.yaml @@ -35,7 +35,7 @@ spec: {{- if (include "useCassandraCA" .Values.config) }} - name: "gundeck-cassandra" secret: - secretName: "gundeck-cassandra" + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} {{- end}} containers: - name: gundeck diff --git a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml index f7ccdf264d..6fd8de25b9 100644 --- a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml +++ b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml @@ -18,14 +18,14 @@ spec: command: ["cqlsh", "--ssl", "k8ssandra-cluster-datacenter-1-service"] env: - name: SSL_CERTFILE - value: "/certs/ca.pem" + value: "/certs/ca.crt" volumeMounts: - - name: cassandra-cert + - name: cassandra-jks-keystore mountPath: "/certs" volumes: - - name: cassandra-cert + - name: cassandra-jks-keystore secret: - secretName: cassandra-client-ca + secretName: cassandra-jks-keystore {{- end }} restartPolicy: OnFailure # Default is 6 retries. 8 is a bit arbitrary, but should be sufficient for diff --git a/charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml b/charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml deleted file mode 100644 index bfa4a24f92..0000000000 --- a/charts/k8ssandra-test-cluster/templates/client-encryption-stores.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.client_encryption_options.enabled }} -apiVersion: v1 -kind: Secret -metadata: - name: client-encryption-stores - namespace: {{ .Release.Namespace }} -type: Opaque -data: - keystore: {{ .Values.client_encryption_options.keystore | quote }} - "keystore-password": {{ .Values.client_encryption_options.keystorePassword | b64enc | quote }} - truststore: {{ .Values.client_encryption_options.truststore | quote }} - "truststore-password": {{ .Values.client_encryption_options.truststorePassword | b64enc | quote }} -{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml b/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml new file mode 100644 index 0000000000..87aff45d13 --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml @@ -0,0 +1,8 @@ +# TODO: Add conditional rendering +apiVersion: v1 +kind: Secret +metadata: + name: jks-password + namespace: {{ .Release.Namespace }} +data: + keystore-pass: {{ .Values.client_encryption_options.keystorePassword | b64enc }} diff --git a/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml b/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml index 506a363adb..35197d8b8f 100644 --- a/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml +++ b/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml @@ -45,9 +45,17 @@ spec: {{- if .Values.client_encryption_options.enabled }} clientEncryptionStores: keystoreSecretRef: - name: client-encryption-stores + name: cassandra-jks-keystore + key: keystore.jks + keystorePasswordSecretRef: + key: keystore-pass + name: jks-password truststoreSecretRef: - name: client-encryption-stores + name: cassandra-jks-keystore + key: truststore.jks + truststorePasswordSecretRef: + key: keystore-pass + name: jks-password {{- end }} reaper: autoScheduling: diff --git a/charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml b/charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml new file mode 100644 index 0000000000..1a4ed014ae --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml @@ -0,0 +1,10 @@ +# TODO: Add conditional rendering +apiVersion: v1 +kind: Secret +metadata: + name: tls-ca-secret + namespace: {{ .Release.Namespace }} +type: tls +data: + tls.crt: {{ .Values.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.tlsKey | b64enc | quote }} diff --git a/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml b/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml new file mode 100644 index 0000000000..4cac3683a0 --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml @@ -0,0 +1,44 @@ +# TODO: Add conditional rendering +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cassandra-certificate + namespace: {{ .Release.Namespace }} +spec: + # Secret names are always required. + secretName: cassandra-jks-keystore + duration: 2160h # 90d + renewBefore: 360h # 15d + subject: + # TODO: Do we need something better here? + organizations: + - PIT squad + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + # commonName: example.com + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + usages: + - server auth + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - k8ssandra-cluster-datacenter-1-service.{{ .Release.Namespace }}.svc.cluster.local + - k8ssandra-cluster-datacenter-1-service + issuerRef: + name: ca-issuer + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: Issuer + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io + keystores: + jks: + create: true + passwordSecretRef: # Password used to encrypt the keystore + key: keystore-pass + name: jks-password diff --git a/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml b/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml new file mode 100644 index 0000000000..09afc7b70a --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml @@ -0,0 +1,9 @@ +# TODO: Add conditional rendering +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer + namespace: {{ .Release.Namespace }} +spec: + ca: + secretName: tls-ca-secret diff --git a/charts/k8ssandra-test-cluster/values.yaml b/charts/k8ssandra-test-cluster/values.yaml index b9d9e9ca49..354bed8b24 100644 --- a/charts/k8ssandra-test-cluster/values.yaml +++ b/charts/k8ssandra-test-cluster/values.yaml @@ -17,10 +17,12 @@ storageSize: 10G client_encryption_options: enabled: false optional: true - keystore: "" keystorePassword: password - truststore: "" truststorePassword: password # TODO: This could be deduced from the keystore. ca: "" + +# TODO: Give structure. Can't these be generated, too? +tlsCrt: "" +tlsKey: "" diff --git a/charts/spar/templates/_helpers.tpl b/charts/spar/templates/_helpers.tpl index 0da2812012..8b991a6d22 100644 --- a/charts/spar/templates/_helpers.tpl +++ b/charts/spar/templates/_helpers.tpl @@ -8,5 +8,13 @@ {{- end -}} {{- define "useCassandraCA" -}} -{{ and (hasKey .cassandra "tlsCa") }} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "brig-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} {{- end -}} diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml index b9b2a986ec..a7844b7c27 100644 --- a/charts/spar/templates/cassandra-secret.yaml +++ b/charts/spar/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (include "useCassandraCA" .Values.config) }} +{{- if (hasKey .Values.config.cassandra "tlsCa") }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index a561871001..675c593381 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -26,7 +26,7 @@ data: filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} {{- if (include "useCassandraCA" .) }} - tlsCa: /etc/wire/spar/cassandra/ca.pem + tlsCa: /etc/wire/spar/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} maxttlAuthreq: {{ .maxttlAuthreq }} diff --git a/charts/spar/templates/deployment.yaml b/charts/spar/templates/deployment.yaml index 26a1145af5..000a34961f 100644 --- a/charts/spar/templates/deployment.yaml +++ b/charts/spar/templates/deployment.yaml @@ -33,7 +33,7 @@ spec: {{- if (include "useCassandraCA" .Values.config) }} - name: "spar-cassandra" secret: - secretName: "spar-cassandra" + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} {{- end}} containers: - name: spar From c86ffe7eef2ca834a57f6e4316708e1e5bcbba58 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 23 Nov 2023 10:01:37 +0100 Subject: [PATCH 50/98] Accept empty tlsCa This makes config easier to write. --- charts/brig/templates/cassandra-secret.yaml | 2 +- charts/elasticsearch-index/templates/cassandra-secret.yaml | 2 +- charts/galley/templates/cassandra-secret.yaml | 2 +- charts/gundeck/templates/cassandra-secret.yaml | 2 +- charts/spar/templates/cassandra-secret.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index fcf1d704ae..8130e4324d 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (hasKey .Values.config.cassandra "tlsCa") }} +{{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/elasticsearch-index/templates/cassandra-secret.yaml b/charts/elasticsearch-index/templates/cassandra-secret.yaml index b332fb0d35..93486dd962 100644 --- a/charts/elasticsearch-index/templates/cassandra-secret.yaml +++ b/charts/elasticsearch-index/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (hasKey .Values.cassandra "tlsCa") }} +{{- if not (empty .Values.cassandra.tlsCa) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml index 335144e088..032a6c361d 100644 --- a/charts/galley/templates/cassandra-secret.yaml +++ b/charts/galley/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (hasKey .Values.config.cassandra "tlsCa") }} +{{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml index 01b1ed0dc7..14c531896a 100644 --- a/charts/gundeck/templates/cassandra-secret.yaml +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (hasKey .Values.config.cassandra "tlsCa") }} +{{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml index a7844b7c27..6912dd988f 100644 --- a/charts/spar/templates/cassandra-secret.yaml +++ b/charts/spar/templates/cassandra-secret.yaml @@ -1,4 +1,4 @@ -{{- if (hasKey .Values.config.cassandra "tlsCa") }} +{{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret metadata: From c9305c38223992e13557f276d71be9360a9a960f Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 23 Nov 2023 10:07:20 +0100 Subject: [PATCH 51/98] Generate key pair in K8s as Secret --- .../templates/cassandra-client-ca.yaml | 10 ---------- .../templates/jks-store-pass.yaml | 3 ++- .../templates/tls-ca-secret.yaml | 10 ---------- .../templates/tls-certificate.yaml | 4 ++-- .../k8ssandra-test-cluster/templates/tls-issuer.yaml | 6 +++--- charts/k8ssandra-test-cluster/values.yaml | 11 +++-------- 6 files changed, 10 insertions(+), 34 deletions(-) delete mode 100644 charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml delete mode 100644 charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml diff --git a/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml b/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml deleted file mode 100644 index 61c60561f2..0000000000 --- a/charts/k8ssandra-test-cluster/templates/cassandra-client-ca.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if .Values.client_encryption_options.enabled }} -apiVersion: v1 -kind: Secret -metadata: - name: cassandra-client-ca - namespace: {{ .Release.Namespace }} -type: Opaque -data: - ca.pem: {{ .Values.client_encryption_options.ca | b64enc | quote }} -{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml b/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml index 87aff45d13..52e6f2d0eb 100644 --- a/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml +++ b/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml @@ -1,4 +1,4 @@ -# TODO: Add conditional rendering +{{- if .Values.client_encryption_options.enabled }} apiVersion: v1 kind: Secret metadata: @@ -6,3 +6,4 @@ metadata: namespace: {{ .Release.Namespace }} data: keystore-pass: {{ .Values.client_encryption_options.keystorePassword | b64enc }} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml b/charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml deleted file mode 100644 index 1a4ed014ae..0000000000 --- a/charts/k8ssandra-test-cluster/templates/tls-ca-secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# TODO: Add conditional rendering -apiVersion: v1 -kind: Secret -metadata: - name: tls-ca-secret - namespace: {{ .Release.Namespace }} -type: tls -data: - tls.crt: {{ .Values.tlsCrt | b64enc | quote }} - tls.key: {{ .Values.tlsKey | b64enc | quote }} diff --git a/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml b/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml index 4cac3683a0..c7efd99c8a 100644 --- a/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml +++ b/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml @@ -1,4 +1,4 @@ -# TODO: Add conditional rendering +{{- if .Values.client_encryption_options.enabled }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -10,7 +10,6 @@ spec: duration: 2160h # 90d renewBefore: 360h # 15d subject: - # TODO: Do we need something better here? organizations: - PIT squad # The use of the common name field has been deprecated since 2000 and is @@ -42,3 +41,4 @@ spec: passwordSecretRef: # Password used to encrypt the keystore key: keystore-pass name: jks-password +{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml b/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml index 09afc7b70a..65bc3dbad3 100644 --- a/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml +++ b/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml @@ -1,9 +1,9 @@ -# TODO: Add conditional rendering +{{- if .Values.client_encryption_options.enabled }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: ca-issuer namespace: {{ .Release.Namespace }} spec: - ca: - secretName: tls-ca-secret + selfSigned: {} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/values.yaml b/charts/k8ssandra-test-cluster/values.yaml index 354bed8b24..f936c2c572 100644 --- a/charts/k8ssandra-test-cluster/values.yaml +++ b/charts/k8ssandra-test-cluster/values.yaml @@ -17,12 +17,7 @@ storageSize: 10G client_encryption_options: enabled: false optional: true + # The password could be secured better. However, this chart is meant to be + # used as test setup. And, protecting a self-signed certificate isn't very + # useful. keystorePassword: password - truststorePassword: password - - # TODO: This could be deduced from the keystore. - ca: "" - -# TODO: Give structure. Can't these be generated, too? -tlsCrt: "" -tlsKey: "" From 68b90b65f93fdd811972dd4ebbaddb95337c7f33 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 23 Nov 2023 17:22:24 +0100 Subject: [PATCH 52/98] Fix secret handling in case a CA PEM string is provided --- charts/brig/templates/_helpers.tpl | 2 +- .../cassandra-migrations/templates/_helpers.tpl | 16 ++++++++-------- .../templates/cassandra-certs.yaml | 8 ++++---- charts/galley/templates/_helpers.tpl | 2 +- charts/gundeck/templates/_helpers.tpl | 2 +- charts/spar/templates/_helpers.tpl | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index 1bb20ba67c..d7546dced8 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -16,6 +16,6 @@ {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} {{- else }} -{{- dict "name" "brig-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- dict "name" "brig-cassandra" "key" "ca.pem" | toYaml -}} {{- end -}} {{- end -}} diff --git a/charts/cassandra-migrations/templates/_helpers.tpl b/charts/cassandra-migrations/templates/_helpers.tpl index e0025dda4a..e3deb51632 100644 --- a/charts/cassandra-migrations/templates/_helpers.tpl +++ b/charts/cassandra-migrations/templates/_helpers.tpl @@ -115,9 +115,9 @@ Thus the order of priority is: {{- define "tlsCaGalley" -}} {{ $cassandraGalley := default dict .Values.cassandraGalley }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa | toYaml }} +{{ .Values.cassandra.tlsCa }} {{- else -}} -{{ $cassandraGalley.tlsCa | toYaml }} +{{ $cassandraGalley.tlsCa }} {{- end -}} {{- end -}} @@ -140,9 +140,9 @@ Thus the order of priority is: {{- define "tlsCaBrig" -}} {{ $cassandraBrig := default dict .Values.cassandraBrig }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa | toYaml }} +{{ .Values.cassandra.tlsCa }} {{- else -}} -{{ $cassandraBrig.tlsCa | toYaml }} +{{ $cassandraBrig.tlsCa }} {{- end -}} {{- end -}} @@ -165,9 +165,9 @@ Thus the order of priority is: {{- define "tlsCaSpar" -}} {{ $cassandraSpar := default dict .Values.cassandraSpar }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa | toYaml }} +{{ .Values.cassandra.tlsCa }} {{- else -}} -{{ $cassandraSpar.tlsCa | toYaml }} +{{ $cassandraSpar.tlsCa }} {{- end -}} {{- end -}} @@ -190,9 +190,9 @@ Thus the order of priority is: {{- define "tlsCaGundeck" -}} {{ $cassandraGundeck := default dict .Values.cassandraGundeck }} {{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa | toYaml }} +{{ .Values.cassandra.tlsCa }} {{- else -}} -{{ $cassandraGundeck.tlsCa | toYaml }} +{{ $cassandraGundeck.tlsCa }} {{- end -}} {{- end -}} diff --git a/charts/cassandra-migrations/templates/cassandra-certs.yaml b/charts/cassandra-migrations/templates/cassandra-certs.yaml index b4d5c54657..988e573b4e 100644 --- a/charts/cassandra-migrations/templates/cassandra-certs.yaml +++ b/charts/cassandra-migrations/templates/cassandra-certs.yaml @@ -1,4 +1,4 @@ -{{- if (include "tlsCaBrig" . | fromYaml) }} +{{- if not (empty (include "tlsCaBrig" .)) }} apiVersion: v1 kind: Secret metadata: @@ -16,7 +16,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaBrig" . | b64enc | quote }} {{- end}} -{{- if (include "tlsCaGalley" . | fromYaml) }} +{{- if not (empty (include "tlsCaGalley" .)) }} --- apiVersion: v1 kind: Secret @@ -35,7 +35,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaGalley" . | b64enc | quote }} {{- end}} -{{- if (include "tlsCaGundeck" . | fromYaml) }} +{{- if not (empty (include "tlsCaGundeck" .)) }} --- apiVersion: v1 kind: Secret @@ -54,7 +54,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaGundeck" . | b64enc | quote }} {{- end}} -{{- if (include "tlsCaSpar" . | fromYaml) }} +{{- if not (empty (include "tlsCaSpar" .)) }} --- apiVersion: v1 kind: Secret diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index 1db8fc993d..2e73ecfe6a 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -16,6 +16,6 @@ {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} {{- else }} -{{- dict "name" "galley-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- dict "name" "galley-cassandra" "key" "ca.pem" | toYaml -}} {{- end -}} {{- end -}} diff --git a/charts/gundeck/templates/_helpers.tpl b/charts/gundeck/templates/_helpers.tpl index 87f178365c..8876add186 100644 --- a/charts/gundeck/templates/_helpers.tpl +++ b/charts/gundeck/templates/_helpers.tpl @@ -16,6 +16,6 @@ {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} {{- else }} -{{- dict "name" "gundeck-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- dict "name" "gundeck-cassandra" "key" "ca.pem" | toYaml -}} {{- end -}} {{- end -}} diff --git a/charts/spar/templates/_helpers.tpl b/charts/spar/templates/_helpers.tpl index 8b991a6d22..5f9280a808 100644 --- a/charts/spar/templates/_helpers.tpl +++ b/charts/spar/templates/_helpers.tpl @@ -15,6 +15,6 @@ {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} {{- else }} -{{- dict "name" "brig-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- dict "name" "spar-cassandra" "key" "ca.pem" | toYaml -}} {{- end -}} {{- end -}} From 70950ad2d3be32a85e22df718252656deb7d816e Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 08:08:05 +0100 Subject: [PATCH 53/98] Add debug trace logs for SSL cert / integration --- integration/test/Testlib/Env.hs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index 93c992b12f..3ac3503ea6 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -14,6 +14,7 @@ import Data.Set (Set) import Data.Set qualified as Set import Data.Yaml qualified as Yaml import Database.CQL.IO qualified as Cassandra +import Debug.Trace (traceM) import Network.HTTP.Client qualified as HTTP import OpenSSL.Session qualified as OpenSSL import System.Directory @@ -67,7 +68,10 @@ mkGlobalEnv cfgFile = do manager <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings + traceM $ "SSL: intConfig.cassandra.cassTlsCa " ++ show intConfig.cassandra.cassTlsCa + mbCassCertFilePath <- liftIO $ getCassCertFilePath + traceM $ "SSL: mbCassCertFilePath " ++ show mbCassCertFilePath mbSSLContext <- liftIO $ createSSLContext mbCassCertFilePath let basicCassSettings = Cassandra.defSettings From e13319eb1cb69dcd9519b4b884a53064692d82ea Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 08:41:58 +0100 Subject: [PATCH 54/98] Helm: Cassandra SSL for integration tests --- .../templates/tests/brig-integration.yaml | 9 +++++++++ .../templates/tests/galley-integration.yaml | 9 +++++++++ .../templates/tests/gundeck-integration.yaml | 9 +++++++++ charts/integration/templates/_helpers.tpl | 14 ++++++++++++- .../templates/cassandra-secret.yaml | 14 +++++++++++++ charts/integration/templates/configmap.yaml | 6 +++++- .../templates/integration-integration.yaml | 20 ++++++++++++++++++- .../templates/tests/spar-integration.yaml | 9 +++++++++ 8 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 charts/integration/templates/cassandra-secret.yaml diff --git a/charts/brig/templates/tests/brig-integration.yaml b/charts/brig/templates/tests/brig-integration.yaml index 1599c3860b..5dcc5cda12 100644 --- a/charts/brig/templates/tests/brig-integration.yaml +++ b/charts/brig/templates/tests/brig-integration.yaml @@ -44,6 +44,11 @@ spec: - name: "brig-integration-secrets" secret: secretName: "brig-integration" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "brig-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: integration image: "{{ .Values.image.repository }}-integration:{{ .Values.image.tag }}" @@ -101,6 +106,10 @@ spec: # non-default locations # (see corresp. TODO in galley.) mountPath: "/etc/wire/integration-secrets" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "brig-cassandra" + mountPath: "/etc/wire/brig/cassandra" + {{- end }} env: # these dummy values are necessary for Amazonka's "Discover" diff --git a/charts/galley/templates/tests/galley-integration.yaml b/charts/galley/templates/tests/galley-integration.yaml index e187022837..a87471f9a7 100644 --- a/charts/galley/templates/tests/galley-integration.yaml +++ b/charts/galley/templates/tests/galley-integration.yaml @@ -40,6 +40,11 @@ spec: - name: "galley-secrets" secret: secretName: "galley" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "galley-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end }} containers: - name: integration image: "{{ .Values.image.repository }}-integration:{{ .Values.image.tag }}" @@ -84,6 +89,10 @@ spec: mountPath: "/etc/wire/integration-secrets" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" + {{- if (include "useCassandraCA" .Values.config)}} + - name: "galley-cassandra" + mountPath: "/etc/wire/galley/cassandra" + {{- end }} env: # these dummy values are necessary for Amazonka's "Discover" - name: AWS_ACCESS_KEY_ID diff --git a/charts/gundeck/templates/tests/gundeck-integration.yaml b/charts/gundeck/templates/tests/gundeck-integration.yaml index 7f92351be5..2413f61588 100644 --- a/charts/gundeck/templates/tests/gundeck-integration.yaml +++ b/charts/gundeck/templates/tests/gundeck-integration.yaml @@ -13,6 +13,11 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "gundeck-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: integration # TODO: When deployed to staging (or real AWS env), _all_ tests should be run @@ -54,6 +59,10 @@ spec: mountPath: "/etc/wire/integration" - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "gundeck-cassandra" + mountPath: "/etc/wire/gundeck/cassandra" + {{- end }} env: # these dummy values are necessary for Amazonka's "Discover" - name: AWS_ACCESS_KEY_ID diff --git a/charts/integration/templates/_helpers.tpl b/charts/integration/templates/_helpers.tpl index e138d2f1bb..97f485a450 100644 --- a/charts/integration/templates/_helpers.tpl +++ b/charts/integration/templates/_helpers.tpl @@ -36,4 +36,16 @@ {{- define "integrationTestHelperNewLabels" -}} {{- (semverCompare ">= 1.23-0" (include "kubeVersion" .)) -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{- define "useCassandraCA" -}} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "integration-cassandra" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/integration/templates/cassandra-secret.yaml b/charts/integration/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..ab49db0039 --- /dev/null +++ b/charts/integration/templates/cassandra-secret.yaml @@ -0,0 +1,14 @@ +{{- if not (empty .Values.config.cassandra.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: integration-cassandra + labels: + app: integration + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/integration/templates/configmap.yaml b/charts/integration/templates/configmap.yaml index 99a247203a..c3b829fc70 100644 --- a/charts/integration/templates/configmap.yaml +++ b/charts/integration/templates/configmap.yaml @@ -120,4 +120,8 @@ data: federatorExternalPort: {{ $dynamicBackend.federatorExternalPort }} {{- end }} cassandra: -{{ toYaml .Values.config.cassandra | indent 6}} + host: {{ .Values.config.cassandra.host }} + port: {{ .Values.config.cassandra.port }} + {{- if (include "useCassandraCA" .Values.config) }} + tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} + {{- end }} diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 2fe7718fa5..dd1a3070af 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -75,7 +75,11 @@ spec: - name: "nginz-secrets" secret: secretName: "nginz" - + {{- if (include "useCassandraCA" .Values.config) }} + - name: integration-cassandra + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end }} restartPolicy: Never initContainers: @@ -86,6 +90,11 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 6 }} {{- end }} + volumeMounts: + {{- if (include "useCassandraCA" .Values.config)}} + - name: "integration-cassandra" + mountPath: "/certs" + {{- end }} env: - name: INTEGRATION_DYNAMIC_BACKENDS_POOLSIZE value: "{{ .Values.config.dynamicBackendsPoolsize }}" @@ -111,7 +120,11 @@ spec: - | set -euo pipefail # FUTUREWORK: Do all of this in the integration test binary + {{- if (include "useCassandraCA" .Values.config) }} + integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} + {{- else }} integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} + {{- end }} integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200 integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} @@ -212,6 +225,11 @@ spec: - name: nginz-secrets mountPath: /etc/wire/nginz/secrets + {{- if (include "useCassandraCA" .Values.config)}} + - name: "integration-cassandra" + mountPath: "/certs" + {{- end }} + env: # these dummy values are necessary for Amazonka's "Discover" - name: AWS_ACCESS_KEY_ID diff --git a/charts/spar/templates/tests/spar-integration.yaml b/charts/spar/templates/tests/spar-integration.yaml index ff937f3d18..bcec26a64b 100644 --- a/charts/spar/templates/tests/spar-integration.yaml +++ b/charts/spar/templates/tests/spar-integration.yaml @@ -16,6 +16,11 @@ spec: - name: "spar-config" configMap: name: "spar" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "spar-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: integration image: "{{ .Values.image.repository }}-integration:{{ .Values.image.tag }}" @@ -56,6 +61,10 @@ spec: mountPath: "/etc/wire/integration" - name: "spar-config" mountPath: "/etc/wire/spar/conf" + {{- if (include "useCassandraCA" .Values.config) }} + - name: "spar-cassandra" + mountPath: "/etc/wire/spar/cassandra" + {{- end }} resources: requests: memory: "512Mi" From 2283d8729265cf369ceb63a6af58aba3a455bd6d Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 08:44:59 +0100 Subject: [PATCH 55/98] Debug log in integration-integration.yaml --- charts/integration/templates/integration-integration.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index dd1a3070af..3237cd369a 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -119,6 +119,8 @@ spec: - -c - | set -euo pipefail + # TODO: Remove after debugging + set -x # FUTUREWORK: Do all of this in the integration test binary {{- if (include "useCassandraCA" .Values.config) }} integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} From bf8a5cdda46fe6cfcfaf893dad505fe0b6009c31 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 09:16:53 +0100 Subject: [PATCH 56/98] Helm: Ensure integration has cassandra certs --- .../templates/integration-integration.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 3237cd369a..18f2877c55 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -150,6 +150,8 @@ spec: - -c - | set -euo pipefail + # TODO: Remove after debugging + set -x if integration --config /etc/wire/integration/integration.yaml; then exit_code=$? @@ -230,6 +232,18 @@ spec: {{- if (include "useCassandraCA" .Values.config)}} - name: "integration-cassandra" mountPath: "/certs" + + - name: "integration-cassandra" + mountPath: "/etc/wire/brig/cassandra" + + - name: "integration-cassandra" + mountPath: "/etc/wire/galley/cassandra" + + - name: "integration-cassandra" + mountPath: "/etc/wire/gundeck/cassandra" + + - name: "integration-cassandra" + mountPath: "/etc/wire/spar/cassandra" {{- end }} env: From dd94bc360a113518a2b2dc8b498ee6d29ff5caf6 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 09:17:37 +0100 Subject: [PATCH 57/98] integration-tests: Deploy TLS secured cassandra --- Makefile | 2 +- .../k8ssandra-test-cluster/values.yaml | 9 +++++ hack/helm_vars/wire-server/values.yaml.gotmpl | 40 +++++++++++++++---- hack/helmfile.yaml | 33 +++++++++++++-- 4 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 hack/helm_vars/k8ssandra-test-cluster/values.yaml diff --git a/Makefile b/Makefile index a445c276e2..bff5a7edad 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ DOCKER_TAG ?= $(USER) # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything) HELM_SEMVER ?= 0.0.42 # The list of helm charts needed on internal kubernetes testing environments -CHARTS_INTEGRATION := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn +CHARTS_INTEGRATION := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn k8ssandra-test-cluster # The list of helm charts to publish on S3 # FUTUREWORK: after we "inline local subcharts", # (e.g. move charts/brig to charts/wire-server/brig) diff --git a/hack/helm_vars/k8ssandra-test-cluster/values.yaml b/hack/helm_vars/k8ssandra-test-cluster/values.yaml new file mode 100644 index 0000000000..8a072a456e --- /dev/null +++ b/hack/helm_vars/k8ssandra-test-cluster/values.yaml @@ -0,0 +1,9 @@ +storageClassName: hcloud-volumes + +client_encryption_options: + enabled: true + optional: false + # The password could be secured better. However, this chart is meant to be + # used as test setup. And, protecting a self-signed certificate isn't very + # useful. + keystorePassword: p4ssw0rd diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 49d6812bb4..4d63c0f315 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -18,15 +18,22 @@ tags: cassandra-migrations: imagePullPolicy: {{ .Values.imagePullPolicy }} cassandra: - host: cassandra-ephemeral + host: k8ssandra-cluster-datacenter-1-service replicationFactor: 1 + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + elasticsearch-index: imagePullPolicy: {{ .Values.imagePullPolicy }} elasticsearch: host: elasticsearch-ephemeral index: directory_test cassandra: - host: cassandra-ephemeral + host: k8ssandra-cluster-datacenter-1-service + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" brig: replicaCount: 1 @@ -41,8 +48,11 @@ brig: teamCreatorWelcome: https://teams.wire.com/login teamMemberWelcome: https://wire.com/download cassandra: - host: cassandra-ephemeral + host: k8ssandra-cluster-datacenter-1-service replicaCount: 1 + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" elasticsearch: host: elasticsearch-ephemeral index: directory_test @@ -186,8 +196,11 @@ galley: imagePullPolicy: {{ .Values.imagePullPolicy }} config: cassandra: - host: cassandra-ephemeral + host: k8ssandra-cluster-datacenter-1-service replicaCount: 1 + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" enableFederation: true # keep in sync with brig.config.enableFederation, cargohold.config.enableFederation and tags.federator! settings: maxConvAndTeamSize: 16 @@ -248,8 +261,11 @@ gundeck: memory: 1024Mi config: cassandra: - host: cassandra-ephemeral + host: k8ssandra-cluster-datacenter-1-service replicaCount: 1 + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" redis: host: redis-ephemeral-master connectionMode: master @@ -322,7 +338,10 @@ spar: config: tlsDisableCertValidation: true cassandra: - host: cassandra-ephemeral + host: k8ssandra-cluster-datacenter-1-service + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" logLevel: Debug domain: zinfra.io appUri: http://spar:8080/ @@ -380,8 +399,15 @@ background-worker: integration: ingress: class: "nginx-{{ .Release.Namespace }}" - {{- if .Values.uploadXml }} config: + cassandra: + host: k8ssandra-cluster-datacenter-1-service + port: 9042 + replicationFactor: 1 + tlsCaSecretRef: + name: cassandra-jks-keystore + key: ca.crt + {{- if .Values.uploadXml }} uploadXml: baseUrl: {{ .Values.uploadXml.baseUrl }} secrets: diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 878cb016f5..5e30d62a66 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -46,7 +46,6 @@ releases: chart: '../.local/charts/fake-aws' values: - './helm_vars/fake-aws/values.yaml' - - name: 'databases-ephemeral' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/databases-ephemeral' @@ -55,6 +54,34 @@ releases: namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/databases-ephemeral' +# - name: 'redis-ephemeral' +# namespace: '{{ .Values.namespace1 }}' +# chart: '../.local/charts/redis-ephemeral' +# +# - name: 'redis-ephemeral' +# namespace: '{{ .Values.namespace2 }}' +# chart: '../.local/charts/redis-ephemeral' +# +# - name: 'elasticsearch-ephemeral' +# namespace: '{{ .Values.namespace1 }}' +# chart: '../.local/charts/redis-ephemeral' +# +# - name: 'elasticsearch-ephemeral' +# namespace: '{{ .Values.namespace2 }}' +# chart: '../.local/charts/redis-ephemeral' + + - name: k8ssandra-test-cluster + chart: '../.local/charts/k8ssandra-test-cluster' + namespace: '{{ .Values.namespace1 }}' + values: + - './helm_vars/k8ssandra-test-cluster/values.yaml' + + - name: k8ssandra-test-cluster + chart: '../.local/charts/k8ssandra-test-cluster' + namespace: '{{ .Values.namespace2 }}' + values: + - './helm_vars/k8ssandra-test-cluster/values.yaml' + - name: 'rabbitmq' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/rabbitmq' @@ -128,7 +155,7 @@ releases: - name: cargohold.config.settings.federationDomain value: {{ .Values.federationDomain1 }} needs: - - 'databases-ephemeral' + - 'k8ssandra-test-cluster' - name: 'wire-server' namespace: '{{ .Values.namespace2 }}' @@ -144,4 +171,4 @@ releases: - name: cargohold.config.settings.federationDomain value: {{ .Values.federationDomain2 }} needs: - - 'databases-ephemeral' + - 'k8ssandra-test-cluster' From bf64d06f36c91397b55c9fbb8e4f6ee191470c54 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 09:39:13 +0100 Subject: [PATCH 58/98] Clean up debug tracing: set -x --- charts/integration/templates/integration-integration.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 18f2877c55..a69ee0ea1e 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -119,8 +119,6 @@ spec: - -c - | set -euo pipefail - # TODO: Remove after debugging - set -x # FUTUREWORK: Do all of this in the integration test binary {{- if (include "useCassandraCA" .Values.config) }} integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} @@ -150,8 +148,6 @@ spec: - -c - | set -euo pipefail - # TODO: Remove after debugging - set -x if integration --config /etc/wire/integration/integration.yaml; then exit_code=$? From d007819a647760d3125ce19ff0bfa423a69bb3f9 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 15:48:29 +0100 Subject: [PATCH 59/98] Replace trace logs with print statements --- integration/test/Testlib/Env.hs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index 3ac3503ea6..f8bd436aa2 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -14,7 +14,6 @@ import Data.Set (Set) import Data.Set qualified as Set import Data.Yaml qualified as Yaml import Database.CQL.IO qualified as Cassandra -import Debug.Trace (traceM) import Network.HTTP.Client qualified as HTTP import OpenSSL.Session qualified as OpenSSL import System.Directory @@ -68,10 +67,7 @@ mkGlobalEnv cfgFile = do manager <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings - traceM $ "SSL: intConfig.cassandra.cassTlsCa " ++ show intConfig.cassandra.cassTlsCa - mbCassCertFilePath <- liftIO $ getCassCertFilePath - traceM $ "SSL: mbCassCertFilePath " ++ show mbCassCertFilePath mbSSLContext <- liftIO $ createSSLContext mbCassCertFilePath let basicCassSettings = Cassandra.defSettings @@ -114,6 +110,7 @@ mkGlobalEnv cfgFile = do where createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) createSSLContext (Just certFilePath) = do + print ("TLS: Connecting to Cassandra with TLS. Provided CA path:" ++ certFilePath) sslContext <- OpenSSL.context OpenSSL.contextSetCAFile sslContext certFilePath OpenSSL.contextSetVerificationMode @@ -124,7 +121,9 @@ mkGlobalEnv cfgFile = do vpCallback = Nothing } pure $ Just sslContext - createSSLContext Nothing = pure Nothing + createSSLContext Nothing = do + print ("TLS: No TLS CA path provided. Connecting to Cassandra without TLS." :: String) + pure Nothing mkEnv :: GlobalEnv -> Codensity IO Env mkEnv ge = do From c684cf03d71c1a89d4f4734ebb85e096f74ab1fb Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 16:49:42 +0100 Subject: [PATCH 60/98] Add documentation --- .../src/developer/reference/config-options.md | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index fd8e6034ad..da342321d0 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -807,3 +807,29 @@ CSP_EXTRA_SCRIPT_SRC: https://*.[[hostname]] CSP_EXTRA_STYLE_SRC: https://*.[[hostname]] CSP_EXTRA_WORKER_SRC: https://*.[[hostname]] ``` + +## TLS-encrypted Cassandra connections + +By default, all connections to Cassandra by the Wire backend are unencrypted. To +configure client-side TLS-encrypted connections (where the Wire backend is the +client), a **C**ertificate **A**uthority in PEM format needs to be configured. + +The ways differ regarding the kind of program: +- *Services* expect a `cassandra.tlsCa: ` attribute in their config file. +- *CLI commands* (e.g. migrations) accept a `--tls-certificate-file ` parameter. + +When a CA PEM file is configured, all Cassandra connections are opened with TLS +encryption. I.e. there is no fallback to unencrypted connections. This ensures +that connections that are expected to be secure, would not silently and +unnoticed be insecure. + +In Helm charts, the CA PEM is provided as multiline string in the `cassandra` block. + +The CA may be self-signed. It is used to validate the certificate of the +Cassandra server. + +How to configure Cassandra to accept TLS-encrypted connections in general is +beyond the scope of this document. The `k8ssandra-test-cluster` provides an +example how to do this for the Kubernetes solution *K8ssandra*. The +corresponding Cassandra options are described in Cassandra's documentation: +[client_encryption_options](https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options) From 12251e7cb93d9f08ec0fe9d2623eee663cf2f0d6 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 27 Nov 2023 16:56:37 +0100 Subject: [PATCH 61/98] More docs --- docs/src/developer/reference/config-options.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index da342321d0..93400114e3 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -823,13 +823,18 @@ encryption. I.e. there is no fallback to unencrypted connections. This ensures that connections that are expected to be secure, would not silently and unnoticed be insecure. -In Helm charts, the CA PEM is provided as multiline string in the `cassandra` block. +In Helm charts, the CA PEM is either provided as multiline string in the +`cassandra.tlsCa` attribute or as a reference to a `Secret` in +`cassandra.tlsCaSecretRef.name` and `cassandra.tlsCaSecretRef.key`. The `name` +is the name of the `Secret`, the `key` is the entry in it. Such a `Secret` can +e.g. be created by `cert-manager`. The CA may be self-signed. It is used to validate the certificate of the Cassandra server. How to configure Cassandra to accept TLS-encrypted connections in general is beyond the scope of this document. The `k8ssandra-test-cluster` provides an -example how to do this for the Kubernetes solution *K8ssandra*. The -corresponding Cassandra options are described in Cassandra's documentation: +example how to do this for the Kubernetes solution *K8ssandra* and a `Secret` +generated by `cert-manager`. The corresponding Cassandra options are described +in Cassandra's documentation: [client_encryption_options](https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options) From 0ad93e640d44c560465b2b8d8d8169dd1025a76f Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 28 Nov 2023 10:55:19 +0100 Subject: [PATCH 62/98] Remove self-signed cert from test setup It was only used locally during development. --- Makefile | 18 +++++++------- hack/cassandra.cert.pem | 30 ----------------------- services/brig/brig.integration.yaml | 1 - services/galley/galley.integration.yaml | 1 - services/gundeck/gundeck.integration.yaml | 1 - services/integration.yaml | 1 - services/spar/spar.integration.yaml | 1 - 7 files changed, 9 insertions(+), 44 deletions(-) delete mode 100644 hack/cassandra.cert.pem diff --git a/Makefile b/Makefile index bff5a7edad..dbe1f612e5 100644 --- a/Makefile +++ b/Makefile @@ -302,15 +302,15 @@ db-reset: c # Migrate all keyspaces and reset the ES index .PHONY: db-migrate db-migrate: c - ./dist/brig-schema --keyspace brig_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/galley-schema --keyspace galley_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/spar-schema --keyspace spar_test --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null - ./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 --tls-certificate-file "./hack/cassandra.cert.pem" > /dev/null + ./dist/brig-schema --keyspace brig_test --replication-factor 1 > /dev/null + ./dist/galley-schema --keyspace galley_test --replication-factor 1 > /dev/null + ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 > /dev/null + ./dist/spar-schema --keyspace spar_test --replication-factor 1 > /dev/null + ./dist/brig-schema --keyspace brig_test2 --replication-factor 1 > /dev/null + ./dist/galley-schema --keyspace galley_test2 --replication-factor 1 > /dev/null + ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 > /dev/null + ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 > /dev/null + ./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 > /dev/null ./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 > /dev/null ./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 > /dev/null ./integration/scripts/integration-dynamic-backends-brig-index.sh --elasticsearch-server http://localhost:9200 > /dev/null diff --git a/hack/cassandra.cert.pem b/hack/cassandra.cert.pem deleted file mode 100644 index b91d091423..0000000000 --- a/hack/cassandra.cert.pem +++ /dev/null @@ -1,30 +0,0 @@ -Bag Attributes - friendlyName: node0 - localKeyID: 54 69 6D 65 20 31 36 39 34 37 30 30 39 39 39 39 36 32 -subject=C = None, L = None, O = None, OU = None, CN = 127.0.0.1 -issuer=C = None, L = None, O = None, OU = None, CN = 127.0.0.1 ------BEGIN CERTIFICATE----- -MIIEQzCCAqugAwIBAgIIQCjt9rPKRJcwDQYJKoZIhvcNAQEMBQAwUDENMAsGA1UE -BhMETm9uZTENMAsGA1UEBxMETm9uZTENMAsGA1UEChMETm9uZTENMAsGA1UECxME -Tm9uZTESMBAGA1UEAxMJMTI3LjAuMC4xMB4XDTIzMDkxNDE0MTYzOVoXDTIzMTIx -MzE0MTYzOVowUDENMAsGA1UEBhMETm9uZTENMAsGA1UEBxMETm9uZTENMAsGA1UE -ChMETm9uZTENMAsGA1UECxMETm9uZTESMBAGA1UEAxMJMTI3LjAuMC4xMIIBojAN -BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA3+TVni16xmp7COX3hLnmhHUxXZxd -H6g8PZJ3dQOlhT/8Sw570ZmATkL+LGK90uAf8vW/RcBidHRpSMWsG57g/vo5fRi4 -zhMV1lFINxzmPJYvnb/CEwdoyHesWht/2SOCvOm02pDwBye8nlftdGp7fdYq/dhk -+dh8SGxB+dkqQG7+Jkv7i6xGqIj9j94UXl3ZbDyU7VJUhFg28H4vf1HiUOcV9oQ/ -JTM2qldaM5ALh+TFvBootXTS1iO9vKfbaGmfdeHibSyY13X1vvTI2GXTGWCaHxbz -1P/PEpjPFCeW6FfqwXJrv+iyB1NNdW0jHQjJzJGBeG7JxC6gDd+3GWA/yWDDKGyp -OlzKYqnqPd0+sDIPQPo1yis/4lwXrT/Wdac3Yvdmz0d9seUZ2LwSekpRZ1Phhxsk -2CsbKOaEF3w3VshoQWLFzATLuVGI25f7EcDzC0WaugrJvGhUMvJwuBXzFToaE1UL -IoNrF1IGxDjM3Qv0F623Sa3zBnDfw9kpcYFzAgMBAAGjITAfMB0GA1UdDgQWBBR5 -C6bMYEmcEtyFuTUAN3Ap55/OyzANBgkqhkiG9w0BAQwFAAOCAYEAoLn5vVYCKzJI -HTv4edUXs6evEqowSFj4dsQjEkwN2YF7MREmV/jWrrA0pRNbThUleMFtmsb5hLvT -Qxdl1eI7ntWHjDBJSLNSz55TD5+s03DyW6giHeRTTBZkuaHcmeL6csXIdRRucRba -nHQk+VVrOtp36JilBbU/cI+L9/JWNCTpOQCQnxn58yt1YoE8xAVVlKSmMPEsbzKA -dKXhIvo8xX/p0NQJ4ClPB++txZ1D/FlbG3N0OsLRAGTlbPFMZoHKMMFhg+PZNpPQ -3cdvMGEOJrk9dIF6p3g6JJRF7sNf5Q2IT1Wyzmdx1P92krx3BMdJGVQCDYdd2ZkU -v+vvaDzD5NTFOb+B7jEd23+zvCpqdqakHPUQDXMhTgGJPkq45Dp2ddAOHXhWF9RG -KU+xxomhNRETcLqNt0FP/9iETMmtgSf3FwTLm+Qro2pKBZtkJWHtP4cLNhi2Ikhj -ctlMm8xGfjzct2tQfWw4bP91S+g1t57ZSXOtEUNKOHHvPFk66b3+ ------END CERTIFICATE----- diff --git a/services/brig/brig.integration.yaml b/services/brig/brig.integration.yaml index e7c6d74f30..6114e56fa7 100644 --- a/services/brig/brig.integration.yaml +++ b/services/brig/brig.integration.yaml @@ -8,7 +8,6 @@ cassandra: port: 9042 keyspace: brig_test # filterNodesByDatacentre: datacenter1 - tlsCa: ../../hack/cassandra.cert.pem elasticsearch: url: http://127.0.0.1:9200 diff --git a/services/galley/galley.integration.yaml b/services/galley/galley.integration.yaml index 558dd7924d..e47801460b 100644 --- a/services/galley/galley.integration.yaml +++ b/services/galley/galley.integration.yaml @@ -8,7 +8,6 @@ cassandra: port: 9042 keyspace: galley_test # filterNodesByDatacentre: datacenter1 - tlsCa: ../../hack/cassandra.cert.pem brig: host: 0.0.0.0 diff --git a/services/gundeck/gundeck.integration.yaml b/services/gundeck/gundeck.integration.yaml index 6571221484..7ceadf3ad8 100644 --- a/services/gundeck/gundeck.integration.yaml +++ b/services/gundeck/gundeck.integration.yaml @@ -12,7 +12,6 @@ cassandra: port: 9042 keyspace: gundeck_test # filterNodesByDatacentre: datacenter1 - tlsCa: ../../hack/cassandra.cert.pem redis: host: 127.0.0.1 diff --git a/services/integration.yaml b/services/integration.yaml index 040a709cf1..65543e45f1 100644 --- a/services/integration.yaml +++ b/services/integration.yaml @@ -142,4 +142,3 @@ rabbitmq: cassandra: host: 127.0.0.1 port: 9042 - tlsCa: hack/cassandra.cert.pem diff --git a/services/spar/spar.integration.yaml b/services/spar/spar.integration.yaml index 31edba552d..6a1eb2f398 100644 --- a/services/spar/spar.integration.yaml +++ b/services/spar/spar.integration.yaml @@ -28,7 +28,6 @@ cassandra: port: 9042 keyspace: spar_test filterNodesByDatacentre: datacenter1 - tlsCa: ../../hack/cassandra.cert.pem # Wire/AWS specific, optional # discoUrl: "https://" From 169e01676f4f2e406778fee8bfd3dc1b5ff717de Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 28 Nov 2023 19:46:01 +0100 Subject: [PATCH 63/98] Deal with strange Helm value punning --- charts/brig/templates/configmap.yaml | 2 +- charts/brig/templates/deployment.yaml | 4 +- .../templates/tests/brig-integration.yaml | 4 +- .../templates/_helpers.tpl | 61 ++++++++++++------- .../templates/cassandra-certs.yaml | 8 +-- .../templates/galley-migrate-data.yaml | 6 +- .../templates/migrate-schema.yaml | 24 ++++---- .../templates/spar-migrate-data.yaml | 12 ++-- charts/cassandra-migrations/values.yaml | 8 --- .../templates/migrate-data.yaml | 6 +- charts/galley/templates/configmap.yaml | 2 +- charts/galley/templates/deployment.yaml | 4 +- .../templates/tests/galley-integration.yaml | 4 +- charts/gundeck/templates/configmap.yaml | 2 +- charts/gundeck/templates/deployment.yaml | 4 +- .../templates/tests/gundeck-integration.yaml | 4 +- charts/integration/templates/configmap.yaml | 2 +- .../templates/integration-integration.yaml | 8 +-- charts/spar/templates/configmap.yaml | 2 +- charts/spar/templates/deployment.yaml | 4 +- .../templates/tests/spar-integration.yaml | 4 +- 21 files changed, 93 insertions(+), 82 deletions(-) diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 77a5eb22c4..c6599c5a9d 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -28,7 +28,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraCA" .) }} + {{- if eq (include "useCassandraCA" .) "true" }} tlsCa: /etc/wire/brig/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/brig/templates/deployment.yaml b/charts/brig/templates/deployment.yaml index e563159a9e..357395a789 100644 --- a/charts/brig/templates/deployment.yaml +++ b/charts/brig/templates/deployment.yaml @@ -46,7 +46,7 @@ spec: - name: "geoip" emptyDir: {} {{- end }} - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "brig-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -107,7 +107,7 @@ spec: - name: "geoip" mountPath: "/usr/share/GeoIP" {{- end }} - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "brig-cassandra" mountPath: "/etc/wire/brig/cassandra" {{- end }} diff --git a/charts/brig/templates/tests/brig-integration.yaml b/charts/brig/templates/tests/brig-integration.yaml index 5dcc5cda12..3cc68fdad3 100644 --- a/charts/brig/templates/tests/brig-integration.yaml +++ b/charts/brig/templates/tests/brig-integration.yaml @@ -44,7 +44,7 @@ spec: - name: "brig-integration-secrets" secret: secretName: "brig-integration" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "brig-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -106,7 +106,7 @@ spec: # non-default locations # (see corresp. TODO in galley.) mountPath: "/etc/wire/integration-secrets" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "brig-cassandra" mountPath: "/etc/wire/brig/cassandra" {{- end }} diff --git a/charts/cassandra-migrations/templates/_helpers.tpl b/charts/cassandra-migrations/templates/_helpers.tpl index e3deb51632..d205368250 100644 --- a/charts/cassandra-migrations/templates/_helpers.tpl +++ b/charts/cassandra-migrations/templates/_helpers.tpl @@ -109,15 +109,20 @@ Thus the order of priority is: {{- define "useTlsGalley" -}} {{ $cassandraGalley := default dict .Values.cassandraGalley }} -{{- or .Values.cassandra.tlsCa $cassandraGalley.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGalley.tlsCaSecretRef -}} +{{- if or .Values.cassandra.tlsCa $cassandraGalley.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGalley.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} {{- end -}} {{- define "tlsCaGalley" -}} {{ $cassandraGalley := default dict .Values.cassandraGalley }} -{{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} -{{- else -}} -{{ $cassandraGalley.tlsCa }} +{{- if hasKey .Values.cassandra "tlsCa" -}} +{{- .Values.cassandra.tlsCa }} +{{- else if hasKey $cassandraGalley "tlsCa" -}} +{{- $cassandraGalley.tlsCa }} +{{ else }} {{- end -}} {{- end -}} @@ -134,15 +139,20 @@ Thus the order of priority is: {{- define "useTlsBrig" -}} {{ $cassandraBrig := default dict .Values.cassandraBrig }} -{{- or .Values.cassandra.tlsCa $cassandraBrig.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraBrig.tlsCaSecretRef -}} +{{- if or .Values.cassandra.tlsCa $cassandraBrig.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraBrig.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} {{- end -}} {{- define "tlsCaBrig" -}} {{ $cassandraBrig := default dict .Values.cassandraBrig }} -{{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} -{{- else -}} -{{ $cassandraBrig.tlsCa }} +{{- if hasKey .Values.cassandra "tlsCa" -}} +{{- .Values.cassandra.tlsCa }} +{{- else if hasKey $cassandraBrig "tlsCa" -}} +{{- $cassandraBrig.tlsCa }} +{{ else }} {{- end -}} {{- end -}} @@ -159,18 +169,22 @@ Thus the order of priority is: {{- define "useTlsSpar" -}} {{ $cassandraSpar := default dict .Values.cassandraSpar }} -{{- or .Values.cassandra.tlsCa $cassandraSpar.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraSpar.tlsCaSecretRef -}} +{{- if or .Values.cassandra.tlsCa $cassandraSpar.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraSpar.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} {{- end -}} {{- define "tlsCaSpar" -}} {{ $cassandraSpar := default dict .Values.cassandraSpar }} -{{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} -{{- else -}} -{{ $cassandraSpar.tlsCa }} +{{- if hasKey .Values.cassandra "tlsCa" -}} +{{- .Values.cassandra.tlsCa }} +{{- else if hasKey $cassandraSpar "tlsCa" -}} +{{- $cassandraSpar.tlsCa }} +{{ else }} {{- end -}} {{- end -}} - {{- define "tlsSecretRefSpar" -}} {{ $cassandraSpar := default dict .Values.cassandraSpar }} {{- if .Values.cassandra.tlsCaSecretRef -}} @@ -184,15 +198,20 @@ Thus the order of priority is: {{- define "useTlsGundeck" -}} {{ $cassandraGundeck := default dict .Values.cassandraGundeck }} -{{- or .Values.cassandra.tlsCa $cassandraGundeck.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGundeck.tlsCaSecretRef -}} +{{- if or .Values.cassandra.tlsCa $cassandraGundeck.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGundeck.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} {{- end -}} {{- define "tlsCaGundeck" -}} {{ $cassandraGundeck := default dict .Values.cassandraGundeck }} -{{- if .Values.cassandra.tlsCa -}} -{{ .Values.cassandra.tlsCa }} -{{- else -}} -{{ $cassandraGundeck.tlsCa }} +{{- if hasKey .Values.cassandra "tlsCa" -}} +{{- .Values.cassandra.tlsCa }} +{{- else if hasKey $cassandraGundeck "tlsCa" -}} +{{- $cassandraGundeck.tlsCa }} +{{ else }} {{- end -}} {{- end -}} diff --git a/charts/cassandra-migrations/templates/cassandra-certs.yaml b/charts/cassandra-migrations/templates/cassandra-certs.yaml index 988e573b4e..3bea0c6f5d 100644 --- a/charts/cassandra-migrations/templates/cassandra-certs.yaml +++ b/charts/cassandra-migrations/templates/cassandra-certs.yaml @@ -1,4 +1,4 @@ -{{- if not (empty (include "tlsCaBrig" .)) }} +{{- if ne (trim (include "tlsCaBrig" .)) "" }} apiVersion: v1 kind: Secret metadata: @@ -16,7 +16,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaBrig" . | b64enc | quote }} {{- end}} -{{- if not (empty (include "tlsCaGalley" .)) }} +{{- if ne (trim (include "tlsCaGalley" .)) "" }} --- apiVersion: v1 kind: Secret @@ -35,7 +35,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaGalley" . | b64enc | quote }} {{- end}} -{{- if not (empty (include "tlsCaGundeck" .)) }} +{{- if ne (trim (include "tlsCaGundeck" .)) "" }} --- apiVersion: v1 kind: Secret @@ -54,7 +54,7 @@ type: Opaque data: ca.pem: {{ include "tlsCaGundeck" . | b64enc | quote }} {{- end}} -{{- if not (empty (include "tlsCaSpar" .)) }} +{{- if ne (trim (include "tlsCaSpar" .)) "" }} --- apiVersion: v1 kind: Secret diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index 63083ce0bf..0e689817ac 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -42,16 +42,16 @@ spec: - "9042" - --cassandra-keyspace - galley - {{- if (include "useTlsGalley" .) }} + {{- if eq (include "useTlsGalley" .) "true" }} - --tls-certificate-file - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsGalley" .) }} + {{- if eq (include "useTlsGalley" .) "true" }} volumeMounts: - name: galley-cassandra-cert mountPath: "/certs/galley" {{- end }} - {{- if (include "useTlsGalley" .) }} + {{- if eq (include "useTlsGalley" .) "true" }} volumes: - name: galley-cassandra-cert secret: diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index 56544c56a1..f8b29e26ad 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -23,22 +23,22 @@ spec: # to avoid 'Column family ID mismatch' / schema disagreements # see https://stackoverflow.com/questions/29030661/creating-new-table-with-cqlsh-on-existing-keyspace-column-family-id-mismatch#40325651 for details. volumes: - {{- if (include "useTlsGundeck" .) }} + {{- if eq (include "useTlsGundeck" .) "true" }} - name: gundeck-cassandra-cert secret: secretName: {{ (include "tlsSecretRefGundeck" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsBrig" .) }} + {{- if eq (include "useTlsBrig" .) "true" }} - name: brig-cassandra-cert secret: secretName: {{ (include "tlsSecretRefBrig" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsGalley" .) }} + {{- if eq (include "useTlsGalley" .) "true" }} - name: galley-cassandra-cert secret: secretName: {{ (include "tlsSecretRefGalley" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsSpar" .) }} + {{- if eq (include "useTlsSpar" .) "true" }} - name: spar-cassandra-cert secret: secretName: {{ (include "tlsSecretRefSpar" . | fromYaml).name }} @@ -62,12 +62,12 @@ spec: - gundeck - {{ template "cassandraGundeckReplicationType" . }} - "{{ template "cassandraGundeckReplicationArg" . }}" - {{- if (include "useTlsGundeck" .) }} + {{- if eq (include "useTlsGundeck" .) "true" }} - --tls-certificate-file - /certs/gundeck/{{- (include "tlsSecretRefGundeck" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsGundeck" .) }} + {{- if eq (include "useTlsGundeck" .) "true" }} volumeMounts: - name: gundeck-cassandra-cert mountPath: "/certs/gundeck" @@ -92,12 +92,12 @@ spec: - brig - {{ template "cassandraBrigReplicationType" . }} - "{{ template "cassandraBrigReplicationArg" . }}" - {{- if (include "useTlsBrig" .) }} + {{- if eq (include "useTlsBrig" .) "true" }} - --tls-certificate-file - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsBrig" .) }} + {{- if eq (include "useTlsBrig" .) "true" }} volumeMounts: - name: brig-cassandra-cert mountPath: "/certs/brig" @@ -122,12 +122,12 @@ spec: - galley - {{ template "cassandraGalleyReplicationType" . }} - "{{ template "cassandraGalleyReplicationArg" . }}" - {{- if (include "useTlsGalley" .) }} + {{- if eq (include "useTlsGalley" .) "true" }} - --tls-certificate-file - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsGalley" .) }} + {{- if eq (include "useTlsGalley" .) "true" }} volumeMounts: - name: galley-cassandra-cert mountPath: "/certs/galley" @@ -152,12 +152,12 @@ spec: - spar - {{ template "cassandraSparReplicationType" . }} - "{{ template "cassandraSparReplicationArg" . }}" - {{- if (include "useTlsSpar" .) }} + {{- if eq (include "useTlsSpar" .) "true" }} - --tls-certificate-file - /certs/spar/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsSpar" .) }} + {{- if eq (include "useTlsSpar" .) "true" }} volumeMounts: - name: spar-cassandra-cert mountPath: "/certs/spar" diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index 239d2b94cb..3fb6157609 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -43,30 +43,30 @@ spec: - "9042" - --cassandra-keyspace-brig - brig - {{- if (include "useTlsBrig" .) }} + {{- if eq (include "useTlsBrig" .) "true" }} - --tls-certificate-file-brig - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} {{- end }} - {{- if (include "useTlsSpar" .) }} + {{- if eq (include "useTlsSpar" .) "true" }} - --tls-certificate-file-spar - /certs/spar/{{- (include "tlsSecretRefSpar" . | fromYaml).key }} {{- end }} volumeMounts: - {{- if (include "useTlsBrig" .) }} + {{- if eq (include "useTlsBrig" .) "true" }} - name: brig-cassandra-cert mountPath: "/certs/brig" {{- end }} - {{- if (include "useTlsSpar" .) }} + {{- if eq (include "useTlsSpar" .) "true" }} - name: spar-cassandra-cert mountPath: "/certs/spar" {{- end }} volumes: - {{- if (include "useTlsBrig" .) }} + {{- if eq (include "useTlsBrig" .) "true" }} - name: brig-cassandra-cert secret: secretName: {{ (include "tlsSecretRefBrig" . | fromYaml).name }} {{- end }} - {{- if (include "useTlsSpar" .) }} + {{- if eq (include "useTlsSpar" .) "true" }} - name: spar-cassandra-cert secret: secretName: {{ (include "tlsSecretRefSpar" . | fromYaml).name }} diff --git a/charts/cassandra-migrations/values.yaml b/charts/cassandra-migrations/values.yaml index f80e3448fa..283a010884 100644 --- a/charts/cassandra-migrations/values.yaml +++ b/charts/cassandra-migrations/values.yaml @@ -86,11 +86,3 @@ podSecurityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - -#cassandra: -# host: cassandra-external -# replicationFactor: 3 -## tlsCa: "foo" -# tlsCaSecretRef: -# name: refN -# key: refK diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index e119401061..4ddf807d3e 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -43,16 +43,16 @@ spec: - "{{ required "missing elasticsearch-index.galley.host!" .Values.galley.host }}" - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" - {{- if (include "useCassandraCA" .Values) }} + {{- if eq (include "useCassandraCA" .Values) "true" }} - --tls-certificate-file - /certs/{{- (include "tlsSecretRef" .Values | fromYaml).key }} {{- end }} - {{- if (include "useCassandraCA" .Values) }} + {{- if eq (include "useCassandraCA" .Values) "true" }} volumeMounts: - name: elasticsearch-index-migrate-cassandra-client-ca mountPath: "/certs" {{- end }} - {{- if (include "useCassandraCA" .Values) }} + {{- if eq (include "useCassandraCA" .Values) "true" }} volumes: - name: elasticsearch-index-migrate-cassandra-client-ca secret: diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 461c8f4324..0599bde5b7 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -21,7 +21,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraCA" .) }} + {{- if eq (include "useCassandraCA" .) "true" }} tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml index 084d49c9e3..a2d8e8a10e 100644 --- a/charts/galley/templates/deployment.yaml +++ b/charts/galley/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: - name: "galley-secrets" secret: secretName: "galley" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "galley-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -54,7 +54,7 @@ spec: mountPath: "/etc/wire/galley/conf" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" - {{- if (include "useCassandraCA" .Values.config)}} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "galley-cassandra" mountPath: "/etc/wire/galley/cassandra" {{- end }} diff --git a/charts/galley/templates/tests/galley-integration.yaml b/charts/galley/templates/tests/galley-integration.yaml index a87471f9a7..461a86edce 100644 --- a/charts/galley/templates/tests/galley-integration.yaml +++ b/charts/galley/templates/tests/galley-integration.yaml @@ -40,7 +40,7 @@ spec: - name: "galley-secrets" secret: secretName: "galley" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "galley-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -89,7 +89,7 @@ spec: mountPath: "/etc/wire/integration-secrets" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" - {{- if (include "useCassandraCA" .Values.config)}} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "galley-cassandra" mountPath: "/etc/wire/galley/cassandra" {{- end }} diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index 7962df079c..d421f9a7b4 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -25,7 +25,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraCA" .) }} + {{- if eq (include "useCassandraCA" .) "true" }} tlsCa: /etc/wire/gundeck/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml index b79a9e9725..b4f829922e 100644 --- a/charts/gundeck/templates/deployment.yaml +++ b/charts/gundeck/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "gundeck-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -48,7 +48,7 @@ spec: volumeMounts: - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} diff --git a/charts/gundeck/templates/tests/gundeck-integration.yaml b/charts/gundeck/templates/tests/gundeck-integration.yaml index 2413f61588..a135f9d8fd 100644 --- a/charts/gundeck/templates/tests/gundeck-integration.yaml +++ b/charts/gundeck/templates/tests/gundeck-integration.yaml @@ -13,7 +13,7 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "gundeck-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -59,7 +59,7 @@ spec: mountPath: "/etc/wire/integration" - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} diff --git a/charts/integration/templates/configmap.yaml b/charts/integration/templates/configmap.yaml index c3b829fc70..b49e919b05 100644 --- a/charts/integration/templates/configmap.yaml +++ b/charts/integration/templates/configmap.yaml @@ -122,6 +122,6 @@ data: cassandra: host: {{ .Values.config.cassandra.host }} port: {{ .Values.config.cassandra.port }} - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} {{- end }} diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index a69ee0ea1e..2011234aab 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -75,7 +75,7 @@ spec: - name: "nginz-secrets" secret: secretName: "nginz" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: integration-cassandra secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -91,7 +91,7 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 6 }} {{- end }} volumeMounts: - {{- if (include "useCassandraCA" .Values.config)}} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "integration-cassandra" mountPath: "/certs" {{- end }} @@ -120,7 +120,7 @@ spec: - | set -euo pipefail # FUTUREWORK: Do all of this in the integration test binary - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} {{- else }} integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} @@ -225,7 +225,7 @@ spec: - name: nginz-secrets mountPath: /etc/wire/nginz/secrets - {{- if (include "useCassandraCA" .Values.config)}} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "integration-cassandra" mountPath: "/certs" diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index 675c593381..a341839951 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -25,7 +25,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if (include "useCassandraCA" .) }} + {{- if eq (include "useCassandraCA" .) "true" }} tlsCa: /etc/wire/spar/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/spar/templates/deployment.yaml b/charts/spar/templates/deployment.yaml index 000a34961f..4d7eb9ed63 100644 --- a/charts/spar/templates/deployment.yaml +++ b/charts/spar/templates/deployment.yaml @@ -30,7 +30,7 @@ spec: - name: "spar-config" configMap: name: "spar" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "spar-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -46,7 +46,7 @@ spec: volumeMounts: - name: "spar-config" mountPath: "/etc/wire/spar/conf" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "spar-cassandra" mountPath: "/etc/wire/spar/cassandra" {{- end }} diff --git a/charts/spar/templates/tests/spar-integration.yaml b/charts/spar/templates/tests/spar-integration.yaml index bcec26a64b..f2b3b02a80 100644 --- a/charts/spar/templates/tests/spar-integration.yaml +++ b/charts/spar/templates/tests/spar-integration.yaml @@ -16,7 +16,7 @@ spec: - name: "spar-config" configMap: name: "spar" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "spar-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -61,7 +61,7 @@ spec: mountPath: "/etc/wire/integration" - name: "spar-config" mountPath: "/etc/wire/spar/conf" - {{- if (include "useCassandraCA" .Values.config) }} + {{- if eq (include "useCassandraCA" .Values.config) "true" }} - name: "spar-cassandra" mountPath: "/etc/wire/spar/cassandra" {{- end }} From 05badb22538d280bdac42bde45e6a6137db22f62 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 28 Nov 2023 19:46:58 +0100 Subject: [PATCH 64/98] Provide environments for integration testing --- hack/bin/integration-setup-federation.sh | 1 + .../{values.yaml => values.yaml.gotmpl} | 2 +- hack/helm_vars/wire-server/values.yaml.gotmpl | 28 ++++++++--- hack/helmfile.yaml | 46 +++++++++++-------- 4 files changed, 50 insertions(+), 27 deletions(-) rename hack/helm_vars/k8ssandra-test-cluster/{values.yaml => values.yaml.gotmpl} (85%) diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh index 294c719cea..d7e19e66ae 100755 --- a/hack/bin/integration-setup-federation.sh +++ b/hack/bin/integration-setup-federation.sh @@ -5,6 +5,7 @@ set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TOP_LEVEL="$DIR/../.." export NAMESPACE=${NAMESPACE:-test-integration} +# Available $HELMFILE_ENV profiles: default, default-ssl, kind, kind-ssl HELMFILE_ENV=${HELMFILE_ENV:-default} CHARTS_DIR="${TOP_LEVEL}/.local/charts" HELM_PARALLELISM=${HELM_PARALLELISM:-1} diff --git a/hack/helm_vars/k8ssandra-test-cluster/values.yaml b/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl similarity index 85% rename from hack/helm_vars/k8ssandra-test-cluster/values.yaml rename to hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl index 8a072a456e..89c1005fd6 100644 --- a/hack/helm_vars/k8ssandra-test-cluster/values.yaml +++ b/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl @@ -1,4 +1,4 @@ -storageClassName: hcloud-volumes +storageClassName: {{ .Values.storageClass }} client_encryption_options: enabled: true diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 4d63c0f315..02c8c47499 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -18,11 +18,13 @@ tags: cassandra-migrations: imagePullPolicy: {{ .Values.imagePullPolicy }} cassandra: - host: k8ssandra-cluster-datacenter-1-service + host: {{ .Values.cassandraHost }} replicationFactor: 1 + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: "cassandra-jks-keystore" key: "ca.crt" + {{- end }} elasticsearch-index: imagePullPolicy: {{ .Values.imagePullPolicy }} @@ -30,10 +32,12 @@ elasticsearch-index: host: elasticsearch-ephemeral index: directory_test cassandra: - host: k8ssandra-cluster-datacenter-1-service + host: {{ .Values.cassandraHost }} + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: "cassandra-jks-keystore" key: "ca.crt" + {{- end }} brig: replicaCount: 1 @@ -48,11 +52,13 @@ brig: teamCreatorWelcome: https://teams.wire.com/login teamMemberWelcome: https://wire.com/download cassandra: - host: k8ssandra-cluster-datacenter-1-service + host: {{ .Values.cassandraHost }} replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: "cassandra-jks-keystore" key: "ca.crt" + {{- end }} elasticsearch: host: elasticsearch-ephemeral index: directory_test @@ -196,11 +202,13 @@ galley: imagePullPolicy: {{ .Values.imagePullPolicy }} config: cassandra: - host: k8ssandra-cluster-datacenter-1-service + host: {{ .Values.cassandraHost }} replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: "cassandra-jks-keystore" key: "ca.crt" + {{- end }} enableFederation: true # keep in sync with brig.config.enableFederation, cargohold.config.enableFederation and tags.federator! settings: maxConvAndTeamSize: 16 @@ -261,11 +269,13 @@ gundeck: memory: 1024Mi config: cassandra: - host: k8ssandra-cluster-datacenter-1-service + host: {{ .Values.cassandraHost }} replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: "cassandra-jks-keystore" key: "ca.crt" + {{- end }} redis: host: redis-ephemeral-master connectionMode: master @@ -338,10 +348,12 @@ spar: config: tlsDisableCertValidation: true cassandra: - host: k8ssandra-cluster-datacenter-1-service + host: {{ .Values.cassandraHost }} + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: "cassandra-jks-keystore" key: "ca.crt" + {{- end }} logLevel: Debug domain: zinfra.io appUri: http://spar:8080/ @@ -401,12 +413,14 @@ integration: class: "nginx-{{ .Release.Namespace }}" config: cassandra: - host: k8ssandra-cluster-datacenter-1-service + host: {{ .Values.cassandraHost }} port: 9042 replicationFactor: 1 + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: cassandra-jks-keystore key: ca.crt + {{- end }} {{- if .Values.uploadXml }} uploadXml: baseUrl: {{ .Values.uploadXml.baseUrl }} diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 5e30d62a66..e0c01fdb3c 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -11,18 +11,40 @@ helmDefaults: timeout: 600 devel: true createNamespace: true - +# k8ssandra-cluster-datacenter-1-service environments: default: values: - ./helm_vars/common.yaml.gotmpl - imagePullPolicy: Always - storageClass: hcloud-volumes + - cassandraHost: cassandra-ephemeral + - useK8ssandraSSL: + enabled: false + default-ssl: + values: + - ./helm_vars/common.yaml.gotmpl + - imagePullPolicy: Always + - storageClass: hcloud-volumes + - cassandraHost: k8ssandra-cluster-datacenter-1-service + - useK8ssandraSSL: + enabled: true kind: values: - ./helm_vars/common.yaml.gotmpl - imagePullPolicy: Never - storageClass: standard + - cassandraHost: cassandra-ephemeral + - useK8ssandraSSL: + enabled: false + kind-ssl: + values: + - ./helm_vars/common.yaml.gotmpl + - imagePullPolicy: Never + - storageClass: standard + - cassandraHost: k8ssandra-cluster-datacenter-1-service + - useK8ssandraSSL: + enabled: true --- repositories: - name: stable @@ -54,33 +76,19 @@ releases: namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/databases-ephemeral' -# - name: 'redis-ephemeral' -# namespace: '{{ .Values.namespace1 }}' -# chart: '../.local/charts/redis-ephemeral' -# -# - name: 'redis-ephemeral' -# namespace: '{{ .Values.namespace2 }}' -# chart: '../.local/charts/redis-ephemeral' -# -# - name: 'elasticsearch-ephemeral' -# namespace: '{{ .Values.namespace1 }}' -# chart: '../.local/charts/redis-ephemeral' -# -# - name: 'elasticsearch-ephemeral' -# namespace: '{{ .Values.namespace2 }}' -# chart: '../.local/charts/redis-ephemeral' - - name: k8ssandra-test-cluster chart: '../.local/charts/k8ssandra-test-cluster' namespace: '{{ .Values.namespace1 }}' values: - - './helm_vars/k8ssandra-test-cluster/values.yaml' + - './helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl' + condition: useK8ssandraSSL.enabled - name: k8ssandra-test-cluster chart: '../.local/charts/k8ssandra-test-cluster' namespace: '{{ .Values.namespace2 }}' values: - - './helm_vars/k8ssandra-test-cluster/values.yaml' + - './helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl' + condition: useK8ssandraSSL.enabled - name: 'rabbitmq' namespace: '{{ .Values.namespace1 }}' From 7fb9f580f80fa04183fa6cc63a11ddca5bd37fa7 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 28 Nov 2023 20:03:00 +0100 Subject: [PATCH 65/98] Consider profile when destorying the Helmfile env Otherwise, K8ssandra things may leak. --- hack/bin/integration-teardown-federation.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/bin/integration-teardown-federation.sh b/hack/bin/integration-teardown-federation.sh index fba9685490..01791d223c 100755 --- a/hack/bin/integration-teardown-federation.sh +++ b/hack/bin/integration-teardown-federation.sh @@ -5,6 +5,7 @@ TOP_LEVEL="$DIR/../.." set -ex +HELMFILE_ENV=${HELMFILE_ENV:-default} NAMESPACE=${NAMESPACE:-test-integration} export NAMESPACE_1="$NAMESPACE" export NAMESPACE_2="$NAMESPACE-fed2" @@ -22,6 +23,6 @@ else fi . "$DIR/helm_overrides.sh" -helmfile --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy --skip-deps --skip-charts --concurrency 0 || echo "Failed to delete helm deployments, ignoring this failure as next steps will the destroy namespaces anyway." +helmfile --environment "$HELMFILE_ENV" --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy --skip-deps --skip-charts --concurrency 0 || echo "Failed to delete helm deployments, ignoring this failure as next steps will the destroy namespaces anyway." kubectl delete namespace "$NAMESPACE_1" "$NAMESPACE_2" From 24bd91d0ebbd49f9b6b70737acdb2ba68c3525d9 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 29 Nov 2023 16:32:15 +0100 Subject: [PATCH 66/98] Hi CI From ee094dec078a6b8aed99fb21c7501aa075fa0d75 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 09:47:06 +0100 Subject: [PATCH 67/98] Improve changelog --- changelog.d/2-features/cassandra-tls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/2-features/cassandra-tls b/changelog.d/2-features/cassandra-tls index 3e3195f1e7..8b0045fd5d 100644 --- a/changelog.d/2-features/cassandra-tls +++ b/changelog.d/2-features/cassandra-tls @@ -1,6 +1,6 @@ Allow the configuration of TLS-secured connections to Cassandra. TLS is used when a certificate is provided. This is either done with -`--tls-certificate-file` for migrations or the configuration attribute +`--tls-certificate-file` for cli commands or the configuration attribute `cassandra.tlsCa` for services. In Helm charts, the certificate is provided as -PEM string in the attribute `cassandra.tlsCa` (analog to service -configuration.) +literal PEM string; either as attribute `cassandra.tlsCa` (analog to service +configuration) or by a reference to a secret (`cassandra.tlsCaSecretRef`.) From 82b5e7e373cf1f3570ed2a08367d1e81d6cd3a6d Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 10:39:49 +0100 Subject: [PATCH 68/98] Better name: useCassandraCA -> useCassandraTLS --- charts/brig/templates/_helpers.tpl | 2 +- charts/brig/templates/configmap.yaml | 2 +- charts/brig/templates/deployment.yaml | 4 ++-- charts/brig/templates/tests/brig-integration.yaml | 4 ++-- charts/elasticsearch-index/templates/_helpers.tpl | 2 +- charts/elasticsearch-index/templates/migrate-data.yaml | 6 +++--- charts/galley/templates/_helpers.tpl | 2 +- charts/galley/templates/configmap.yaml | 2 +- charts/galley/templates/deployment.yaml | 4 ++-- charts/galley/templates/tests/galley-integration.yaml | 4 ++-- charts/gundeck/templates/_helpers.tpl | 2 +- charts/gundeck/templates/configmap.yaml | 2 +- charts/gundeck/templates/deployment.yaml | 4 ++-- charts/gundeck/templates/tests/gundeck-integration.yaml | 4 ++-- charts/integration/templates/_helpers.tpl | 2 +- charts/integration/templates/configmap.yaml | 2 +- charts/integration/templates/integration-integration.yaml | 8 ++++---- charts/spar/templates/_helpers.tpl | 2 +- charts/spar/templates/configmap.yaml | 2 +- charts/spar/templates/deployment.yaml | 4 ++-- charts/spar/templates/tests/spar-integration.yaml | 4 ++-- 21 files changed, 34 insertions(+), 34 deletions(-) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index d7546dced8..a55b90faa1 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -8,7 +8,7 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraCA" -}} +{{- define "useCassandraTLS" -}} {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index c6599c5a9d..f2a43d2ed8 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -28,7 +28,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if eq (include "useCassandraCA" .) "true" }} + {{- if eq (include "useCassandraTLS" .) "true" }} tlsCa: /etc/wire/brig/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/brig/templates/deployment.yaml b/charts/brig/templates/deployment.yaml index 357395a789..bc1261391b 100644 --- a/charts/brig/templates/deployment.yaml +++ b/charts/brig/templates/deployment.yaml @@ -46,7 +46,7 @@ spec: - name: "geoip" emptyDir: {} {{- end }} - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "brig-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -107,7 +107,7 @@ spec: - name: "geoip" mountPath: "/usr/share/GeoIP" {{- end }} - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "brig-cassandra" mountPath: "/etc/wire/brig/cassandra" {{- end }} diff --git a/charts/brig/templates/tests/brig-integration.yaml b/charts/brig/templates/tests/brig-integration.yaml index 3cc68fdad3..aff0f6d525 100644 --- a/charts/brig/templates/tests/brig-integration.yaml +++ b/charts/brig/templates/tests/brig-integration.yaml @@ -44,7 +44,7 @@ spec: - name: "brig-integration-secrets" secret: secretName: "brig-integration" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "brig-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -106,7 +106,7 @@ spec: # non-default locations # (see corresp. TODO in galley.) mountPath: "/etc/wire/integration-secrets" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "brig-cassandra" mountPath: "/etc/wire/brig/cassandra" {{- end }} diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl index 9638afd547..4c48aac06f 100644 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ b/charts/elasticsearch-index/templates/_helpers.tpl @@ -8,7 +8,7 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraCA" -}} +{{- define "useCassandraTLS" -}} {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index 4ddf807d3e..ac8836d324 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -43,16 +43,16 @@ spec: - "{{ required "missing elasticsearch-index.galley.host!" .Values.galley.host }}" - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" - {{- if eq (include "useCassandraCA" .Values) "true" }} + {{- if eq (include "useCassandraTLS" .Values) "true" }} - --tls-certificate-file - /certs/{{- (include "tlsSecretRef" .Values | fromYaml).key }} {{- end }} - {{- if eq (include "useCassandraCA" .Values) "true" }} + {{- if eq (include "useCassandraTLS" .Values) "true" }} volumeMounts: - name: elasticsearch-index-migrate-cassandra-client-ca mountPath: "/certs" {{- end }} - {{- if eq (include "useCassandraCA" .Values) "true" }} + {{- if eq (include "useCassandraTLS" .Values) "true" }} volumes: - name: elasticsearch-index-migrate-cassandra-client-ca secret: diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index 2e73ecfe6a..cd920f29cd 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -8,7 +8,7 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraCA" -}} +{{- define "useCassandraTLS" -}} {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 0599bde5b7..3ac139136d 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -21,7 +21,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if eq (include "useCassandraCA" .) "true" }} + {{- if eq (include "useCassandraTLS" .) "true" }} tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml index a2d8e8a10e..df9eee0c20 100644 --- a/charts/galley/templates/deployment.yaml +++ b/charts/galley/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: - name: "galley-secrets" secret: secretName: "galley" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "galley-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -54,7 +54,7 @@ spec: mountPath: "/etc/wire/galley/conf" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "galley-cassandra" mountPath: "/etc/wire/galley/cassandra" {{- end }} diff --git a/charts/galley/templates/tests/galley-integration.yaml b/charts/galley/templates/tests/galley-integration.yaml index 461a86edce..1fdd9e206a 100644 --- a/charts/galley/templates/tests/galley-integration.yaml +++ b/charts/galley/templates/tests/galley-integration.yaml @@ -40,7 +40,7 @@ spec: - name: "galley-secrets" secret: secretName: "galley" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "galley-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -89,7 +89,7 @@ spec: mountPath: "/etc/wire/integration-secrets" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "galley-cassandra" mountPath: "/etc/wire/galley/cassandra" {{- end }} diff --git a/charts/gundeck/templates/_helpers.tpl b/charts/gundeck/templates/_helpers.tpl index 8876add186..f743b4e392 100644 --- a/charts/gundeck/templates/_helpers.tpl +++ b/charts/gundeck/templates/_helpers.tpl @@ -8,7 +8,7 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraCA" -}} +{{- define "useCassandraTLS" -}} {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index d421f9a7b4..b01a63d844 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -25,7 +25,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if eq (include "useCassandraCA" .) "true" }} + {{- if eq (include "useCassandraTLS" .) "true" }} tlsCa: /etc/wire/gundeck/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml index b4f829922e..20ca798824 100644 --- a/charts/gundeck/templates/deployment.yaml +++ b/charts/gundeck/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "gundeck-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -48,7 +48,7 @@ spec: volumeMounts: - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} diff --git a/charts/gundeck/templates/tests/gundeck-integration.yaml b/charts/gundeck/templates/tests/gundeck-integration.yaml index a135f9d8fd..8b00f2c986 100644 --- a/charts/gundeck/templates/tests/gundeck-integration.yaml +++ b/charts/gundeck/templates/tests/gundeck-integration.yaml @@ -13,7 +13,7 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "gundeck-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -59,7 +59,7 @@ spec: mountPath: "/etc/wire/integration" - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "gundeck-cassandra" mountPath: "/etc/wire/gundeck/cassandra" {{- end }} diff --git a/charts/integration/templates/_helpers.tpl b/charts/integration/templates/_helpers.tpl index 97f485a450..7a134d5763 100644 --- a/charts/integration/templates/_helpers.tpl +++ b/charts/integration/templates/_helpers.tpl @@ -38,7 +38,7 @@ {{- (semverCompare ">= 1.23-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraCA" -}} +{{- define "useCassandraTLS" -}} {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} diff --git a/charts/integration/templates/configmap.yaml b/charts/integration/templates/configmap.yaml index b49e919b05..e18128cbf5 100644 --- a/charts/integration/templates/configmap.yaml +++ b/charts/integration/templates/configmap.yaml @@ -122,6 +122,6 @@ data: cassandra: host: {{ .Values.config.cassandra.host }} port: {{ .Values.config.cassandra.port }} - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} {{- end }} diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 2011234aab..2818063a45 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -75,7 +75,7 @@ spec: - name: "nginz-secrets" secret: secretName: "nginz" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: integration-cassandra secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -91,7 +91,7 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 6 }} {{- end }} volumeMounts: - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "integration-cassandra" mountPath: "/certs" {{- end }} @@ -120,7 +120,7 @@ spec: - | set -euo pipefail # FUTUREWORK: Do all of this in the integration test binary - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} {{- else }} integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} @@ -225,7 +225,7 @@ spec: - name: nginz-secrets mountPath: /etc/wire/nginz/secrets - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "integration-cassandra" mountPath: "/certs" diff --git a/charts/spar/templates/_helpers.tpl b/charts/spar/templates/_helpers.tpl index 5f9280a808..1afa528dff 100644 --- a/charts/spar/templates/_helpers.tpl +++ b/charts/spar/templates/_helpers.tpl @@ -7,7 +7,7 @@ {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} -{{- define "useCassandraCA" -}} +{{- define "useCassandraTLS" -}} {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index a341839951..8ae7b5c371 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -25,7 +25,7 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} - {{- if eq (include "useCassandraCA" .) "true" }} + {{- if eq (include "useCassandraTLS" .) "true" }} tlsCa: /etc/wire/spar/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} {{- end }} diff --git a/charts/spar/templates/deployment.yaml b/charts/spar/templates/deployment.yaml index 4d7eb9ed63..c09fc2beac 100644 --- a/charts/spar/templates/deployment.yaml +++ b/charts/spar/templates/deployment.yaml @@ -30,7 +30,7 @@ spec: - name: "spar-config" configMap: name: "spar" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "spar-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -46,7 +46,7 @@ spec: volumeMounts: - name: "spar-config" mountPath: "/etc/wire/spar/conf" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "spar-cassandra" mountPath: "/etc/wire/spar/cassandra" {{- end }} diff --git a/charts/spar/templates/tests/spar-integration.yaml b/charts/spar/templates/tests/spar-integration.yaml index f2b3b02a80..9cae732bfb 100644 --- a/charts/spar/templates/tests/spar-integration.yaml +++ b/charts/spar/templates/tests/spar-integration.yaml @@ -16,7 +16,7 @@ spec: - name: "spar-config" configMap: name: "spar" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "spar-cassandra" secret: secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} @@ -61,7 +61,7 @@ spec: mountPath: "/etc/wire/integration" - name: "spar-config" mountPath: "/etc/wire/spar/conf" - {{- if eq (include "useCassandraCA" .Values.config) "true" }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "spar-cassandra" mountPath: "/etc/wire/spar/cassandra" {{- end }} From 3cca3e7c456728615880f89968d66b93a7c04e29 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 10:47:18 +0100 Subject: [PATCH 69/98] Add comments --- charts/brig/templates/_helpers.tpl | 4 ++++ charts/elasticsearch-index/templates/_helpers.tpl | 4 ++++ charts/galley/templates/_helpers.tpl | 4 ++++ charts/gundeck/templates/_helpers.tpl | 4 ++++ charts/integration/templates/_helpers.tpl | 4 ++++ charts/spar/templates/_helpers.tpl | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index a55b90faa1..857c0203de 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -12,6 +12,10 @@ {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} {{- define "tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl index 4c48aac06f..47bf703112 100644 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ b/charts/elasticsearch-index/templates/_helpers.tpl @@ -12,6 +12,10 @@ {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} {{- define "tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index cd920f29cd..a9de4a20a9 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -12,6 +12,10 @@ {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} {{- define "tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} diff --git a/charts/gundeck/templates/_helpers.tpl b/charts/gundeck/templates/_helpers.tpl index f743b4e392..ed317e0b21 100644 --- a/charts/gundeck/templates/_helpers.tpl +++ b/charts/gundeck/templates/_helpers.tpl @@ -12,6 +12,10 @@ {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} {{- define "tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} diff --git a/charts/integration/templates/_helpers.tpl b/charts/integration/templates/_helpers.tpl index 7a134d5763..e278f287d1 100644 --- a/charts/integration/templates/_helpers.tpl +++ b/charts/integration/templates/_helpers.tpl @@ -42,6 +42,10 @@ {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} {{- define "tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} diff --git a/charts/spar/templates/_helpers.tpl b/charts/spar/templates/_helpers.tpl index 1afa528dff..958a0acc36 100644 --- a/charts/spar/templates/_helpers.tpl +++ b/charts/spar/templates/_helpers.tpl @@ -11,6 +11,10 @@ {{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} {{- end -}} +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} {{- define "tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} {{ .cassandra.tlsCaSecretRef | toYaml }} From b0859dcd2728e8612d9141cb540c598ccb2bcbcf Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 10:53:03 +0100 Subject: [PATCH 70/98] Add comments about cassandra secrets --- charts/brig/templates/cassandra-secret.yaml | 1 + charts/galley/templates/cassandra-secret.yaml | 1 + charts/gundeck/templates/cassandra-secret.yaml | 1 + charts/integration/templates/cassandra-secret.yaml | 1 + charts/spar/templates/cassandra-secret.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml index 8130e4324d..fa84800147 100644 --- a/charts/brig/templates/cassandra-secret.yaml +++ b/charts/brig/templates/cassandra-secret.yaml @@ -1,3 +1,4 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} {{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml index 032a6c361d..eb34aeb30b 100644 --- a/charts/galley/templates/cassandra-secret.yaml +++ b/charts/galley/templates/cassandra-secret.yaml @@ -1,3 +1,4 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} {{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml index 14c531896a..68dd7c9d34 100644 --- a/charts/gundeck/templates/cassandra-secret.yaml +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -1,3 +1,4 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} {{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret diff --git a/charts/integration/templates/cassandra-secret.yaml b/charts/integration/templates/cassandra-secret.yaml index ab49db0039..dd76b65067 100644 --- a/charts/integration/templates/cassandra-secret.yaml +++ b/charts/integration/templates/cassandra-secret.yaml @@ -1,3 +1,4 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} {{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml index 6912dd988f..0a480e01bb 100644 --- a/charts/spar/templates/cassandra-secret.yaml +++ b/charts/spar/templates/cassandra-secret.yaml @@ -1,3 +1,4 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} {{- if not (empty .Values.config.cassandra.tlsCa) }} apiVersion: v1 kind: Secret From cfc3e717da806d46d27cfc4e145e8653cc13dcef Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 10:58:59 +0100 Subject: [PATCH 71/98] Unify comments about TLS in values.yaml(s) --- charts/brig/values.yaml | 2 +- charts/elasticsearch-index/values.yaml | 6 +++++- charts/galley/values.yaml | 6 +++++- charts/gundeck/values.yaml | 6 +++++- charts/spar/values.yaml | 6 +++++- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index a8845f82b6..5774c277ea 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -21,7 +21,7 @@ config: cassandra: host: aws-cassandra # To enable TLS: -# tlsCa: # Or: # tlsCaSecretRef: # name: diff --git a/charts/elasticsearch-index/values.yaml b/charts/elasticsearch-index/values.yaml index 81ee2ff043..c0a937b895 100644 --- a/charts/elasticsearch-index/values.yaml +++ b/charts/elasticsearch-index/values.yaml @@ -9,7 +9,11 @@ cassandra: port: 9042 keyspace: brig # To enable TLS: -# tlsCa: +# Or: +# tlsCaSecretRef: +# name: +# key: galley: host: galley port: 8080 diff --git a/charts/galley/values.yaml b/charts/galley/values.yaml index facae104e5..dbbbe7383e 100644 --- a/charts/galley/values.yaml +++ b/charts/galley/values.yaml @@ -23,7 +23,11 @@ config: host: aws-cassandra replicaCount: 3 # To enable TLS: -# tlsCa: +# Or: +# tlsCaSecretRef: +# name: +# key: enableFederation: false # keep enableFederation default in sync with brig and cargohold chart's config.enableFederation as well as wire-server chart's tags.federation # Not used if enableFederation is false rabbitmq: diff --git a/charts/gundeck/values.yaml b/charts/gundeck/values.yaml index 450a052302..bdb600f43d 100644 --- a/charts/gundeck/values.yaml +++ b/charts/gundeck/values.yaml @@ -21,7 +21,11 @@ config: cassandra: host: aws-cassandra # To enable TLS: -# tlsCa: +# Or: +# tlsCaSecretRef: +# name: +# key: redis: host: redis-ephemeral-master port: 6379 diff --git a/charts/spar/values.yaml b/charts/spar/values.yaml index 228b48b590..048c485ffc 100644 --- a/charts/spar/values.yaml +++ b/charts/spar/values.yaml @@ -18,7 +18,11 @@ config: cassandra: host: aws-cassandra # To enable TLS: -# tlsCa: +# Or: +# tlsCaSecretRef: +# name: +# key: richInfoLimit: 5000 maxScimTokens: 0 logLevel: Info From fe39cfff08509c8e03caa23a59a96ed491447958 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 16:01:10 +0100 Subject: [PATCH 72/98] cassandra-migrations: Specific config wins over general one --- .../templates/_helpers.tpl | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/charts/cassandra-migrations/templates/_helpers.tpl b/charts/cassandra-migrations/templates/_helpers.tpl index d205368250..e4c875e285 100644 --- a/charts/cassandra-migrations/templates/_helpers.tpl +++ b/charts/cassandra-migrations/templates/_helpers.tpl @@ -107,9 +107,24 @@ Thus the order of priority is: {{- end -}} {{- end -}} +{{/* NOTE: Cassandra TLS helpers + +Cassandra connections can be configured per service or with a general configuration. +Thus, there are three functions per service that fallback to the general +configuration if the specific one does not exist: + +- useTls -> Bool: Do we use Cassandra TLS connections for this + service? + +- tlsCa -> String: TLS CA PEM string (if configured) + +- tlsSecretRefGalley -> YAML: Dict with keys `name` (name of the + secret to use) and `key` (name of the entry in the secret) +*/}} + {{- define "useTlsGalley" -}} -{{ $cassandraGalley := default dict .Values.cassandraGalley }} -{{- if or .Values.cassandra.tlsCa $cassandraGalley.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGalley.tlsCaSecretRef -}} +{{ $cassandraGalley := default .Values.cassandra .Values.cassandraGalley }} +{{- if or $cassandraGalley.tlsCa $cassandraGalley.tlsCaSecretRef -}} true {{- else}} false @@ -117,20 +132,16 @@ false {{- end -}} {{- define "tlsCaGalley" -}} -{{ $cassandraGalley := default dict .Values.cassandraGalley }} -{{- if hasKey .Values.cassandra "tlsCa" -}} -{{- .Values.cassandra.tlsCa }} -{{- else if hasKey $cassandraGalley "tlsCa" -}} +{{ $cassandraGalley := default .Values.cassandra .Values.cassandraGalley }} +{{- if hasKey $cassandraGalley "tlsCa" -}} {{- $cassandraGalley.tlsCa }} {{ else }} {{- end -}} {{- end -}} {{- define "tlsSecretRefGalley" -}} -{{ $cassandraGalley := default dict .Values.cassandraGalley }} -{{- if .Values.cassandra.tlsCaSecretRef -}} -{{ .Values.cassandra.tlsCaSecretRef | toYaml }} -{{- else if $cassandraGalley.tlsCaSecretRef -}} +{{ $cassandraGalley := default .Values.cassandra .Values.cassandraGalley }} +{{- if $cassandraGalley.tlsCaSecretRef -}} {{ $cassandraGalley.tlsCaSecretRef | toYaml }} {{- else }} {{- dict "name" "galley-cassandra-cert" "key" "ca.pem" | toYaml -}} @@ -138,8 +149,8 @@ false {{- end -}} {{- define "useTlsBrig" -}} -{{ $cassandraBrig := default dict .Values.cassandraBrig }} -{{- if or .Values.cassandra.tlsCa $cassandraBrig.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraBrig.tlsCaSecretRef -}} +{{ $cassandraBrig := default .Values.cassandra .Values.cassandraBrig }} +{{- if or $cassandraBrig.tlsCa $cassandraBrig.tlsCaSecretRef -}} true {{- else}} false @@ -147,20 +158,16 @@ false {{- end -}} {{- define "tlsCaBrig" -}} -{{ $cassandraBrig := default dict .Values.cassandraBrig }} -{{- if hasKey .Values.cassandra "tlsCa" -}} -{{- .Values.cassandra.tlsCa }} -{{- else if hasKey $cassandraBrig "tlsCa" -}} +{{ $cassandraBrig := default .Values.cassandra .Values.cassandraBrig }} +{{- if hasKey $cassandraBrig "tlsCa" -}} {{- $cassandraBrig.tlsCa }} {{ else }} {{- end -}} {{- end -}} {{- define "tlsSecretRefBrig" -}} -{{ $cassandraBrig := default dict .Values.cassandraBrig }} -{{- if .Values.cassandra.tlsCaSecretRef -}} -{{ .Values.cassandra.tlsCaSecretRef | toYaml }} -{{- else if $cassandraBrig.tlsCaSecretRef -}} +{{ $cassandraBrig := default .Values.cassandra .Values.cassandraBrig }} +{{- if $cassandraBrig.tlsCaSecretRef -}} {{ $cassandraBrig.tlsCaSecretRef | toYaml }} {{- else }} {{- dict "name" "brig-cassandra-cert" "key" "ca.pem" | toYaml -}} @@ -168,8 +175,8 @@ false {{- end -}} {{- define "useTlsSpar" -}} -{{ $cassandraSpar := default dict .Values.cassandraSpar }} -{{- if or .Values.cassandra.tlsCa $cassandraSpar.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraSpar.tlsCaSecretRef -}} +{{ $cassandraSpar := default .Values.cassandra .Values.cassandraSpar }} +{{- if or $cassandraSpar.tlsCa $cassandraSpar.tlsCaSecretRef -}} true {{- else}} false @@ -177,19 +184,16 @@ false {{- end -}} {{- define "tlsCaSpar" -}} -{{ $cassandraSpar := default dict .Values.cassandraSpar }} -{{- if hasKey .Values.cassandra "tlsCa" -}} -{{- .Values.cassandra.tlsCa }} -{{- else if hasKey $cassandraSpar "tlsCa" -}} +{{ $cassandraSpar := default .Values.cassandra .Values.cassandraSpar }} +{{- if hasKey $cassandraSpar "tlsCa" -}} {{- $cassandraSpar.tlsCa }} {{ else }} {{- end -}} {{- end -}} + {{- define "tlsSecretRefSpar" -}} -{{ $cassandraSpar := default dict .Values.cassandraSpar }} -{{- if .Values.cassandra.tlsCaSecretRef -}} -{{ .Values.cassandra.tlsCaSecretRef | toYaml }} -{{- else if $cassandraSpar.tlsCaSecretRef -}} +{{ $cassandraSpar := default .Values.cassandra .Values.cassandraSpar }} +{{- if $cassandraSpar.tlsCaSecretRef -}} {{ $cassandraSpar.tlsCaSecretRef | toYaml }} {{- else }} {{- dict "name" "spar-cassandra-cert" "key" "ca.pem" | toYaml -}} @@ -197,8 +201,8 @@ false {{- end -}} {{- define "useTlsGundeck" -}} -{{ $cassandraGundeck := default dict .Values.cassandraGundeck }} -{{- if or .Values.cassandra.tlsCa $cassandraGundeck.tlsCa .Values.cassandra.tlsCaSecretRef $cassandraGundeck.tlsCaSecretRef -}} +{{ $cassandraGundeck := default .Values.cassandra .Values.cassandraGundeck }} +{{- if or $cassandraGundeck.tlsCa $cassandraGundeck.tlsCaSecretRef -}} true {{- else}} false @@ -206,20 +210,16 @@ false {{- end -}} {{- define "tlsCaGundeck" -}} -{{ $cassandraGundeck := default dict .Values.cassandraGundeck }} -{{- if hasKey .Values.cassandra "tlsCa" -}} -{{- .Values.cassandra.tlsCa }} -{{- else if hasKey $cassandraGundeck "tlsCa" -}} +{{ $cassandraGundeck := default .Values.cassandra .Values.cassandraGundeck }} +{{- if hasKey $cassandraGundeck "tlsCa" -}} {{- $cassandraGundeck.tlsCa }} {{ else }} {{- end -}} {{- end -}} {{- define "tlsSecretRefGundeck" -}} -{{ $cassandraGundeck := default dict .Values.cassandraGundeck }} -{{- if .Values.cassandra.tlsCaSecretRef -}} -{{ .Values.cassandra.tlsCaSecretRef | toYaml }} -{{- else if $cassandraGundeck.tlsCaSecretRef -}} +{{ $cassandraGundeck := default .Values.cassandra .Values.cassandraGundeck }} +{{- if $cassandraGundeck.tlsCaSecretRef -}} {{ $cassandraGundeck.tlsCaSecretRef | toYaml }} {{- else }} {{- dict "name" "gundeck-cassandra-cert" "key" "ca.pem" | toYaml -}} From cd6ddbcfb04901fab4d4ac515c9bb4e321cc1b03 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 16:05:49 +0100 Subject: [PATCH 73/98] Typo --- charts/cassandra-migrations/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cassandra-migrations/templates/_helpers.tpl b/charts/cassandra-migrations/templates/_helpers.tpl index e4c875e285..0d805051ff 100644 --- a/charts/cassandra-migrations/templates/_helpers.tpl +++ b/charts/cassandra-migrations/templates/_helpers.tpl @@ -118,7 +118,7 @@ configuration if the specific one does not exist: - tlsCa -> String: TLS CA PEM string (if configured) -- tlsSecretRefGalley -> YAML: Dict with keys `name` (name of the +- tlsSecretRef -> YAML: Dict with keys `name` (name of the secret to use) and `key` (name of the entry in the secret) */}} From 43ce7421dec4bba4822d8c155f994fadfbce724b Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Fri, 1 Dec 2023 16:07:17 +0100 Subject: [PATCH 74/98] Useless formatting --- charts/cassandra-migrations/templates/migrate-schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index f8b29e26ad..64ed10d1a6 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -19,7 +19,7 @@ spec: release: {{ .Release.Name }} spec: restartPolicy: OnFailure - # specifying cassandra-migrations as initContainers executes them sequentially, rather than in parallel + # specifying cassandra-migrations as initContainers executes them sequentially, rather than in parallel # to avoid 'Column family ID mismatch' / schema disagreements # see https://stackoverflow.com/questions/29030661/creating-new-table-with-cqlsh-on-existing-keyspace-column-family-id-mismatch#40325651 for details. volumes: From 4b5fe84c531de4d672d07c551d7388fc6a0c033e Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 4 Dec 2023 10:23:00 +0100 Subject: [PATCH 75/98] Fix wrong secret reference --- charts/cassandra-migrations/templates/migrate-schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index 64ed10d1a6..27ba182e6c 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -154,7 +154,7 @@ spec: - "{{ template "cassandraSparReplicationArg" . }}" {{- if eq (include "useTlsSpar" .) "true" }} - --tls-certificate-file - - /certs/spar/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} + - /certs/spar/{{- (include "tlsSecretRefSpar" . | fromYaml).key }} {{- end }} {{- if eq (include "useTlsSpar" .) "true" }} From cddd92506ccb916e1dd1fd3ea813519be4827876 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 4 Dec 2023 10:45:26 +0100 Subject: [PATCH 76/98] Better TLS comment --- charts/brig/values.yaml | 5 +++-- charts/elasticsearch-index/values.yaml | 11 ++++++----- charts/galley/values.yaml | 5 +++-- charts/gundeck/values.yaml | 5 +++-- charts/spar/values.yaml | 5 +++-- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index 5774c277ea..305502e30e 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -20,9 +20,10 @@ config: logNetStrings: false cassandra: host: aws-cassandra -# To enable TLS: +# To enable TLS provide a CA: # tlsCa: -# Or: +# +# Or refer to an existing secret (containing the CA): # tlsCaSecretRef: # name: # key: diff --git a/charts/elasticsearch-index/values.yaml b/charts/elasticsearch-index/values.yaml index c0a937b895..93e8a97ef6 100644 --- a/charts/elasticsearch-index/values.yaml +++ b/charts/elasticsearch-index/values.yaml @@ -8,12 +8,13 @@ cassandra: # host: port: 9042 keyspace: brig -# To enable TLS: +# To enable TLS provide a CA: # tlsCa: -# Or: -# tlsCaSecretRef: -# name: -# key: +# +# Or refer to an existing secret (containing the CA): +# tlsCaSecretRef: +# name: +# key: galley: host: galley port: 8080 diff --git a/charts/galley/values.yaml b/charts/galley/values.yaml index dbbbe7383e..d96f07b6e7 100644 --- a/charts/galley/values.yaml +++ b/charts/galley/values.yaml @@ -22,9 +22,10 @@ config: cassandra: host: aws-cassandra replicaCount: 3 -# To enable TLS: +# To enable TLS provide a CA: # tlsCa: -# Or: +# +# Or refer to an existing secret (containing the CA): # tlsCaSecretRef: # name: # key: diff --git a/charts/gundeck/values.yaml b/charts/gundeck/values.yaml index bdb600f43d..87c338a69d 100644 --- a/charts/gundeck/values.yaml +++ b/charts/gundeck/values.yaml @@ -20,9 +20,10 @@ config: logNetStrings: false cassandra: host: aws-cassandra -# To enable TLS: +# To enable TLS provide a CA: # tlsCa: -# Or: +# +# Or refer to an existing secret (containing the CA): # tlsCaSecretRef: # name: # key: diff --git a/charts/spar/values.yaml b/charts/spar/values.yaml index 048c485ffc..9cb6c2c969 100644 --- a/charts/spar/values.yaml +++ b/charts/spar/values.yaml @@ -17,9 +17,10 @@ service: config: cassandra: host: aws-cassandra -# To enable TLS: +# To enable TLS provide a CA: # tlsCa: -# Or: +# +# Or refer to an existing secret (containing the CA): # tlsCaSecretRef: # name: # key: From dc588691fb37c5309ea6764ee5c22ad2c6187796 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 4 Dec 2023 11:03:43 +0100 Subject: [PATCH 77/98] Better comment --- hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl b/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl index 89c1005fd6..b550775fbd 100644 --- a/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl +++ b/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl @@ -3,7 +3,7 @@ storageClassName: {{ .Values.storageClass }} client_encryption_options: enabled: true optional: false - # The password could be secured better. However, this chart is meant to be - # used as test setup. And, protecting a self-signed certificate isn't very - # useful. + # This password is used to decrypt the internal Java Keystore. No need to be + # careful about it: It's worthless without cluster access and even with it, + # you could impersonate as a test cassandra db... keystorePassword: p4ssw0rd From f818180eaab4041998cfe9b439755b91efa9fdc7 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 4 Dec 2023 11:07:31 +0100 Subject: [PATCH 78/98] Requiring the databases-ephemeral should be fine --- hack/helmfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index e0c01fdb3c..000643a81f 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -163,7 +163,7 @@ releases: - name: cargohold.config.settings.federationDomain value: {{ .Values.federationDomain1 }} needs: - - 'k8ssandra-test-cluster' + - 'databases-ephemeral' - name: 'wire-server' namespace: '{{ .Values.namespace2 }}' @@ -179,4 +179,4 @@ releases: - name: cargohold.config.settings.federationDomain value: {{ .Values.federationDomain2 }} needs: - - 'k8ssandra-test-cluster' + - 'databases-ephemeral' From afb253b2628753fb70b057c7ca18cedb0b045a54 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 4 Dec 2023 11:08:02 +0100 Subject: [PATCH 79/98] Cleanup --- hack/helmfile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 000643a81f..e82a1373a3 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -11,7 +11,6 @@ helmDefaults: timeout: 600 devel: true createNamespace: true -# k8ssandra-cluster-datacenter-1-service environments: default: values: From b942703b00985aad457b0419536bfdaf6fee941b Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 4 Dec 2023 12:06:39 +0100 Subject: [PATCH 80/98] Hi CI From e1778bcaf7b2acecbe8b6bc338506d7bcbd2a4ea Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 15:02:52 +0100 Subject: [PATCH 81/98] Update charts/elasticsearch-index/templates/migrate-data.yaml Formatting Co-authored-by: Akshay Mankar --- charts/elasticsearch-index/templates/migrate-data.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index ac8836d324..b699ca383c 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -52,9 +52,9 @@ spec: - name: elasticsearch-index-migrate-cassandra-client-ca mountPath: "/certs" {{- end }} - {{- if eq (include "useCassandraTLS" .Values) "true" }} + {{- if eq (include "useCassandraTLS" .Values) "true" }} volumes: - name: elasticsearch-index-migrate-cassandra-client-ca secret: secretName: {{ (include "tlsSecretRef" .Values | fromYaml).name }} - {{- end}} + {{- end}} From 801603a33fcd79680d8899dfa0494a72d7e1b851 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 15:28:43 +0100 Subject: [PATCH 82/98] Update charts/integration/templates/integration-integration.yaml Reduce duplication in conditional block. Co-authored-by: Akshay Mankar --- .../templates/integration-integration.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 2818063a45..c08de334ea 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -120,11 +120,15 @@ spec: - | set -euo pipefail # FUTUREWORK: Do all of this in the integration test binary - {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} - {{- else }} - integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} - {{- end }} + integration-dynamic-backends-db-schemas.sh \ + --host {{ .Values.config.cassandra.host }} \ + --port {{ .Values.config.cassandra.port }} \ + --replication-factor {{ .Values.config.cassandra.replicationFactor }}\ + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} + {{- else }} + + {{- end}} integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200 integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} From b32027d4286a3d797d9516ef0a140164a8a06195 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 15:32:46 +0100 Subject: [PATCH 83/98] Rename: tls-certificate-file -> tls-ca-certificate-file It's a bit more descriptive. --- changelog.d/2-features/cassandra-tls | 2 +- .../templates/galley-migrate-data.yaml | 2 +- charts/cassandra-migrations/templates/migrate-schema.yaml | 8 ++++---- .../cassandra-migrations/templates/spar-migrate-data.yaml | 4 ++-- charts/elasticsearch-index/templates/migrate-data.yaml | 2 +- charts/integration/templates/integration-integration.yaml | 2 +- docs/src/developer/reference/config-options.md | 2 +- libs/cassandra-util/src/Cassandra/Schema.hs | 2 +- services/brig/src/Brig/Index/Options.hs | 2 +- services/galley/migrate-data/src/Galley/DataMigration.hs | 2 +- .../spar/migrate-data/src/Spar/DataMigration/Options.hs | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/changelog.d/2-features/cassandra-tls b/changelog.d/2-features/cassandra-tls index 8b0045fd5d..e8baaaf2ed 100644 --- a/changelog.d/2-features/cassandra-tls +++ b/changelog.d/2-features/cassandra-tls @@ -1,6 +1,6 @@ Allow the configuration of TLS-secured connections to Cassandra. TLS is used when a certificate is provided. This is either done with -`--tls-certificate-file` for cli commands or the configuration attribute +`--tls-ca-certificate-file` for cli commands or the configuration attribute `cassandra.tlsCa` for services. In Helm charts, the certificate is provided as literal PEM string; either as attribute `cassandra.tlsCa` (analog to service configuration) or by a reference to a secret (`cassandra.tlsCaSecretRef`.) diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index 0e689817ac..127a6ab0b5 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -43,7 +43,7 @@ spec: - --cassandra-keyspace - galley {{- if eq (include "useTlsGalley" .) "true" }} - - --tls-certificate-file + - --tls-ca-certificate-file - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} {{- if eq (include "useTlsGalley" .) "true" }} diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index 27ba182e6c..e06aa2288a 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -63,7 +63,7 @@ spec: - {{ template "cassandraGundeckReplicationType" . }} - "{{ template "cassandraGundeckReplicationArg" . }}" {{- if eq (include "useTlsGundeck" .) "true" }} - - --tls-certificate-file + - --tls-ca-certificate-file - /certs/gundeck/{{- (include "tlsSecretRefGundeck" . | fromYaml).key }} {{- end }} @@ -93,7 +93,7 @@ spec: - {{ template "cassandraBrigReplicationType" . }} - "{{ template "cassandraBrigReplicationArg" . }}" {{- if eq (include "useTlsBrig" .) "true" }} - - --tls-certificate-file + - --tls-ca-certificate-file - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} {{- end }} @@ -123,7 +123,7 @@ spec: - {{ template "cassandraGalleyReplicationType" . }} - "{{ template "cassandraGalleyReplicationArg" . }}" {{- if eq (include "useTlsGalley" .) "true" }} - - --tls-certificate-file + - --tls-ca-certificate-file - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} {{- end }} @@ -153,7 +153,7 @@ spec: - {{ template "cassandraSparReplicationType" . }} - "{{ template "cassandraSparReplicationArg" . }}" {{- if eq (include "useTlsSpar" .) "true" }} - - --tls-certificate-file + - --tls-ca-certificate-file - /certs/spar/{{- (include "tlsSecretRefSpar" . | fromYaml).key }} {{- end }} diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index 3fb6157609..051946ac2b 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -44,11 +44,11 @@ spec: - --cassandra-keyspace-brig - brig {{- if eq (include "useTlsBrig" .) "true" }} - - --tls-certificate-file-brig + - --tls-ca-certificate-file-brig - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} {{- end }} {{- if eq (include "useTlsSpar" .) "true" }} - - --tls-certificate-file-spar + - --tls-ca-certificate-file-spar - /certs/spar/{{- (include "tlsSecretRefSpar" . | fromYaml).key }} {{- end }} volumeMounts: diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index b699ca383c..3d54e1f51b 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -44,7 +44,7 @@ spec: - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" {{- if eq (include "useCassandraTLS" .Values) "true" }} - - --tls-certificate-file + - --tls-ca-certificate-file - /certs/{{- (include "tlsSecretRef" .Values | fromYaml).key }} {{- end }} {{- if eq (include "useCassandraTLS" .Values) "true" }} diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index c08de334ea..1797312ca8 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -125,7 +125,7 @@ spec: --port {{ .Values.config.cassandra.port }} \ --replication-factor {{ .Values.config.cassandra.replicationFactor }}\ {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - --tls-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} + --tls-ca-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} {{- else }} {{- end}} diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 93400114e3..a88bca4e54 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -816,7 +816,7 @@ client), a **C**ertificate **A**uthority in PEM format needs to be configured. The ways differ regarding the kind of program: - *Services* expect a `cassandra.tlsCa: ` attribute in their config file. -- *CLI commands* (e.g. migrations) accept a `--tls-certificate-file ` parameter. +- *CLI commands* (e.g. migrations) accept a `--tls-ca-certificate-file ` parameter. When a CA PEM file is configured, all Cassandra connections are opened with TLS encryption. I.e. there is no fallback to unencrypted connections. This ensures diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index fc2b2b4a10..72676858a4 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -190,7 +190,7 @@ migrationOptsParser = <> help "Reset the keyspace before running migrations" ) <*> ( (optional . strOption) - ( long "tls-certificate-file" + ( long "tls-ca-certificate-file" <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" ) ) diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index 3b91ae132e..adbad901e3 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -251,7 +251,7 @@ cassandraSettingsParser = ) ) <*> ( (optional . strOption) - ( long "tls-certificate-file" + ( long "tls-ca-certificate-file" <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" <> showDefault ) diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 92676dec1b..825e8a4ee7 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -55,7 +55,7 @@ cassandraSettingsParser = ) ) <*> ( (Opts.optional . Opts.strOption) - ( Opts.long "tls-certificate-file" + ( Opts.long "tls-ca-certificate-file" <> Opts.help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" ) ) diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs index 84d6445f9e..6850ca92ec 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs @@ -72,7 +72,7 @@ cassandraSettingsParser ks = ) ) <*> ( (optional . strOption) - ( long ("tls-certificate-file-" ++ ks) + ( long ("tls-ca-certificate-file-" ++ ks) <> help ("Location of a PEM encoded list of CA certificates to be used when verifying" ++ ks ++ "'s Cassandra server's certificate") <> showDefault ) From ac26d937ecaba60775be6cdfdd15a72e4395a610 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 15:43:42 +0100 Subject: [PATCH 84/98] Simplify certFilePath selection expression Co-authored-by: Akshay Mankar --- integration/test/Testlib/Env.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index f8bd436aa2..1c41e9c0e4 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -61,7 +61,7 @@ mkGlobalEnv cfgFile = do ( \certFilePath -> if isAbsolute certFilePath then pure $ Just certFilePath - else maybe (pure Nothing) (\projectRoot -> (Just <$> (makeAbsolute) (combine projectRoot certFilePath))) devEnvProjectRoot + else for devEnvProjectRoot $ \projectRoot -> makeAbsolute $ combine projectRoot certFilePath ) intConfig.cassandra.cassTlsCa From d3e04463e542e36e96bb7d2d4c71b4b2673131ac Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 15:45:10 +0100 Subject: [PATCH 85/98] Default cannot be shown for option --tls-ca-certificate-file Because there's none. --- services/brig/src/Brig/Index/Options.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index adbad901e3..f899d75814 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -253,7 +253,6 @@ cassandraSettingsParser = <*> ( (optional . strOption) ( long "tls-ca-certificate-file" <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" - <> showDefault ) ) From eba57b555a3a80fec45ad771fb1919e5090346b9 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 15:51:13 +0100 Subject: [PATCH 86/98] Update docs/src/developer/reference/config-options.md Co-authored-by: Akshay Mankar --- docs/src/developer/reference/config-options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index a88bca4e54..1f32ec6de9 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -819,7 +819,7 @@ The ways differ regarding the kind of program: - *CLI commands* (e.g. migrations) accept a `--tls-ca-certificate-file ` parameter. When a CA PEM file is configured, all Cassandra connections are opened with TLS -encryption. I.e. there is no fallback to unencrypted connections. This ensures +encryption i.e. there is no fallback to unencrypted connections. This ensures that connections that are expected to be secure, would not silently and unnoticed be insecure. From 124cd3c872c319fd9155f99e90da31e92e822de4 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 17:45:53 +0100 Subject: [PATCH 87/98] Fix missing import for `for` --- integration/test/Testlib/Env.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index 1c41e9c0e4..f6416590cd 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -12,6 +12,7 @@ import Data.Map qualified as Map import Data.Maybe (fromMaybe) import Data.Set (Set) import Data.Set qualified as Set +import Data.Traversable (for) import Data.Yaml qualified as Yaml import Database.CQL.IO qualified as Cassandra import Network.HTTP.Client qualified as HTTP From 3a5313ac54ee4252d95a053f226d40fb21a217e1 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 17:51:11 +0100 Subject: [PATCH 88/98] Move Cassandra Options --- libs/cassandra-util/cassandra-util.cabal | 2 + libs/cassandra-util/src/Cassandra/Helpers.hs | 25 +++++++++++ libs/cassandra-util/src/Cassandra/Options.hs | 38 +++++++++++++++++ libs/types-common/src/Util/Options.hs | 44 +++++--------------- libs/types-common/src/Util/Options/Common.hs | 29 +++---------- 5 files changed, 82 insertions(+), 56 deletions(-) create mode 100644 libs/cassandra-util/src/Cassandra/Helpers.hs create mode 100644 libs/cassandra-util/src/Cassandra/Options.hs diff --git a/libs/cassandra-util/cassandra-util.cabal b/libs/cassandra-util/cassandra-util.cabal index 4df1169fd6..af2e009420 100644 --- a/libs/cassandra-util/cassandra-util.cabal +++ b/libs/cassandra-util/cassandra-util.cabal @@ -15,7 +15,9 @@ library Cassandra Cassandra.CQL Cassandra.Exec + Cassandra.Helpers Cassandra.MigrateSchema + Cassandra.Options Cassandra.Schema Cassandra.Settings Cassandra.Util diff --git a/libs/cassandra-util/src/Cassandra/Helpers.hs b/libs/cassandra-util/src/Cassandra/Helpers.hs new file mode 100644 index 0000000000..8a260d530b --- /dev/null +++ b/libs/cassandra-util/src/Cassandra/Helpers.hs @@ -0,0 +1,25 @@ +module Cassandra.Helpers where + +import Data.Aeson.TH +import Imports + +-- | Convenient helper to convert record field names to use as YAML fields. +-- NOTE: We typically use this for options in the configuration files! +-- If you are looking into converting record field name to JSON to be used +-- over the API, look for toJSONFieldName in the Data.Json.Util module. +-- It converts field names into snake_case +-- +-- Example: +-- newtype TeamName = TeamName { teamName :: Text } +-- deriveJSON toJSONFieldName ''teamName +-- +-- would generate {To/From}JSON instances where +-- the field name is "teamName" +toOptionFieldName :: Options +toOptionFieldName = defaultOptions {fieldLabelModifier = lowerFirst . dropPrefix} + where + lowerFirst :: String -> String + lowerFirst (x : xs) = toLower x : xs + lowerFirst [] = "" + dropPrefix :: String -> String + dropPrefix = dropWhile ('_' ==) diff --git a/libs/cassandra-util/src/Cassandra/Options.hs b/libs/cassandra-util/src/Cassandra/Options.hs new file mode 100644 index 0000000000..f1f62056ee --- /dev/null +++ b/libs/cassandra-util/src/Cassandra/Options.hs @@ -0,0 +1,38 @@ +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} + +module Cassandra.Options where + +import Cassandra.Helpers +import Control.Lens +import Data.Aeson.TH +import Imports + +data Endpoint = Endpoint + { _host :: !Text, + _port :: !Word16 + } + deriving (Show, Generic) + +deriveFromJSON toOptionFieldName ''Endpoint + +makeLenses ''Endpoint + +data CassandraOpts = CassandraOpts + { _endpoint :: !Endpoint, + _keyspace :: !Text, + -- | If this option is unset, use all available nodes. + -- If this option is set, use only cassandra nodes in the given datacentre + -- + -- This option is most likely only necessary during a cassandra DC migration + -- FUTUREWORK: remove this option again, or support a datacentre migration feature + _filterNodesByDatacentre :: !(Maybe Text), + _tlsCa :: Maybe FilePath + } + deriving (Show, Generic) + +deriveFromJSON toOptionFieldName ''CassandraOpts + +makeLenses ''CassandraOpts diff --git a/libs/types-common/src/Util/Options.hs b/libs/types-common/src/Util/Options.hs index 40ca142908..74437b78a2 100644 --- a/libs/types-common/src/Util/Options.hs +++ b/libs/types-common/src/Util/Options.hs @@ -20,10 +20,15 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module Util.Options where - +module Util.Options + ( module Util.Options, + -- TODO: Switch denpendees to the original module? + module Cassandra.Options, + ) +where + +import Cassandra.Options import Control.Lens -import Data.Aeson.TH import Data.ByteString.Char8 qualified as BS import Data.ByteString.Conversion import Data.Text.Encoding (encodeUtf8) @@ -48,17 +53,17 @@ instance FromByteString AWSEndpoint where "https" -> pure True "http" -> pure False x -> fail ("Unsupported scheme: " ++ show x) - host <- case url ^. authorityL <&> view (authorityHostL . hostBSL) of + awsHost <- case url ^. authorityL <&> view (authorityHostL . hostBSL) of Just h -> pure h Nothing -> fail ("No host in: " ++ show url) - port <- case urlPort url of + awsPort <- case urlPort url of Just p -> pure p Nothing -> pure $ if secure then 443 else 80 - pure $ AWSEndpoint host secure port + pure $ AWSEndpoint awsHost secure awsPort instance FromJSON AWSEndpoint where parseJSON = @@ -73,33 +78,6 @@ urlPort u = do makeLenses ''AWSEndpoint -data Endpoint = Endpoint - { _host :: !Text, - _port :: !Word16 - } - deriving (Show, Generic) - -deriveFromJSON toOptionFieldName ''Endpoint - -makeLenses ''Endpoint - -data CassandraOpts = CassandraOpts - { _endpoint :: !Endpoint, - _keyspace :: !Text, - -- | If this option is unset, use all available nodes. - -- If this option is set, use only cassandra nodes in the given datacentre - -- - -- This option is most likely only necessary during a cassandra DC migration - -- FUTUREWORK: remove this option again, or support a datacentre migration feature - _filterNodesByDatacentre :: !(Maybe Text), - _tlsCa :: Maybe FilePath - } - deriving (Show, Generic) - -deriveFromJSON toOptionFieldName ''CassandraOpts - -makeLenses ''CassandraOpts - newtype FilePathSecrets = FilePathSecrets FilePath deriving (Eq, Show, FromJSON) diff --git a/libs/types-common/src/Util/Options/Common.hs b/libs/types-common/src/Util/Options/Common.hs index c052a53c33..14b997bee7 100644 --- a/libs/types-common/src/Util/Options/Common.hs +++ b/libs/types-common/src/Util/Options/Common.hs @@ -15,36 +15,19 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module Util.Options.Common where +module Util.Options.Common + ( module Cassandra.Helpers, + module Util.Options.Common, + ) +where -import Data.Aeson.TH +import Cassandra.Helpers (toOptionFieldName) import Data.ByteString.Char8 qualified as C import Data.Text qualified as T import Imports hiding (reader) import Options.Applicative import System.Posix.Env qualified as Posix --- | Convenient helper to convert record field names to use as YAML fields. --- NOTE: We typically use this for options in the configuration files! --- If you are looking into converting record field name to JSON to be used --- over the API, look for toJSONFieldName in the Data.Json.Util module. --- It converts field names into snake_case --- --- Example: --- newtype TeamName = TeamName { teamName :: Text } --- deriveJSON toJSONFieldName ''teamName --- --- would generate {To/From}JSON instances where --- the field name is "teamName" -toOptionFieldName :: Options -toOptionFieldName = defaultOptions {fieldLabelModifier = lowerFirst . dropPrefix} - where - lowerFirst :: String -> String - lowerFirst (x : xs) = toLower x : xs - lowerFirst [] = "" - dropPrefix :: String -> String - dropPrefix = dropWhile ('_' ==) - optOrEnv :: (a -> b) -> Maybe a -> (String -> b) -> String -> IO b optOrEnv getter conf reader var = case conf of Nothing -> reader <$> getEnv var From dde9690970dff7f1766c3989e64fb49587ffeade Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 19:18:36 +0100 Subject: [PATCH 89/98] Use CassandraOpts to hand over connection parameters --- libs/cassandra-util/src/Cassandra/Util.hs | 36 ++++++++----------- services/brig/src/Brig/App.hs | 6 +--- services/brig/src/Brig/Index/Eval.hs | 9 +---- services/brig/src/Brig/Index/Migrations.hs | 9 +---- services/brig/src/Brig/Index/Options.hs | 13 ++++++- services/brig/test/integration/Run.hs | 6 +--- .../migrate-data/src/Galley/DataMigration.hs | 18 ++++++---- services/galley/src/Galley/App.hs | 7 +--- services/galley/test/integration/Run.hs | 6 +--- services/gundeck/src/Gundeck/Env.hs | 7 +--- services/gundeck/test/integration/Main.hs | 6 +--- .../src/Spar/DataMigration/Run.hs | 8 +---- .../src/Spar/DataMigration/Types.hs | 13 ++++++- services/spar/src/Spar/Run.hs | 6 +--- 14 files changed, 60 insertions(+), 90 deletions(-) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index c0dc51120b..f8b793f77d 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -25,8 +25,10 @@ module Cassandra.Util where import Cassandra.CQL +import Cassandra.Options import Cassandra.Schema import Cassandra.Settings (dcFilterPolicyIfConfigured, initialContactsDisco, initialContactsPlain, mkLogger) +import Control.Lens import Data.Aeson import Data.Fixed import Data.List.NonEmpty qualified as NE @@ -40,52 +42,44 @@ import Imports hiding (init) import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log -defInitCassandra :: Text -> Text -> Word16 -> Maybe FilePath -> Log.Logger -> IO ClientState -defInitCassandra ks h p mbTlsCaPath logger = do +defInitCassandra :: CassandraOpts -> Log.Logger -> IO ClientState +defInitCassandra opts logger = do let basicCasSettings = setLogger (CT.mkLogger logger) - . setPortNumber (fromIntegral p) - . setContacts (unpack h) [] - . setKeyspace (Keyspace ks) + . setPortNumber (fromIntegral (opts ^. endpoint . port)) + . setContacts (unpack (opts ^. endpoint . host)) [] + . setKeyspace (Keyspace (opts ^. keyspace)) . setProtocolVersion V4 $ defSettings - initCassandra basicCasSettings mbTlsCaPath logger + initCassandra basicCasSettings (opts ^. tlsCa) logger -- | Create Cassandra `ClientState` ("connection") for a service --- --- Unfortunately, we have to deal with many function arguments here, because --- @CassandraOpts@ is defined in @types-common@ which depends on --- @cassandra-util@ (this package.) initCassandraForService :: - Text -> - Word16 -> + CassandraOpts -> String -> - Text -> - Maybe FilePath -> - Maybe Text -> Maybe Text -> Maybe Int32 -> Log.Logger -> IO ClientState -initCassandraForService host port serviceName keyspace mbTlsCaPath filterNodesByDatacentre discoUrl mbSchemaVersion logger = do +initCassandraForService opts serviceName discoUrl mbSchemaVersion logger = do c <- maybe - (initialContactsPlain host) + (initialContactsPlain (opts ^. endpoint . host)) (initialContactsDisco ("cassandra_" ++ serviceName) . unpack) discoUrl let basicCasSettings = setLogger (mkLogger (Log.clone (Just (pack ("cassandra." ++ serviceName))) logger)) . setContacts (NE.head c) (NE.tail c) - . setPortNumber (fromIntegral port) - . setKeyspace (Keyspace keyspace) + . setPortNumber (fromIntegral (opts ^. endpoint . port)) + . setKeyspace (Keyspace (opts ^. keyspace)) . setMaxConnections 4 . setPoolStripes 4 . setSendTimeout 3 . setResponseTimeout 10 . setProtocolVersion V4 - . setPolicy (dcFilterPolicyIfConfigured logger filterNodesByDatacentre) + . setPolicy (dcFilterPolicyIfConfigured logger (opts ^. filterNodesByDatacentre)) $ defSettings - p <- initCassandra basicCasSettings mbTlsCaPath logger + p <- initCassandra basicCasSettings (opts ^. tlsCa) logger maybe (pure ()) (\v -> runClient p $ (versionCheck v)) mbSchemaVersion pure p diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index b33db9d42f..616bc4a79d 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -423,12 +423,8 @@ initExtGetManager = do initCassandra :: Opts -> Logger -> IO Cas.ClientState initCassandra o g = initCassandraForService - (Opt.cassandra o ^. endpoint . host) - (Opt.cassandra o ^. endpoint . port) + (Opt.cassandra o) "brig" - (Opt.cassandra o ^. keyspace) - (Opt.cassandra o ^. tlsCa) - (Opt.cassandra o ^. filterNodesByDatacentre) (Opt.discoUrl o) (Just schemaVersion) g diff --git a/services/brig/src/Brig/Index/Eval.hs b/services/brig/src/Brig/Index/Eval.hs index 5a6650ec85..ed412d8d0d 100644 --- a/services/brig/src/Brig/Index/Eval.hs +++ b/services/brig/src/Brig/Index/Eval.hs @@ -33,7 +33,6 @@ import Control.Retry import Data.Aeson (FromJSON) import Data.Aeson qualified as Aeson import Data.Metrics qualified as Metrics -import Data.Text qualified as Text import Database.Bloodhound qualified as ES import Imports import Network.HTTP.Client as HTTP @@ -102,13 +101,7 @@ runCommand l = \case <*> pure mgr initES esURI mgr = ES.mkBHEnv (toESServer esURI) mgr - initDb cas = - defInitCassandra - (C.unKeyspace (cas ^. cKeyspace)) - (Text.pack (cas ^. cHost)) - (cas ^. cPort) - (cas ^. cTlsCa) - l + initDb cas = defInitCassandra (toCassandraOpts cas) l waitForTaskToComplete :: forall a m. (ES.MonadBH m, MonadThrow m, FromJSON a) => Int -> ES.TaskNodeId -> m () waitForTaskToComplete timeoutSeconds taskNodeId = do diff --git a/services/brig/src/Brig/Index/Migrations.hs b/services/brig/src/Brig/Index/Migrations.hs index c00550b4e5..da7e78cc1a 100644 --- a/services/brig/src/Brig/Index/Migrations.hs +++ b/services/brig/src/Brig/Index/Migrations.hs @@ -23,7 +23,6 @@ where import Brig.Index.Migrations.Types import Brig.Index.Options qualified as Opts import Brig.User.Search.Index qualified as Search -import Cassandra qualified as C import Cassandra.Util (defInitCassandra) import Control.Lens (view, (^.)) import Control.Monad.Catch (MonadThrow, catchAll, finally, throwM) @@ -86,13 +85,7 @@ mkEnv l es cas galleyEndpoint = do <*> pure mgr <*> pure galleyEndpoint where - initCassandra = - defInitCassandra - (C.unKeyspace (cas ^. Opts.cKeyspace)) - (Text.pack (cas ^. Opts.cHost)) - (cas ^. Opts.cPort) - (cas ^. Opts.cTlsCa) - l + initCassandra = defInitCassandra (Opts.toCassandraOpts cas) l initLogger = pure l diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index f899d75814..89da5997cb 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -29,6 +29,7 @@ module Brig.Index.Options esIndexRefreshInterval, esDeleteTemplate, CassandraSettings, + toCassandraOpts, cHost, cPort, cTlsCa, @@ -50,6 +51,7 @@ import Brig.Index.Types (CreateIndexSettings (..)) import Cassandra qualified as C import Control.Lens import Data.ByteString.Lens +import Data.Text qualified as Text import Data.Text.Strict.Lens import Data.Time.Clock (NominalDiffTime) import Database.Bloodhound qualified as ES @@ -57,7 +59,7 @@ import Imports import Options.Applicative import URI.ByteString import URI.ByteString.QQ -import Util.Options (Endpoint (..)) +import Util.Options (CassandraOpts (..), Endpoint (..)) data Command = Create ElasticSettings Endpoint @@ -102,6 +104,15 @@ makeLenses ''CassandraSettings makeLenses ''ReindexFromAnotherIndexSettings +toCassandraOpts :: CassandraSettings -> CassandraOpts +toCassandraOpts cas = + CassandraOpts + { _endpoint = Endpoint (Text.pack (cas ^. cHost)) (cas ^. cPort), + _keyspace = C.unKeyspace (cas ^. cKeyspace), + _filterNodesByDatacentre = Nothing, + _tlsCa = cas ^. cTlsCa + } + mkCreateIndexSettings :: ElasticSettings -> CreateIndexSettings mkCreateIndexSettings es = CreateIndexSettings diff --git a/services/brig/test/integration/Run.hs b/services/brig/test/integration/Run.hs index ad45070078..babbbdce48 100644 --- a/services/brig/test/integration/Run.hs +++ b/services/brig/test/integration/Run.hs @@ -133,13 +133,9 @@ runTests iConf brigOpts otherArgs = do Opts.TurnSourceFiles files -> files Opts.TurnSourceDNS _ -> error "The integration tests can only be run when TurnServers are sourced from files" localDomain = brigOpts ^. Opts.optionSettings . Opts.federationDomain - casHost = (\v -> Opts.cassandra v ^. endpoint . host) brigOpts - casPort = (\v -> Opts.cassandra v ^. endpoint . port) brigOpts - casKey = (\v -> Opts.cassandra v ^. keyspace) brigOpts - casTlsCa = (\v -> Opts.cassandra v ^. tlsCa) brigOpts awsOpts = Opts.aws brigOpts lg <- Logger.new Logger.defSettings -- TODO: use mkLogger'? - db <- defInitCassandra casKey casHost casPort casTlsCa lg + db <- defInitCassandra (brigOpts.cassandra) lg mg <- newManager tlsManagerSettings let fedBrigClient = FedClient @'Brig mg (brig iConf) emailAWSOpts <- parseEmailAWSOpts diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 825e8a4ee7..ac79bcc0fc 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -18,6 +18,7 @@ module Galley.DataMigration (cassandraSettingsParser, migrate) where import Cassandra qualified as C +import Cassandra.Options import Cassandra.Util (defInitCassandra) import Control.Monad.Catch (finally) import Data.Text qualified as Text @@ -36,6 +37,15 @@ data CassandraSettings = CassandraSettings cTlsCa :: Maybe FilePath } +toCassandraOpts :: CassandraSettings -> CassandraOpts +toCassandraOpts cas = + CassandraOpts + { _endpoint = Endpoint (Text.pack (cas.cHost)) (cas.cPort), + _keyspace = C.unKeyspace (cas.cKeyspace), + _filterNodesByDatacentre = Nothing, + _tlsCa = cas.cTlsCa + } + cassandraSettingsParser :: Parser CassandraSettings cassandraSettingsParser = CassandraSettings @@ -75,13 +85,7 @@ mkEnv l cas = <$> initCassandra <*> initLogger where - initCassandra = - defInitCassandra - ((C.unKeyspace . cKeyspace) cas) - ((Text.pack . cHost) cas) - (cPort cas) - (cTlsCa cas) - l + initCassandra = defInitCassandra (toCassandraOpts cas) l initLogger = pure l -- | Runs only the migrations which need to run diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs index ab81a87294..14873001de 100644 --- a/services/galley/src/Galley/App.hs +++ b/services/galley/src/Galley/App.hs @@ -103,7 +103,6 @@ import System.Logger qualified as Log import System.Logger.Class (Logger) import System.Logger.Extended qualified as Logger import UnliftIO.Exception qualified as UnliftIO -import Util.Options import Wire.API.Conversation.Protocol import Wire.API.Error import Wire.API.Federation.Error @@ -172,12 +171,8 @@ createEnv m o l = do initCassandra :: Opts -> Logger -> IO ClientState initCassandra o l = initCassandraForService - (o ^. cassandra . endpoint . host) - (o ^. cassandra . endpoint . port) + (o ^. cassandra) "galley" - (o ^. cassandra . keyspace) - (o ^. cassandra . tlsCa) - (o ^. cassandra . filterNodesByDatacentre) (o ^. discoUrl) Nothing l diff --git a/services/galley/test/integration/Run.hs b/services/galley/test/integration/Run.hs index 4b2a3e6130..149c935262 100644 --- a/services/galley/test/integration/Run.hs +++ b/services/galley/test/integration/Run.hs @@ -124,12 +124,8 @@ main = withOpenSSL $ runTests go convMaxSize <- optOrEnv maxSize gConf read "CONV_MAX_SIZE" awsEnv <- initAwsEnv e q -- Initialize cassandra - let ch = fromJust gConf ^. cassandra . endpoint . host - let cp = fromJust gConf ^. cassandra . endpoint . port - let ck = fromJust gConf ^. cassandra . keyspace - let cTlsCa = fromJust gConf ^. cassandra . tlsCa lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp cTlsCa lg + db <- defInitCassandra (fromJust gConf ^. cassandra) lg teamEventWatcher <- sequence $ SQS.watchSQSQueue <$> ((^. Aws.awsEnv) <$> awsEnv) <*> q pure $ TestSetup (fromJust gConf) (fromJust iConf) m g b c awsEnv convMaxSize db (FedClient m galleyEndpoint) teamEventWatcher queueName' = fmap (view queueName) . view journal diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index 6d3f7bf6e4..fdc67f2f22 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -43,7 +43,6 @@ import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.TLS (tlsManagerSettings) import System.Logger qualified as Log import System.Logger.Extended qualified as Logger -import Util.Options data Env = Env { _reqId :: !RequestId, @@ -85,12 +84,8 @@ createEnv m o = do p <- initCassandraForService - (o ^. cassandra . endpoint . host) - (o ^. cassandra . endpoint . port) + (o ^. cassandra) "gundeck" - (o ^. cassandra . keyspace) - (o ^. cassandra . tlsCa) - (o ^. cassandra . filterNodesByDatacentre) (o ^. discoUrl) Nothing l diff --git a/services/gundeck/test/integration/Main.hs b/services/gundeck/test/integration/Main.hs index d25e2b6b01..767f28a4ae 100644 --- a/services/gundeck/test/integration/Main.hs +++ b/services/gundeck/test/integration/Main.hs @@ -112,12 +112,8 @@ main = withOpenSSL $ runTests go c = CannonR . mkRequest $ cannon iConf c2 = CannonR . mkRequest $ cannon2 iConf b = BrigR $ mkRequest iConf.brig - ch = gConf ^. cassandra . endpoint . host - cp = gConf ^. cassandra . endpoint . port - ck = gConf ^. cassandra . keyspace - cTlsCa = gConf ^. cassandra . tlsCa lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp cTlsCa lg + db <- defInitCassandra (gConf ^. cassandra) lg pure $ TestSetup m g c c2 b db lg gConf (redis2 iConf) releaseOpts _ = pure () mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index 3721ba6d20..c41bd2d2cc 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -67,13 +67,7 @@ mkEnv settings = do $ Log.defSettings initCassandra :: CassandraSettings -> Log.Logger -> IO ClientState - initCassandra cas l = - defInitCassandra - (C.unKeyspace (cas ^. cKeyspace)) - (Text.pack (cas ^. cHosts)) - (cas ^. cPort) - (cas ^. tlsCa) - l + initCassandra cas l = defInitCassandra (toCassandraOpts cas) l cleanup :: (MonadIO m) => Env -> m () cleanup env = do diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs index 6331b10159..64d7a13c0e 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs @@ -21,7 +21,9 @@ module Spar.DataMigration.Types where import qualified Cassandra as C +import Cassandra.Options import Control.Lens +import qualified Data.Text as Text import Imports import Numeric.Natural (Natural) import qualified System.Logger as Logger @@ -63,10 +65,19 @@ data CassandraSettings = CassandraSettings { _cHosts :: !String, _cPort :: !Word16, _cKeyspace :: !C.Keyspace, - _tlsCa :: Maybe FilePath + _cTlsCa :: Maybe FilePath } deriving (Show) makeLenses ''MigratorSettings makeLenses ''CassandraSettings + +toCassandraOpts :: CassandraSettings -> CassandraOpts +toCassandraOpts cas = + CassandraOpts + { _endpoint = Endpoint (Text.pack (cas ^. cHosts)) (cas ^. cPort), + _keyspace = C.unKeyspace (cas ^. cKeyspace), + _filterNodesByDatacentre = Nothing, + _tlsCa = cas ^. cTlsCa + } diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index 29d49b72a7..2a9a427f64 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -63,12 +63,8 @@ import Wire.Sem.Logger.TinyLog initCassandra :: Opts -> Logger -> IO ClientState initCassandra opts lgr = initCassandraForService - (Opt.cassandra opts ^. endpoint . host) - (Opt.cassandra opts ^. endpoint . port) + (Opt.cassandra opts) "spar" - (Opt.cassandra opts ^. keyspace) - (Opt.cassandra opts ^. tlsCa) - (Opt.cassandra opts ^. filterNodesByDatacentre) (Opt.discoUrl opts) (Just Data.schemaVersion) lgr From a2ac6e0777ee6772f2ca4794d975312b9fd771f3 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Mon, 18 Dec 2023 19:22:32 +0100 Subject: [PATCH 90/98] More descriptive variable name --- libs/cassandra-util/src/Cassandra/MigrateSchema.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/cassandra-util/src/Cassandra/MigrateSchema.hs b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs index 35f00a4da1..33b0825834 100644 --- a/libs/cassandra-util/src/Cassandra/MigrateSchema.hs +++ b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs @@ -42,8 +42,8 @@ migrateSchema l o ms = do . setResponseTimeout 50 . setProtocolVersion V4 $ defSettings - p <- initCassandra cqlSettings o.migTlsCa l - runClient p $ do + cas <- initCassandra cqlSettings o.migTlsCa l + runClient cas $ do let keyspace = Keyspace . migKeyspace $ o when (migReset o) $ do info "Dropping keyspace." From b0f94b9361eac014870ac3fe6527eefb5437c76a Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 19 Dec 2023 08:47:52 +0100 Subject: [PATCH 91/98] Avoid type annotations by using monomorphic print function --- integration/test/Testlib/Env.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index f6416590cd..bd3d7058b3 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -123,7 +123,7 @@ mkGlobalEnv cfgFile = do } pure $ Just sslContext createSSLContext Nothing = do - print ("TLS: No TLS CA path provided. Connecting to Cassandra without TLS." :: String) + putStrLn "TLS: No TLS CA path provided. Connecting to Cassandra without TLS." pure Nothing mkEnv :: GlobalEnv -> Codensity IO Env From a4e898b00b68f8837bf73bd441579579f637f2e2 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Tue, 19 Dec 2023 16:49:57 +0100 Subject: [PATCH 92/98] Remove superfluous log line --- integration/test/Testlib/Env.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index bd3d7058b3..63b99d9633 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -122,9 +122,7 @@ mkGlobalEnv cfgFile = do vpCallback = Nothing } pure $ Just sslContext - createSSLContext Nothing = do - putStrLn "TLS: No TLS CA path provided. Connecting to Cassandra without TLS." - pure Nothing + createSSLContext Nothing = pure Nothing mkEnv :: GlobalEnv -> Codensity IO Env mkEnv ge = do From cc10ae301e34e3b94c6eae5b26b6bee17646c13a Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 20 Dec 2023 08:50:56 +0100 Subject: [PATCH 93/98] Cleanup --replication-factor expression --- charts/integration/templates/integration-integration.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 1797312ca8..c5bac71d07 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -123,12 +123,10 @@ spec: integration-dynamic-backends-db-schemas.sh \ --host {{ .Values.config.cassandra.host }} \ --port {{ .Values.config.cassandra.port }} \ - --replication-factor {{ .Values.config.cassandra.replicationFactor }}\ + --replication-factor {{ .Values.config.cassandra.replicationFactor }} \ {{- if eq (include "useCassandraTLS" .Values.config) "true" }} --tls-ca-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} - {{- else }} - - {{- end}} + {{- end }} integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200 integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} From 10a7d4abf6bc5ef47104a15335022f27b41c36e3 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 20 Dec 2023 09:40:04 +0100 Subject: [PATCH 94/98] Allow newline to prevent negative wrapping --- charts/integration/templates/integration-integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index c5bac71d07..38458fc685 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -126,7 +126,7 @@ spec: --replication-factor {{ .Values.config.cassandra.replicationFactor }} \ {{- if eq (include "useCassandraTLS" .Values.config) "true" }} --tls-ca-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} - {{- end }} + {{ end }} integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200 integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} From a436d1ec44f6d89589272092092942a2848d0db2 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 20 Dec 2023 10:04:09 +0100 Subject: [PATCH 95/98] New line to guard against line concatenating --- charts/integration/templates/integration-integration.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 38458fc685..b3abde74cc 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -127,6 +127,7 @@ spec: {{- if eq (include "useCassandraTLS" .Values.config) "true" }} --tls-ca-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} {{ end }} + integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200 integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} From 765ebc1d017971f2cb54e215cc10c23faa204eba Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Wed, 20 Dec 2023 17:57:56 +0100 Subject: [PATCH 96/98] Use trust-manager to sync TLS CA secret --- .../templates/tls-certificate-bundle.yaml | 24 +++++++++++++++++++ charts/k8ssandra-test-cluster/values.yaml | 9 +++++++ .../src/developer/reference/config-options.md | 11 +++++---- 3 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml diff --git a/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml b/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml new file mode 100644 index 0000000000..d58d8a9059 --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.client_encryption_options.enabled .Values.syncCACertToSecret }} +# Let trust-manager sync the CA PEM (and only that!) into secrets named +# `k8ssandra-tls-ca-certificate-` in all configured namespaces or only +# one if syncCACertNamespace is defined. This way we can hide the private key +# from public. +apiVersion: trust.cert-manager.io/v1alpha1 +kind: Bundle +metadata: + name: k8ssandra-tls-ca-certificate-{{- .Release.Namespace }} + namespace: {{ .Release.Namespace }} +spec: + sources: + - secret: + name: "cassandra-jks-keystore" + key: "ca.crt" + target: + secret: + key: "ca.crt" + {{- if hasKey .Values "syncCACertNamespace" }} + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.syncCACertNamespace }} + {{- end }} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/values.yaml b/charts/k8ssandra-test-cluster/values.yaml index f936c2c572..bb286dce51 100644 --- a/charts/k8ssandra-test-cluster/values.yaml +++ b/charts/k8ssandra-test-cluster/values.yaml @@ -21,3 +21,12 @@ client_encryption_options: # used as test setup. And, protecting a self-signed certificate isn't very # useful. keystorePassword: password + +# Guard the private key by syncing only the CA certificate to +# `k8ssandra-test-cluster-tls-ca-certificate` secrets. Requires `trust-manager` +# Helm chart to be installed (including CDRs.) +syncCACertToSecret: false + +# Limit syncing to this namespace. Otherwise, the secret is synced to all +# namespaces. +# syncCACertNamespace: diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 1f32ec6de9..00b2ce6d56 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -833,8 +833,11 @@ The CA may be self-signed. It is used to validate the certificate of the Cassandra server. How to configure Cassandra to accept TLS-encrypted connections in general is -beyond the scope of this document. The `k8ssandra-test-cluster` provides an -example how to do this for the Kubernetes solution *K8ssandra* and a `Secret` -generated by `cert-manager`. The corresponding Cassandra options are described -in Cassandra's documentation: +beyond the scope of this document. The `k8ssandra-test-cluster` Helm chart +provides an example how to do this for the Kubernetes solution *K8ssandra*. In +the example `cert-manager` generates a `Certificate` including Java KeyStores, +then `trust-manager` creates synchronized `Secret`s to make only the CA PEM +accessible to services (and not the private key.) + +The corresponding Cassandra options are described in Cassandra's documentation: [client_encryption_options](https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options) From a826252a09065c3cc6b39bba1d2fa520a725de91 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 21 Dec 2023 10:44:09 +0100 Subject: [PATCH 97/98] Typo Co-authored-by: Akshay Mankar --- charts/k8ssandra-test-cluster/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/k8ssandra-test-cluster/values.yaml b/charts/k8ssandra-test-cluster/values.yaml index bb286dce51..a34ca0da5f 100644 --- a/charts/k8ssandra-test-cluster/values.yaml +++ b/charts/k8ssandra-test-cluster/values.yaml @@ -24,7 +24,7 @@ client_encryption_options: # Guard the private key by syncing only the CA certificate to # `k8ssandra-test-cluster-tls-ca-certificate` secrets. Requires `trust-manager` -# Helm chart to be installed (including CDRs.) +# Helm chart to be installed (including CRDs.) syncCACertToSecret: false # Limit syncing to this namespace. Otherwise, the secret is synced to all From 54489cd3ed22a8e1871f3324377e4147424af9e5 Mon Sep 17 00:00:00 2001 From: Sven Tennie Date: Thu, 21 Dec 2023 10:49:24 +0100 Subject: [PATCH 98/98] Simplify name of trust-manager sync'ed secret Co-authored-by: Akshay Mankar --- .../templates/tls-certificate-bundle.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml b/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml index d58d8a9059..4b06b31110 100644 --- a/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml +++ b/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml @@ -6,7 +6,7 @@ apiVersion: trust.cert-manager.io/v1alpha1 kind: Bundle metadata: - name: k8ssandra-tls-ca-certificate-{{- .Release.Namespace }} + name: k8ssandra-tls-ca-certificate namespace: {{ .Release.Namespace }} spec: sources: