From f82f0a0f5cf5e74d0b4a404801371540841b1284 Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Wed, 8 May 2024 13:23:52 +0200 Subject: [PATCH 1/9] gundeck: Introduce data migrations First one will delete any VOIP push tokens --- services/gundeck/default.nix | 4 + services/gundeck/gundeck.cabal | 91 +++++++++++-- services/gundeck/migrate-data/main.hs | 1 + .../migrate-data/src/Gundeck/DataMigration.hs | 120 ++++++++++++++++++ .../src/Gundeck/DataMigration/Types.hs | 78 ++++++++++++ services/gundeck/migrate-data/src/Run.hs | 32 +++++ .../src/V1_DeleteApnsVoipTokens.hs | 90 +++++++++++++ 7 files changed, 408 insertions(+), 8 deletions(-) create mode 100644 services/gundeck/migrate-data/main.hs create mode 100644 services/gundeck/migrate-data/src/Gundeck/DataMigration.hs create mode 100644 services/gundeck/migrate-data/src/Gundeck/DataMigration/Types.hs create mode 100644 services/gundeck/migrate-data/src/Run.hs create mode 100644 services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs diff --git a/services/gundeck/default.nix b/services/gundeck/default.nix index a1c3759ede2..3614f31e11b 100644 --- a/services/gundeck/default.nix +++ b/services/gundeck/default.nix @@ -18,6 +18,7 @@ , bytestring , bytestring-conversion , cassandra-util +, conduit , containers , criterion , errors @@ -153,8 +154,10 @@ mkDerivation { bytestring bytestring-conversion cassandra-util + conduit containers exceptions + extended gundeck-types HsOpenSSL http-client @@ -175,6 +178,7 @@ mkDerivation { tasty-ant-xml tasty-hunit text + time tinylog types-common uuid diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index 45b786e9ca2..d55c7fdaeb8 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -1,4 +1,4 @@ -cabal-version: 1.12 +cabal-version: 3.0 name: gundeck version: 1.45.0 synopsis: Push Notification Hub @@ -6,7 +6,7 @@ category: Network author: Wire Swiss GmbH maintainer: Wire Swiss GmbH copyright: (c) 2017 Wire Swiss GmbH -license: AGPL-3 +license: AGPL-3.0-only license-file: LICENSE build-type: Simple @@ -110,7 +110,7 @@ library -Wredundant-constraints -Wunused-packages build-depends: - aeson >=2.0.1.0 + , aeson >=2.0.1.0 , amazonka >=2 , amazonka-core >=2 , amazonka-sns >=2 @@ -219,7 +219,7 @@ executable gundeck -Wunused-packages build-depends: - base + , base , gundeck , HsOpenSSL , imports @@ -288,7 +288,7 @@ executable gundeck-integration -threaded -Wredundant-constraints -Wunused-packages build-depends: - aeson + , aeson , async , base >=4 && <5 , base16-bytestring >=0.1 @@ -329,6 +329,81 @@ executable gundeck-integration default-language: GHC2021 +executable gundeck-migrate-data + main-is: ../main.hs + hs-source-dirs: migrate-data/src + default-extensions: + AllowAmbiguousTypes + BangPatterns + ConstraintKinds + DataKinds + DefaultSignatures + DeriveFunctor + DeriveGeneric + DeriveLift + DeriveTraversable + DerivingStrategies + DerivingVia + DuplicateRecordFields + EmptyCase + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTs + InstanceSigs + KindSignatures + LambdaCase + MultiParamTypeClasses + MultiWayIf + NamedFieldPuns + NoImplicitPrelude + OverloadedRecordDot + OverloadedStrings + PackageImports + PatternSynonyms + PolyKinds + QuasiQuotes + RankNTypes + ScopedTypeVariables + StandaloneDeriving + TupleSections + TypeApplications + TypeFamilies + TypeFamilyDependencies + TypeOperators + UndecidableInstances + ViewPatterns + + ghc-options: + -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates + -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path + -threaded -Wredundant-constraints -Wunused-packages + + -- cabal-fmt: expand migrate-data/src + other-modules: + Gundeck.DataMigration + Gundeck.DataMigration.Types + Run + V1_DeleteApnsVoipTokens + + build-depends: + , base + , cassandra-util + , conduit + , exceptions + , extended + , imports + , optparse-applicative + , text + , time + , tinylog + , types-common + + if flag(static) + ld-options: -static + + default-language: GHC2021 + executable gundeck-schema main-is: Main.hs hs-source-dirs: schema @@ -380,7 +455,7 @@ executable gundeck-schema -threaded -Wredundant-constraints -Wunused-packages build-depends: - gundeck + , gundeck , imports if flag(static) @@ -451,7 +526,7 @@ test-suite gundeck-tests -threaded -Wredundant-constraints -Wunused-packages build-depends: - aeson + , aeson , aeson-pretty , amazonka , amazonka-core @@ -540,7 +615,7 @@ benchmark gundeck-bench -Wredundant-constraints -Wunused-packages build-depends: - amazonka + , amazonka , base , criterion , gundeck diff --git a/services/gundeck/migrate-data/main.hs b/services/gundeck/migrate-data/main.hs new file mode 100644 index 00000000000..a26473d24ee --- /dev/null +++ b/services/gundeck/migrate-data/main.hs @@ -0,0 +1 @@ +import Run diff --git a/services/gundeck/migrate-data/src/Gundeck/DataMigration.hs b/services/gundeck/migrate-data/src/Gundeck/DataMigration.hs new file mode 100644 index 00000000000..a96d7aa829f --- /dev/null +++ b/services/gundeck/migrate-data/src/Gundeck/DataMigration.hs @@ -0,0 +1,120 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Gundeck.DataMigration (cassandraSettingsParser, migrate) where + +import Cassandra qualified as C +import Cassandra.Options +import Cassandra.Util (defInitCassandra) +import Control.Monad.Catch (finally) +import Data.Text qualified as Text +import Data.Time (UTCTime, getCurrentTime) +import Gundeck.DataMigration.Types +import Imports +import Options.Applicative (Parser) +import Options.Applicative qualified as Opts +import System.Logger.Class (Logger) +import System.Logger.Class qualified as Log + +data CassandraSettings = CassandraSettings + { cHost :: String, + cPort :: Word16, + cKeyspace :: C.Keyspace, + cTlsCa :: Maybe FilePath + } + +toCassandraOpts :: CassandraSettings -> CassandraOpts +toCassandraOpts cas = + CassandraOpts + { _endpoint = Endpoint (Text.pack (cas.cHost)) (cas.cPort), + _keyspace = C.unKeyspace (cas.cKeyspace), + _filterNodesByDatacentre = Nothing, + _tlsCa = cas.cTlsCa + } + +cassandraSettingsParser :: Parser CassandraSettings +cassandraSettingsParser = + CassandraSettings + <$> Opts.strOption + ( Opts.long "cassandra-host" + <> Opts.value "localhost" + ) + <*> Opts.option + Opts.auto + ( Opts.long "cassandra-port" + <> Opts.value 9042 + ) + <*> ( C.Keyspace + <$> Opts.strOption + ( Opts.long "cassandra-keyspace" + <> Opts.value "galley_test" + ) + ) + <*> ( (Opts.optional . Opts.strOption) + ( Opts.long "tls-ca-certificate-file" + <> Opts.help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ) + ) + +migrate :: Logger -> CassandraSettings -> [Migration] -> IO () +migrate l cas ms = do + env <- mkEnv l cas + finally (go env) (cleanup env) + where + go env = + runMigrationAction env $ + runMigrations ms + +mkEnv :: Logger -> CassandraSettings -> IO Env +mkEnv l cas = + Env + <$> initCassandra + <*> initLogger + where + initCassandra = defInitCassandra (toCassandraOpts cas) l + initLogger = pure l + +-- | Runs only the migrations which need to run +runMigrations :: [Migration] -> MigrationActionT IO () +runMigrations migrations = do + vmax <- latestMigrationVersion + let pendingMigrations = filter (\m -> version m > vmax) migrations + if null pendingMigrations + then info "No new migrations." + else info "New migrations found." + mapM_ runMigration pendingMigrations + +runMigration :: Migration -> MigrationActionT IO () +runMigration (Migration ver txt mig) = do + info $ "Running: [" <> show (migrationVersion ver) <> "] " <> Text.unpack txt + mig + persistVersion ver txt =<< liftIO getCurrentTime + +latestMigrationVersion :: MigrationActionT IO MigrationVersion +latestMigrationVersion = MigrationVersion . maybe 0 fromIntegral <$> C.query1 cql (C.params C.LocalQuorum ()) + where + cql :: C.QueryString C.R () (Identity Int32) + cql = "select version from data_migration where id=1 order by version desc limit 1" + +persistVersion :: MigrationVersion -> Text -> UTCTime -> MigrationActionT IO () +persistVersion (MigrationVersion v) desc time = C.write cql (C.params C.LocalQuorum (fromIntegral v, desc, time)) + where + cql :: C.QueryString C.W (Int32, Text, UTCTime) () + cql = "insert into data_migration (id, version, descr, date) values (1,?,?,?)" + +info :: Log.MonadLogger m => String -> m () +info = Log.info . Log.msg diff --git a/services/gundeck/migrate-data/src/Gundeck/DataMigration/Types.hs b/services/gundeck/migrate-data/src/Gundeck/DataMigration/Types.hs new file mode 100644 index 00000000000..4ad3dbb20d9 --- /dev/null +++ b/services/gundeck/migrate-data/src/Gundeck/DataMigration/Types.hs @@ -0,0 +1,78 @@ +{-# LANGUAGE GeneralizedNewtypeDeriving #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Gundeck.DataMigration.Types where + +import Cassandra qualified as C +import Control.Monad.Catch (MonadThrow) +import Imports +import Numeric.Natural (Natural) +import System.Logger qualified as Logger +import System.Logger.Class (MonadLogger (..)) + +data Migration = Migration + { version :: MigrationVersion, + text :: Text, + action :: MigrationActionT IO () + } + +newtype MigrationVersion = MigrationVersion {migrationVersion :: Natural} + deriving (Show, Eq, Ord) + +newtype MigrationActionT m a = MigrationActionT {unMigrationAction :: ReaderT Env m a} + deriving + ( Functor, + Applicative, + Monad, + MonadIO, + MonadThrow, + MonadReader Env, + MonadUnliftIO + ) + +instance MonadTrans MigrationActionT where + lift = MigrationActionT . lift + +instance (MonadIO m, MonadThrow m) => C.MonadClient (MigrationActionT m) where + liftClient = liftCassandra + localState f = local (\env -> env {cassandraClientState = f $ cassandraClientState env}) + +instance MonadIO m => MonadLogger (MigrationActionT m) where + log level f = do + env <- ask + Logger.log (logger env) level f + +data Env = Env + { cassandraClientState :: C.ClientState, + logger :: Logger.Logger + } + +runMigrationAction :: Env -> MigrationActionT m a -> m a +runMigrationAction env action = + runReaderT (unMigrationAction action) env + +liftCassandra :: MonadIO m => C.Client a -> MigrationActionT m a +liftCassandra m = do + env <- ask + lift $ C.runClient (cassandraClientState env) m + +cleanup :: (MonadIO m) => Env -> m () +cleanup env = do + C.shutdown (cassandraClientState env) + Logger.close (logger env) diff --git a/services/gundeck/migrate-data/src/Run.hs b/services/gundeck/migrate-data/src/Run.hs new file mode 100644 index 00000000000..42f539fcd56 --- /dev/null +++ b/services/gundeck/migrate-data/src/Run.hs @@ -0,0 +1,32 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Run where + +import Gundeck.DataMigration +import Imports +import Options.Applicative +import System.Logger.Extended qualified as Log +import V1_DeleteApnsVoipTokens qualified + +main :: IO () +main = do + o <- execParser (info (helper <*> cassandraSettingsParser) desc) + l <- Log.mkLogger Log.Debug Nothing Nothing + migrate l o [ V1_DeleteApnsVoipTokens.migration ] + where + desc = header "Gundeck Cassandra Data Migrations" <> fullDesc diff --git a/services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs b/services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs new file mode 100644 index 00000000000..2b4a2749750 --- /dev/null +++ b/services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs @@ -0,0 +1,90 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module V1_DeleteApnsVoipTokens where + +import Cassandra +import Conduit +import Data.Conduit.Internal (zipSources) +import Data.Conduit.List qualified as C +import Data.Id +import Gundeck.DataMigration.Types +import Imports +import System.Logger.Class qualified as Log +import Data.Text qualified as Text + +migration :: Migration +migration = + Migration + { version = MigrationVersion 1, + text = "Delete APNS_VOIP push tokens", + action = + runConduit $ + zipSources + (C.sourceList [(1 :: Int32) ..]) + getPushTokens + .| C.mapM + ( \(i, p) -> + Log.info (Log.field "push tokens" (show (i * pageSize))) + >> pure p + ) + .| C.concatMap (filter isVoipToken) + .| C.map (\(uid, token, app, transport, _mArn) -> (uid, token, app, transport)) + .| C.mapM_ deletePushToken + } + +pageSize :: Int32 +pageSize = 1000 + +---------------------------------------------------------------------------- +-- Queries + +-- | We do not use the push token types here because they will likely be +-- changed in future breaking this migration. +getPushTokens :: + MonadClient m => + ConduitM () [(UserId, Text, Text, Int32, Maybe Text)] m () +getPushTokens = paginateC cql (paramsP LocalQuorum () pageSize) x5 + where + cql :: PrepQuery R () (UserId, Text, Text, Int32, Maybe Text) + cql = "SELECT usr, ptoken, app, transport, arn FROM user_push" + +deletePushToken :: MonadClient m => (UserId, Text, Text, Int32) -> m () +deletePushToken pair = + retry x5 $ write cql (params LocalQuorum pair) + where + cql :: PrepQuery W (UserId, Text, Text, Int32) () + cql = "DELETE FROM user_push where usr = ? AND ptoken = ? AND app = ? AND transport = ?" + +isVoipTransport :: Int32 -> Bool +isVoipTransport 3 = True -- APNS_VOIP +isVoipTransport 4 = True -- APNS_VOIP_SANDBOX +isVoipTransport _ = False + +isVoipArn :: Text -> Bool +isVoipArn arn = + case Text.splitOn ":" arn of + ["arn", "aws", "sns", _region, _accountId, topic] -> + case Text.splitOn "/" topic of + ("endpoint" : "APNS_VOIP" : _ ) -> True + ("endpoint" : "APNS_VOIP_SANDBOX" : _ ) -> True + _ -> False + _ -> False + +isVoipToken :: (UserId, Text, Text, Int32, Maybe Text) -> Bool +isVoipToken (_, _, _, transport, mArn) = + isVoipTransport transport || maybe False isVoipArn mArn From 4139f8298754e6ac76ad7d039804ad81ace7cfd6 Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Wed, 8 May 2024 14:26:46 +0200 Subject: [PATCH 2/9] nix: Add gundeck-migrate-data image --- nix/wire-server.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/wire-server.nix b/nix/wire-server.nix index 9815de26869..eca106680b7 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -76,7 +76,7 @@ let cargohold = [ "cargohold" "cargohold-integration" ]; federator = [ "federator" "federator-integration" ]; galley = [ "galley" "galley-integration" "galley-schema" "galley-migrate-data" ]; - gundeck = [ "gundeck" "gundeck-integration" "gundeck-schema" ]; + gundeck = [ "gundeck" "gundeck-integration" "gundeck-schema" "gundeck-migrate-data" ]; proxy = [ "proxy" ]; spar = [ "spar" "spar-integration" "spar-schema" "spar-migrate-data" ]; stern = [ "stern" "stern-integration" ]; From 92d54f4cf9e91ee8843c1cd6a1470e5eb7b8bf41 Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Wed, 8 May 2024 14:33:56 +0200 Subject: [PATCH 3/9] charts/cassandra-migrations: Run gundeck-migrate-data --- .../templates/galley-migrate-data.yaml | 1 - .../templates/gundeck-migrate-data.yaml | 59 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 charts/cassandra-migrations/templates/gundeck-migrate-data.yaml diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index 127a6ab0b54..768f5a94326 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -20,7 +20,6 @@ spec: metadata: name: "{{.Release.Name}}" labels: - app: galley-migrate-data app: galley-migrate-data heritage: {{.Release.Service | quote }} release: {{.Release.Name | quote }} diff --git a/charts/cassandra-migrations/templates/gundeck-migrate-data.yaml b/charts/cassandra-migrations/templates/gundeck-migrate-data.yaml new file mode 100644 index 00000000000..159fdfbcfd2 --- /dev/null +++ b/charts/cassandra-migrations/templates/gundeck-migrate-data.yaml @@ -0,0 +1,59 @@ +# This jobs runs migrations on the gundeck DB using the gundeck-migrate-data tool. +# The source for the tool can be found at services/gundeck in the wire-server +# repository. +{{- if .Values.enableGundeckMigrations }} +apiVersion: batch/v1 +kind: Job +metadata: + name: gundeck-migrate-data + labels: + app: "cassandra-migrations" + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": "before-hook-creation" +spec: + template: + metadata: + name: "{{.Release.Name}}" + labels: + app: gundeck-migrate-data + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + spec: + restartPolicy: OnFailure + containers: + - name: gundeck-migrate-data + image: "{{ .Values.images.gundeckMigrateData }}:{{ .Values.images.tag }}" + imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }} + {{- if eq (include "includeSecurityContext" .) "true" }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + {{- end }} + args: + - --cassandra-host + - "{{ template "cassandraGundeckHost" . }}" + - --cassandra-port + - "9042" + - --cassandra-keyspace + - gundeck + {{- if eq (include "useTlsGundeck" .) "true" }} + - --tls-ca-certificate-file + - /certs/gundeck/{{- (include "tlsSecretRefGundeck" . | fromYaml).key }} + {{- end }} + {{- if eq (include "useTlsGundeck" .) "true" }} + volumeMounts: + - name: gundeck-cassandra-cert + mountPath: "/certs/gundeck" + {{- end }} + {{- if eq (include "useTlsGundeck" .) "true" }} + volumes: + - name: gundeck-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefGundeck" . | fromYaml).name }} + {{- end }} +{{- end }} From c3c3ec5100fd2b6cebbd31684779070439a8dfdc Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Wed, 8 May 2024 15:24:51 +0200 Subject: [PATCH 4/9] changelog --- changelog.d/2-features/delete-voip-tokens | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/2-features/delete-voip-tokens diff --git a/changelog.d/2-features/delete-voip-tokens b/changelog.d/2-features/delete-voip-tokens new file mode 100644 index 00000000000..3014514221c --- /dev/null +++ b/changelog.d/2-features/delete-voip-tokens @@ -0,0 +1 @@ +gundeck: Delete all APNS_VOIP and APNS_VOIP_SANDBOX push tokens From 756c70ee8f11792cca85143f8e0fb2e8c67c8ffd Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Wed, 8 May 2024 15:35:15 +0200 Subject: [PATCH 5/9] format --- services/gundeck/migrate-data/src/Run.hs | 2 +- .../migrate-data/src/V1_DeleteApnsVoipTokens.hs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/services/gundeck/migrate-data/src/Run.hs b/services/gundeck/migrate-data/src/Run.hs index 42f539fcd56..655851e58a1 100644 --- a/services/gundeck/migrate-data/src/Run.hs +++ b/services/gundeck/migrate-data/src/Run.hs @@ -27,6 +27,6 @@ main :: IO () main = do o <- execParser (info (helper <*> cassandraSettingsParser) desc) l <- Log.mkLogger Log.Debug Nothing Nothing - migrate l o [ V1_DeleteApnsVoipTokens.migration ] + migrate l o [V1_DeleteApnsVoipTokens.migration] where desc = header "Gundeck Cassandra Data Migrations" <> fullDesc diff --git a/services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs b/services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs index 2b4a2749750..0c7645797b8 100644 --- a/services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs +++ b/services/gundeck/migrate-data/src/V1_DeleteApnsVoipTokens.hs @@ -22,10 +22,10 @@ import Conduit import Data.Conduit.Internal (zipSources) import Data.Conduit.List qualified as C import Data.Id +import Data.Text qualified as Text import Gundeck.DataMigration.Types import Imports import System.Logger.Class qualified as Log -import Data.Text qualified as Text migration :: Migration migration = @@ -43,7 +43,7 @@ migration = >> pure p ) .| C.concatMap (filter isVoipToken) - .| C.map (\(uid, token, app, transport, _mArn) -> (uid, token, app, transport)) + .| C.map (\(uid, token, app, transport, _mArn) -> (uid, token, app, transport)) .| C.mapM_ deletePushToken } @@ -76,15 +76,15 @@ isVoipTransport 4 = True -- APNS_VOIP_SANDBOX isVoipTransport _ = False isVoipArn :: Text -> Bool -isVoipArn arn = +isVoipArn arn = case Text.splitOn ":" arn of - ["arn", "aws", "sns", _region, _accountId, topic] -> + ["arn", "aws", "sns", _region, _accountId, topic] -> case Text.splitOn "/" topic of - ("endpoint" : "APNS_VOIP" : _ ) -> True - ("endpoint" : "APNS_VOIP_SANDBOX" : _ ) -> True + ("endpoint" : "APNS_VOIP" : _) -> True + ("endpoint" : "APNS_VOIP_SANDBOX" : _) -> True _ -> False _ -> False - + isVoipToken :: (UserId, Text, Text, Int32, Maybe Text) -> Bool isVoipToken (_, _, _, transport, mArn) = isVoipTransport transport || maybe False isVoipArn mArn From d48f01e4580c9ad646375c6cc10d7f2d3fb7f572 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Wed, 8 May 2024 16:16:51 +0200 Subject: [PATCH 6/9] charts/cassandra-migrations: Add missing image name for gundeckMigrateData --- charts/cassandra-migrations/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/cassandra-migrations/values.yaml b/charts/cassandra-migrations/values.yaml index 283a010884f..a66fc6bc231 100644 --- a/charts/cassandra-migrations/values.yaml +++ b/charts/cassandra-migrations/values.yaml @@ -5,6 +5,7 @@ images: galley: quay.io/wire/galley-schema spar: quay.io/wire/spar-schema galleyMigrateData: quay.io/wire/galley-migrate-data + gundeckMigrateData: quay.io/wire/gundeck-migrate-data sparMigrateData: quay.io/wire/spar-migrate-data # Setting cassandra host name and replication is mandatory to specify. From 57da67c1536adb0db96387a70d698e1b04be4c69 Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Mon, 13 May 2024 10:13:20 +0200 Subject: [PATCH 7/9] [gundeck] fixed reference to galley test --- services/gundeck/migrate-data/src/Gundeck/DataMigration.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/gundeck/migrate-data/src/Gundeck/DataMigration.hs b/services/gundeck/migrate-data/src/Gundeck/DataMigration.hs index a96d7aa829f..6c01b748bf7 100644 --- a/services/gundeck/migrate-data/src/Gundeck/DataMigration.hs +++ b/services/gundeck/migrate-data/src/Gundeck/DataMigration.hs @@ -61,7 +61,7 @@ cassandraSettingsParser = <*> ( C.Keyspace <$> Opts.strOption ( Opts.long "cassandra-keyspace" - <> Opts.value "galley_test" + <> Opts.value "gundeck_test" ) ) <*> ( (Opts.optional . Opts.strOption) From 944240a71322ae98e3f6194a2f5eaa94aeaef164 Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Mon, 13 May 2024 11:03:43 +0200 Subject: [PATCH 8/9] [gundeck] Added missing data_migration table. --- services/gundeck/gundeck.cabal | 1 + services/gundeck/src/Gundeck/Schema/Run.hs | 4 ++- services/gundeck/src/Gundeck/Schema/V11.hs | 35 ++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 services/gundeck/src/Gundeck/Schema/V11.hs diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index d55c7fdaeb8..52532c525ad 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -46,6 +46,7 @@ library Gundeck.Schema.Run Gundeck.Schema.V1 Gundeck.Schema.V10 + Gundeck.Schema.V11 Gundeck.Schema.V2 Gundeck.Schema.V3 Gundeck.Schema.V4 diff --git a/services/gundeck/src/Gundeck/Schema/Run.hs b/services/gundeck/src/Gundeck/Schema/Run.hs index ccec5141e4f..247f7a69488 100644 --- a/services/gundeck/src/Gundeck/Schema/Run.hs +++ b/services/gundeck/src/Gundeck/Schema/Run.hs @@ -22,6 +22,7 @@ import Cassandra.Schema import Control.Exception (finally) import Gundeck.Schema.V1 qualified as V1 import Gundeck.Schema.V10 qualified as V10 +import Gundeck.Schema.V11 qualified as V11 import Gundeck.Schema.V2 qualified as V2 import Gundeck.Schema.V3 qualified as V3 import Gundeck.Schema.V4 qualified as V4 @@ -61,5 +62,6 @@ migrations = V7.migration, V8.migration, V9.migration, - V10.migration + V10.migration, + V11.migration ] diff --git a/services/gundeck/src/Gundeck/Schema/V11.hs b/services/gundeck/src/Gundeck/Schema/V11.hs new file mode 100644 index 00000000000..c734aab0a2f --- /dev/null +++ b/services/gundeck/src/Gundeck/Schema/V11.hs @@ -0,0 +1,35 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Gundeck.Schema.V11 (migration) where + +import Cassandra.Schema +import Imports +import Text.RawString.QQ + +migration :: Migration +migration = Migration 11 "Create table `data_migration`" $ do + schema' + [r| + CREATE TABLE data_migration ( + id int, + version int, + descr text, + date timestamp, + PRIMARY KEY (id, version) + ); + |] From be28b473ed09cc551f57a29af56a2ad4b3efd42a Mon Sep 17 00:00:00 2001 From: Igor Ranieri Date: Mon, 13 May 2024 11:15:02 +0200 Subject: [PATCH 9/9] cassandra-schema.cql: updated. --- cassandra-schema.cql | 52 +++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/cassandra-schema.cql b/cassandra-schema.cql index a35870fedfd..06052e52b77 100644 --- a/cassandra-schema.cql +++ b/cassandra-schema.cql @@ -1720,19 +1720,17 @@ CREATE TABLE galley_test.mls_proposal_refs ( AND speculative_retry = '99PERCENTILE'; CREATE KEYSPACE gundeck_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true; -CREATE TABLE gundeck_test.push ( - ptoken text, - app text, - transport int, - client text, - connection blob, - usr uuid, - PRIMARY KEY (ptoken, app, transport) -) WITH CLUSTERING ORDER BY (app ASC, transport ASC) - AND bloom_filter_fp_chance = 0.1 +CREATE TABLE gundeck_test.data_migration ( + id int, + version int, + date timestamp, + descr text, + PRIMARY KEY (id, version) +) WITH CLUSTERING ORDER BY (version ASC) + AND bloom_filter_fp_chance = 0.01 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} AND comment = '' - AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND crc_check_chance = 1.0 AND dclocal_read_repair_chance = 0.1 @@ -1790,10 +1788,16 @@ CREATE TABLE gundeck_test.meta ( AND read_repair_chance = 0.0 AND speculative_retry = '99PERCENTILE'; -CREATE TABLE gundeck_test.notification_payload ( - id uuid PRIMARY KEY, - payload blob -) WITH bloom_filter_fp_chance = 0.1 +CREATE TABLE gundeck_test.push ( + ptoken text, + app text, + transport int, + client text, + connection blob, + usr uuid, + PRIMARY KEY (ptoken, app, transport) +) WITH CLUSTERING ORDER BY (app ASC, transport ASC) + AND bloom_filter_fp_chance = 0.1 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} AND comment = '' AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} @@ -1832,6 +1836,24 @@ CREATE TABLE gundeck_test.user_push ( AND min_index_interval = 128 AND read_repair_chance = 0.0 AND speculative_retry = '99PERCENTILE'; + +CREATE TABLE gundeck_test.notification_payload ( + id uuid PRIMARY KEY, + payload blob +) WITH bloom_filter_fp_chance = 0.1 + AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} + AND comment = '' + AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} + AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND dclocal_read_repair_chance = 0.1 + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair_chance = 0.0 + AND speculative_retry = '99PERCENTILE'; CREATE KEYSPACE spar_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true; CREATE TABLE spar_test.bind_cookie (