diff --git a/cassandra-schema.cql b/cassandra-schema.cql index c2ba0cbe06..322e27ef81 100644 --- a/cassandra-schema.cql +++ b/cassandra-schema.cql @@ -1623,24 +1623,6 @@ CREATE TABLE brig_test.service_prefix ( CREATE KEYSPACE spar_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true; -CREATE TABLE spar_test.scim_external_ids ( - external text PRIMARY KEY, - user uuid -) WITH bloom_filter_fp_chance = 0.1 - AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} - AND comment = '' - AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'} - AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} - AND crc_check_chance = 1.0 - AND dclocal_read_repair_chance = 0.1 - AND default_time_to_live = 0 - AND gc_grace_seconds = 864000 - AND max_index_interval = 2048 - AND memtable_flush_period_in_ms = 0 - AND min_index_interval = 128 - AND read_repair_chance = 0.0 - AND speculative_retry = '99PERCENTILE'; - CREATE TABLE spar_test.bind_cookie ( cookie text PRIMARY KEY, session_owner uuid diff --git a/changelog.d/0-release-notes/cleanup-cassandra-schema b/changelog.d/0-release-notes/cleanup-cassandra-schema new file mode 100644 index 0000000000..47d4af17ee --- /dev/null +++ b/changelog.d/0-release-notes/cleanup-cassandra-schema @@ -0,0 +1,3 @@ +If you have not upgraded to [release 2021-03-21 (Chart Release 2.103.0)](https://github.com/wireapp/wire-server/releases/tag/v2021-03-21) yet, please do that now! + +NB: we only support releases 6 months back, so this should not be an issue. But in this particular case we are positive that things will break if you don't do an intermediate upgrade. (#2768) diff --git a/changelog.d/5-internal/cleanup-cassandra-schema b/changelog.d/5-internal/cleanup-cassandra-schema new file mode 100644 index 0000000000..539c8f1afd --- /dev/null +++ b/changelog.d/5-internal/cleanup-cassandra-schema @@ -0,0 +1,3 @@ +Remove deprecated table for storing scim external_ids. + +Data has been migrated away in [release 2021-03-21 (Chart Release 2.103.0)](https://github.com/wireapp/wire-server/releases/tag/v2021-03-21) (see `/services/spar/migrate-data/src/Spar/DataMigration/V1_ExternalIds.hs`); last time it has been touched in production is before upgrade to [release 2021-03-23 (Chart Release 2.104.0)](https://github.com/wireapp/wire-server/releases/tag/v2021-03-23). diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index f1be2836d4..4b13b42578 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -29,7 +29,6 @@ import Imports import qualified Options.Applicative as Opts import Spar.DataMigration.Options (settingsParser) import Spar.DataMigration.Types -import qualified Spar.DataMigration.V1_ExternalIds as V1 import qualified Spar.DataMigration.V2_UserV2 as V2 import qualified System.Logger as Log @@ -38,7 +37,8 @@ main = do settings <- Opts.execParser (Opts.info (Opts.helper <*> settingsParser) desc) migrate settings - [ V1.migration, + [ -- V1.migration has been deleted in https://github.com/wireapp/wire-server/pull/2768 + -- (because the deprecated source table has been removed). V2.migration ] where diff --git a/services/spar/migrate-data/src/Spar/DataMigration/V1_ExternalIds.hs b/services/spar/migrate-data/src/Spar/DataMigration/V1_ExternalIds.hs deleted file mode 100644 index ca5b9baf2a..0000000000 --- a/services/spar/migrate-data/src/Spar/DataMigration/V1_ExternalIds.hs +++ /dev/null @@ -1,199 +0,0 @@ -{-# LANGUAGE RecordWildCards #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Spar.DataMigration.V1_ExternalIds where - -import Cassandra -import qualified Cassandra as C -import Control.Lens -import Data.Conduit -import qualified Data.Conduit.Combinators as CC -import Data.Conduit.Internal (zipSources) -import qualified Data.Conduit.List as CL -import Data.Id -import qualified Data.Map.Strict as Map -import Imports -import Spar.DataMigration.RIO (RIO (..), modifyRef, readRef, runRIO) -import Spar.DataMigration.Types hiding (logger) -import qualified Spar.DataMigration.Types as Types -import System.Logger (Logger) -import qualified System.Logger as Log - -migration :: Migration -migration = - Migration - { version = MigrationVersion 1, - text = "Backfill spar.scim_external", - action = migrationAction - } - -type LegacyExternalId = (Text, UserId) - -type UserTeam = (UserId, Maybe TeamId) - -type NewExternalId = (TeamId, Text, UserId) - -data ResolveTeamResult - = UserHasNoTeam UserId Text - | NewExternalId NewExternalId - ---------------------------------------------------------------------------------- - -class HasMigEnv env where - migEnv :: env -> Env - -askMigEnv :: HasMigEnv env => RIO env Env -askMigEnv = asks migEnv - -class HasFailCount env where - failCount :: env -> IORef Int32 - -class HasLogger env where - logger :: env -> Logger - -logDebug :: HasLogger env => String -> RIO env () -logDebug msg = do - log_ :: Logger <- asks logger - Log.debug log_ (Log.msg msg) - -logWarn :: HasLogger env => String -> RIO env () -logWarn msg = do - log_ :: Logger <- asks logger - Log.warn log_ (Log.msg msg) - -class HasSpar env where - sparClientState :: env -> C.ClientState - -runSpar :: HasSpar env => Client a -> RIO env a -runSpar cl = do - cs <- asks sparClientState - runClient cs cl - -class HasBrig env where - brigClientState :: env -> C.ClientState - -runBrig :: HasBrig env => Client a -> RIO env a -runBrig cl = do - cs <- asks brigClientState - runClient cs cl - ---------------------------------------------------------------------------------- - -data V1Env = V1Env {v1FailCount :: IORef Int32, migrationEnv :: Env} - -instance HasSpar V1Env where - sparClientState = sparCassandra . migrationEnv - -instance HasBrig V1Env where - brigClientState = brigCassandra . migrationEnv - -instance HasLogger V1Env where - logger = Types.logger . migrationEnv - -instance HasMigEnv V1Env where - migEnv = migrationEnv - -instance HasFailCount V1Env where - failCount = v1FailCount - -migrationAction :: Env -> IO () -migrationAction migrationEnv = do - v1FailCount <- newIORef 0 - let v1env = V1Env {..} - runRIO v1env migrationMain - -migrationMain :: - ( HasSpar env, - HasBrig env, - HasLogger env, - HasMigEnv env, - HasFailCount env - ) => - RIO env () -migrationMain = do - runConduit $ - zipSources - (CL.sourceList [(1 :: Int32) ..]) - readLegacyExternalIds - .| CC.mapM resolveTeam - .| sink - - count <- readRef failCount - when (count > 0) $ - logWarn (show count <> " external ids have *NOT* been migrated.\nAn external id fails to be migrated if the mapped user doesn't exist or doesn't have a team.") - -readLegacyExternalIds :: (HasSpar env, HasMigEnv env) => ConduitM () [LegacyExternalId] (RIO env) () -readLegacyExternalIds = do - pSize <- lift $ pageSize <$> askMigEnv - transPipe runSpar $ - paginateC select (paramsP LocalQuorum () pSize) x5 - where - select :: PrepQuery R () LegacyExternalId - select = "SELECT external, user FROM scim_external_ids" - -resolveTeam :: (HasLogger env, HasBrig env) => (Int32, [LegacyExternalId]) -> RIO env [ResolveTeamResult] -resolveTeam (page, exts) = do - userToTeam <- Map.fromList <$> readUserTeam (fmap snd exts) - logDebug $ "Page " <> show page - pure $ - exts <&> \(extid, uid) -> - case uid `Map.lookup` userToTeam of - Just (Just tid) -> NewExternalId (tid, extid, uid) - _ -> UserHasNoTeam uid extid - where - readUserTeam :: HasBrig env => [UserId] -> RIO env [UserTeam] - readUserTeam uids = - runBrig $ do - query select (params LocalQuorum (Identity uids)) - where - select :: PrepQuery R (Identity [UserId]) UserTeam - select = "SELECT id, team FROM user where id in ?" - -sink :: - ( HasSpar env, - HasLogger env, - HasMigEnv env, - HasFailCount env - ) => - ConduitM [ResolveTeamResult] Void (RIO env) () -sink = go - where - go = do - mbResult <- await - for_ mbResult $ \results -> do - for_ results $ \case - UserHasNoTeam uid extid -> do - lift $ do - modifyRef failCount (+ 1) - dbg <- debug <$> askMigEnv - when (dbg == Debug) $ - logDebug ("No team for user " <> show uid <> " from extid " <> show extid) - NewExternalId (tid, extid, uid) -> - lift $ - askMigEnv - >>= ( \case - DryRun -> pure () - NoDryRun -> - runSpar $ - write insert (params LocalQuorum (tid, extid, uid)) - ) - . dryRun - go - insert :: PrepQuery W (TeamId, Text, UserId) () - insert = "INSERT INTO scim_external (team, external_id, user) VALUES (?, ?, ?)" diff --git a/services/spar/schema/src/Run.hs b/services/spar/schema/src/Run.hs index 03224b6de3..e82ba618bd 100644 --- a/services/spar/schema/src/Run.hs +++ b/services/spar/schema/src/Run.hs @@ -31,6 +31,7 @@ import qualified V13 import qualified V14 import qualified V15 import qualified V16 +import qualified V17 import qualified V2 import qualified V3 import qualified V4 @@ -65,7 +66,8 @@ main = do V13.migration, V14.migration, V15.migration, - V16.migration + V16.migration, + V17.migration -- When adding migrations here, don't forget to update -- 'schemaVersion' in Spar.Data diff --git a/services/spar/schema/src/V17.hs b/services/spar/schema/src/V17.hs new file mode 100644 index 0000000000..bccc4aab7d --- /dev/null +++ b/services/spar/schema/src/V17.hs @@ -0,0 +1,33 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module V17 + ( migration, + ) +where + +import Cassandra.Schema +import Imports +import Text.RawString.QQ + +migration :: Migration +migration = Migration 17 "Remove table `scim_external_ids` (from db migration V10, deprecated in favor of `scim_external`, data migrated in `/services/spar/migrate-data/src/Spar/DataMigration/V1_ExternalIds.hs`)" $ do + void $ + schema' + [r| + DROP TABLE if exists scim_external_ids; + |] diff --git a/services/spar/spar.cabal b/services/spar/spar.cabal index 655f2b3401..8a879ee934 100644 --- a/services/spar/spar.cabal +++ b/services/spar/spar.cabal @@ -474,7 +474,6 @@ executable spar-migrate-data Spar.DataMigration.RIO Spar.DataMigration.Run Spar.DataMigration.Types - Spar.DataMigration.V1_ExternalIds Spar.DataMigration.V2_UserV2 hs-source-dirs: migrate-data/src @@ -611,6 +610,7 @@ executable spar-schema V14 V15 V16 + V17 V2 V3 V4 diff --git a/services/spar/src/Spar/Data.hs b/services/spar/src/Spar/Data.hs index 7f197154f7..3c53a9d9aa 100644 --- a/services/spar/src/Spar/Data.hs +++ b/services/spar/src/Spar/Data.hs @@ -48,7 +48,7 @@ import Wire.API.User.Saml -- | A lower bound: @schemaVersion <= whatWeFoundOnCassandra@, not @==@. schemaVersion :: Int32 -schemaVersion = 16 +schemaVersion = 17 ---------------------------------------------------------------------- -- helpers