From ddb124cf27de23a246c57107722e0b68cac6e65f Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Mon, 8 Jul 2024 15:47:16 +0000
Subject: [PATCH 01/13] delete phone keys

---
 cabal.project                                 |   1 +
 changelog.d/5-internal/WPB-8709               |   1 +
 nix/local-haskell-packages.nix                |   1 +
 nix/wire-server.nix                           |   2 +-
 tools/phone-data-migration/.ormolu            |   1 +
 tools/phone-data-migration/README.md          |   1 +
 tools/phone-data-migration/app/Main.hs        |  23 +++
 tools/phone-data-migration/default.nix        |  40 +++++
 .../phone-data-migration.cabal                |  92 +++++++++++
 .../src/PhoneDataMigration/Lib.hs             |  74 +++++++++
 .../src/PhoneDataMigration/Types.hs           | 151 ++++++++++++++++++
 11 files changed, 386 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/5-internal/WPB-8709
 create mode 120000 tools/phone-data-migration/.ormolu
 create mode 100644 tools/phone-data-migration/README.md
 create mode 100644 tools/phone-data-migration/app/Main.hs
 create mode 100644 tools/phone-data-migration/default.nix
 create mode 100644 tools/phone-data-migration/phone-data-migration.cabal
 create mode 100644 tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
 create mode 100644 tools/phone-data-migration/src/PhoneDataMigration/Types.hs

diff --git a/cabal.project b/cabal.project
index fe2c42af262..3c0a1bf2993 100644
--- a/cabal.project
+++ b/cabal.project
@@ -52,6 +52,7 @@ packages:
   , tools/db/repair-brig-clients-table/
   , tools/db/service-backfill/
   , tools/fedcalls/
+  , tools/phone-data-migration
   , tools/rabbitmq-consumer
   , tools/rex/
   , tools/stern/
diff --git a/changelog.d/5-internal/WPB-8709 b/changelog.d/5-internal/WPB-8709
new file mode 100644
index 00000000000..7c644d2437b
--- /dev/null
+++ b/changelog.d/5-internal/WPB-8709
@@ -0,0 +1 @@
+phone data is removed from DB
diff --git a/nix/local-haskell-packages.nix b/nix/local-haskell-packages.nix
index 133fcd9afae..14013d63bbf 100644
--- a/nix/local-haskell-packages.nix
+++ b/nix/local-haskell-packages.nix
@@ -54,6 +54,7 @@
   service-backfill = hself.callPackage ../tools/db/service-backfill/default.nix { inherit gitignoreSource; };
   fedcalls = hself.callPackage ../tools/fedcalls/default.nix { inherit gitignoreSource; };
   mlsstats = hself.callPackage ../tools/mlsstats/default.nix { inherit gitignoreSource; };
+  phone-data-migration = hself.callPackage ../tools/phone-data-migration/default.nix { inherit gitignoreSource; };
   rabbitmq-consumer = hself.callPackage ../tools/rabbitmq-consumer/default.nix { inherit gitignoreSource; };
   rex = hself.callPackage ../tools/rex/default.nix { inherit gitignoreSource; };
   stern = hself.callPackage ../tools/stern/default.nix { inherit gitignoreSource; };
diff --git a/nix/wire-server.nix b/nix/wire-server.nix
index 6fafadb5efa..d052b10c7d6 100644
--- a/nix/wire-server.nix
+++ b/nix/wire-server.nix
@@ -477,7 +477,7 @@ let
       out = import ./all-toplevel-derivations.nix {
         inherit (pkgs) lib;
         fn = mk;
-        # more than two takes more than 32GB of RAM, so this is what 
+        # more than two takes more than 32GB of RAM, so this is what
         # we're limiting ourselves to
         recursionDepth = 2;
         keyFilter = k: k != "passthru";
diff --git a/tools/phone-data-migration/.ormolu b/tools/phone-data-migration/.ormolu
new file mode 120000
index 00000000000..ffc2ca9745e
--- /dev/null
+++ b/tools/phone-data-migration/.ormolu
@@ -0,0 +1 @@
+../../../.ormolu
\ No newline at end of file
diff --git a/tools/phone-data-migration/README.md b/tools/phone-data-migration/README.md
new file mode 100644
index 00000000000..6f4a9f26e4e
--- /dev/null
+++ b/tools/phone-data-migration/README.md
@@ -0,0 +1 @@
+# Phone data migration
diff --git a/tools/phone-data-migration/app/Main.hs b/tools/phone-data-migration/app/Main.hs
new file mode 100644
index 00000000000..6ba1f3d6873
--- /dev/null
+++ b/tools/phone-data-migration/app/Main.hs
@@ -0,0 +1,23 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2024 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Main where
+
+import qualified PhoneDataMigration.Lib as Lib
+
+main :: IO ()
+main = Lib.main
diff --git a/tools/phone-data-migration/default.nix b/tools/phone-data-migration/default.nix
new file mode 100644
index 00000000000..fdada762120
--- /dev/null
+++ b/tools/phone-data-migration/default.nix
@@ -0,0 +1,40 @@
+# WARNING: GENERATED FILE, DO NOT EDIT.
+# This file is generated by running hack/bin/generate-local-nix-packages.sh and
+# must be regenerated whenever local packages are added or removed, or
+# dependencies are added or removed.
+{ mkDerivation
+, attoparsec
+, base
+, cassandra-util
+, conduit
+, exceptions
+, gitignoreSource
+, imports
+, lens
+, lib
+, optparse-applicative
+, text
+, tinylog
+}:
+mkDerivation {
+  pname = "phone-data-migration";
+  version = "1.0.0";
+  src = gitignoreSource ./.;
+  isLibrary = true;
+  isExecutable = true;
+  libraryHaskellDepends = [
+    attoparsec
+    cassandra-util
+    conduit
+    exceptions
+    imports
+    lens
+    optparse-applicative
+    text
+    tinylog
+  ];
+  executableHaskellDepends = [ base ];
+  description = "remove phone data from wire-server";
+  license = lib.licenses.agpl3Only;
+  mainProgram = "phone-data-migration";
+}
diff --git a/tools/phone-data-migration/phone-data-migration.cabal b/tools/phone-data-migration/phone-data-migration.cabal
new file mode 100644
index 00000000000..86ef2dfdd26
--- /dev/null
+++ b/tools/phone-data-migration/phone-data-migration.cabal
@@ -0,0 +1,92 @@
+cabal-version: 3.0
+name:          phone-data-migration
+version:       1.0.0
+synopsis:      remove phone data from wire-server
+category:      Network
+author:        Wire Swiss GmbH
+maintainer:    Wire Swiss GmbH <backend@wire.com>
+copyright:     (c) 2024 Wire Swiss GmbH
+license:       AGPL-3.0-only
+build-type:    Simple
+
+library
+  hs-source-dirs:     src
+  exposed-modules:
+    PhoneDataMigration.Lib
+    PhoneDataMigration.Types
+
+  ghc-options:
+    -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates
+    -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path
+    -funbox-strict-fields -threaded -with-rtsopts=-N
+    -Wredundant-constraints -Wunused-packages
+
+  build-depends:
+    , attoparsec
+    , cassandra-util
+    , conduit
+    , exceptions
+    , imports
+    , lens
+    , optparse-applicative
+    , text
+    , tinylog
+
+  default-extensions:
+    AllowAmbiguousTypes
+    BangPatterns
+    ConstraintKinds
+    DataKinds
+    DefaultSignatures
+    DeriveFunctor
+    DeriveGeneric
+    DeriveLift
+    DeriveTraversable
+    DerivingStrategies
+    DerivingVia
+    DuplicateRecordFields
+    EmptyCase
+    FlexibleContexts
+    FlexibleInstances
+    FunctionalDependencies
+    GADTs
+    GeneralizedNewtypeDeriving
+    InstanceSigs
+    KindSignatures
+    LambdaCase
+    MultiParamTypeClasses
+    MultiWayIf
+    NamedFieldPuns
+    NoImplicitPrelude
+    OverloadedLabels
+    OverloadedRecordDot
+    OverloadedStrings
+    PackageImports
+    PatternSynonyms
+    PolyKinds
+    QuasiQuotes
+    RankNTypes
+    RecordWildCards
+    ScopedTypeVariables
+    StandaloneDeriving
+    TupleSections
+    TypeApplications
+    TypeFamilies
+    TypeFamilyDependencies
+    TypeOperators
+    UndecidableInstances
+    ViewPatterns
+
+executable phone-data-migration
+  main-is:          Main.hs
+  build-depends:
+    , base
+    , phone-data-migration
+
+  hs-source-dirs:   app
+  default-language: Haskell2010
+  ghc-options:
+    -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates
+    -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path
+    -funbox-strict-fields -threaded -with-rtsopts=-N
+    -Wredundant-constraints -Wunused-packages
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs b/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
new file mode 100644
index 00000000000..bdaa029bb1b
--- /dev/null
+++ b/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
@@ -0,0 +1,74 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2024 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module PhoneDataMigration.Lib where
+
+import Cassandra as C
+import Data.Conduit
+import qualified Data.Conduit.Combinators as Conduit
+import qualified Data.Conduit.List as ConduitList
+import Imports
+import Options.Applicative
+import PhoneDataMigration.Types
+import qualified System.Logger.Class as Log
+import System.Logger.Message ((.=))
+
+pageSize :: Int32
+pageSize = 1000
+
+getKeys :: (MonadClient m) => ConduitM () Phone m ()
+getKeys =
+  paginateC cql (paramsP LocalQuorum () pageSize) x5
+    .| Conduit.concatMap (mapMaybe (parsePhone . runIdentity))
+  where
+    cql :: PrepQuery R () (Identity Text)
+    cql = "SELECT key FROM user_keys"
+
+deleteKey :: (MonadClient m) => Phone -> m IntSum
+deleteKey p = do
+  retry x5 $ write cql (params LocalQuorum (Identity $ fromPhone p))
+  pure 1
+  where
+    cql :: PrepQuery W (Identity Text) ()
+    cql = "DELETE FROM user_keys WHERE key = ?"
+
+deleteKeys :: [Phone] -> ConduitT a IntSum (AppT IO) ()
+deleteKeys keys = do
+  ConduitList.sourceList keys
+    .| Conduit.mapM deleteKey
+    .| Conduit.scanl (<>) mempty
+    .| Conduit.iterM (logEvery 100000)
+  where
+    logEvery :: Int -> IntSum -> AppT IO ()
+    logEvery i s =
+      when (unIntSum s `mod` i == 0) $ Log.info $ "deleted_keys" .= show s
+
+run :: AppT IO ()
+run = do
+  phoneKeys <- runConduit $ getKeys .| Conduit.sinkList
+  result <- runConduit $ deleteKeys phoneKeys .| Conduit.lastDef mempty
+  Log.info $ "final_deleted_keys" .= show result
+
+main :: IO ()
+main = do
+  opts <- execParser (info (helper <*> optsParser) desc)
+  env <- mkEnv opts
+  runReaderT (unAppT run) env
+  where
+    desc = header "phone-data-migration" <> progDesc "Remove phone data from wire-server" <> fullDesc
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Types.hs b/tools/phone-data-migration/src/PhoneDataMigration/Types.hs
new file mode 100644
index 00000000000..afdd1450479
--- /dev/null
+++ b/tools/phone-data-migration/src/PhoneDataMigration/Types.hs
@@ -0,0 +1,151 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2024 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module PhoneDataMigration.Types where
+
+import Cassandra as C
+import qualified Cassandra.Settings as C
+import Control.Lens
+import Control.Monad.Catch (MonadThrow)
+import Data.Attoparsec.Text (char, count, digit, endOfInput, parseOnly)
+import qualified Data.Text as Text
+import Data.Text.Strict.Lens
+import Imports
+import Options.Applicative
+import qualified System.Logger as Log
+import System.Logger.Class (MonadLogger, log)
+
+data Env = Env
+  { casClient :: C.ClientState,
+    logger :: Log.Logger
+  }
+
+newtype AppT m a = AppT {unAppT :: ReaderT Env m a}
+  deriving
+    ( Functor,
+      Applicative,
+      Monad,
+      MonadIO,
+      MonadThrow,
+      MonadReader Env,
+      MonadUnliftIO
+    )
+
+instance MonadTrans AppT where
+  lift = AppT . lift
+
+instance (MonadIO m, MonadThrow m) => C.MonadClient (AppT m) where
+  liftClient m = do
+    env <- ask
+    lift . C.runClient env.casClient $ m
+  localState f = local (\env -> env {casClient = f $ env.casClient})
+
+instance (MonadIO m) => MonadLogger (AppT m) where
+  log level f = do
+    env <- ask
+    lift $ Log.log (env.logger) level f
+
+mkEnv :: Opts -> IO Env
+mkEnv opts = do
+  logger <- initLogger
+  brigClient <- initCas opts.brigDb logger
+  pure $ Env brigClient logger
+  where
+    initLogger =
+      Log.new
+        . Log.setOutput Log.StdOut
+        . Log.setFormat Nothing
+        . Log.setBufSize 0
+        $ Log.defSettings
+    initCas settings l =
+      C.init
+        . C.setLogger (C.mkLogger l)
+        . C.setContacts settings.host []
+        . C.setPortNumber (fromIntegral settings.port)
+        . C.setKeyspace settings.keyspace
+        . C.setProtocolVersion C.V4
+        $ C.defSettings
+
+data CassandraSettings = CassandraSettings
+  { host :: String,
+    port :: Int,
+    keyspace :: C.Keyspace
+  }
+
+data Opts = Opts
+  { brigDb :: CassandraSettings
+  }
+
+optsParser :: Parser Opts
+optsParser =
+  Opts <$> brigCassandraParser
+
+brigCassandraParser :: Parser CassandraSettings
+brigCassandraParser =
+  CassandraSettings
+    <$> strOption
+      ( long "brig-cassandra-host"
+          <> metavar "HOST"
+          <> help "Cassandra Host for brig"
+          <> value "localhost"
+          <> showDefault
+      )
+    <*> option
+      auto
+      ( long "brig-cassandra-port"
+          <> metavar "PORT"
+          <> help "Cassandra Port for brig"
+          <> value 9042
+          <> showDefault
+      )
+    <*> ( C.Keyspace
+            . view packed
+            <$> strOption
+              ( long "brig-cassandra-keyspace"
+                  <> metavar "STRING"
+                  <> help "Cassandra Keyspace for brig"
+                  <> value "brig_test"
+                  <> showDefault
+              )
+        )
+
+newtype IntSum = IntSum {unIntSum :: Int}
+  deriving newtype (Num, Show)
+
+instance Semigroup IntSum where
+  IntSum a <> IntSum b = IntSum (a + b)
+
+instance Monoid IntSum where
+  mempty = IntSum 0
+
+newtype Phone = Phone {fromPhone :: Text}
+  deriving stock (Eq, Ord, Show, Generic)
+
+-- | Parses a phone number in E.164 format with a mandatory leading '+'.
+parsePhone :: Text -> Maybe Phone
+parsePhone p =
+  let canonicalPhone = Text.filter (not . isSpace) p
+   in if isValidPhone canonicalPhone
+        then Just $ Phone canonicalPhone
+        else Nothing
+
+-- | Checks whether a phone number is valid, i.e. it is in E.164 format
+-- with a mandatory leading '+' followed by 10-15 digits.
+isValidPhone :: Text -> Bool
+isValidPhone = either (const False) (const True) . parseOnly e164
+  where
+    e164 = char '+' *> count 8 digit *> count 7 (optional digit) *> endOfInput

From 466310e9b8cc6bbacc4480c5daade06ac59c5190 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 08:25:14 +0000
Subject: [PATCH 02/13] hide ctor of phone for safety

---
 .../phone-data-migration.cabal                |  1 +
 .../src/PhoneDataMigration/Lib.hs             |  8 ++++--
 .../src/PhoneDataMigration/Phone.hs           | 28 +++++++++++++++++++
 .../src/PhoneDataMigration/Types.hs           | 20 -------------
 4 files changed, 35 insertions(+), 22 deletions(-)
 create mode 100644 tools/phone-data-migration/src/PhoneDataMigration/Phone.hs

diff --git a/tools/phone-data-migration/phone-data-migration.cabal b/tools/phone-data-migration/phone-data-migration.cabal
index 86ef2dfdd26..daeee0211c5 100644
--- a/tools/phone-data-migration/phone-data-migration.cabal
+++ b/tools/phone-data-migration/phone-data-migration.cabal
@@ -13,6 +13,7 @@ library
   hs-source-dirs:     src
   exposed-modules:
     PhoneDataMigration.Lib
+    PhoneDataMigration.Phone
     PhoneDataMigration.Types
 
   ghc-options:
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs b/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
index bdaa029bb1b..f0316704fe1 100644
--- a/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
+++ b/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
@@ -25,6 +25,7 @@ import qualified Data.Conduit.Combinators as Conduit
 import qualified Data.Conduit.List as ConduitList
 import Imports
 import Options.Applicative
+import PhoneDataMigration.Phone
 import PhoneDataMigration.Types
 import qualified System.Logger.Class as Log
 import System.Logger.Message ((.=))
@@ -42,7 +43,7 @@ getKeys =
 
 deleteKey :: (MonadClient m) => Phone -> m IntSum
 deleteKey p = do
-  retry x5 $ write cql (params LocalQuorum (Identity $ fromPhone p))
+  retry x5 $ write cql (params LocalQuorum (Identity $ toText p))
   pure 1
   where
     cql :: PrepQuery W (Identity Text) ()
@@ -61,9 +62,12 @@ deleteKeys keys = do
 
 run :: AppT IO ()
 run = do
+  -- deleting from user_keys while paginating through it is probably not a good idea
+  -- therefore we read all keys first and then delete them
   phoneKeys <- runConduit $ getKeys .| Conduit.sinkList
+  Log.info $ "phone_keys" .= show (length phoneKeys)
   result <- runConduit $ deleteKeys phoneKeys .| Conduit.lastDef mempty
-  Log.info $ "final_deleted_keys" .= show result
+  Log.info $ "final_deleted_phone_keys" .= show result
 
 main :: IO ()
 main = do
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs b/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs
new file mode 100644
index 00000000000..8640fa02068
--- /dev/null
+++ b/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs
@@ -0,0 +1,28 @@
+
+module PhoneDataMigration.Phone (Phone, parsePhone, toText) where
+
+import Data.Attoparsec.Text (char, count, digit, endOfInput, parseOnly)
+import qualified Data.Text as Text
+import Imports
+import Options.Applicative
+
+newtype Phone = Phone {fromPhone :: Text}
+  deriving stock (Eq, Ord, Show, Generic)
+
+toText :: Phone -> Text
+toText = fromPhone
+
+-- | Parses a phone number in E.164 format with a mandatory leading '+'.
+parsePhone :: Text -> Maybe Phone
+parsePhone p =
+  let canonicalPhone = Text.filter (not . isSpace) p
+   in if isValidPhone canonicalPhone
+        then Just $ Phone canonicalPhone
+        else Nothing
+
+-- | Checks whether a phone number is valid, i.e. it is in E.164 format
+-- with a mandatory leading '+' followed by 10-15 digits.
+isValidPhone :: Text -> Bool
+isValidPhone = either (const False) (const True) . parseOnly e164
+  where
+    e164 = char '+' *> count 8 digit *> count 7 (optional digit) *> endOfInput
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Types.hs b/tools/phone-data-migration/src/PhoneDataMigration/Types.hs
index afdd1450479..3e9dd1edf3a 100644
--- a/tools/phone-data-migration/src/PhoneDataMigration/Types.hs
+++ b/tools/phone-data-migration/src/PhoneDataMigration/Types.hs
@@ -21,8 +21,6 @@ import Cassandra as C
 import qualified Cassandra.Settings as C
 import Control.Lens
 import Control.Monad.Catch (MonadThrow)
-import Data.Attoparsec.Text (char, count, digit, endOfInput, parseOnly)
-import qualified Data.Text as Text
 import Data.Text.Strict.Lens
 import Imports
 import Options.Applicative
@@ -131,21 +129,3 @@ instance Semigroup IntSum where
 
 instance Monoid IntSum where
   mempty = IntSum 0
-
-newtype Phone = Phone {fromPhone :: Text}
-  deriving stock (Eq, Ord, Show, Generic)
-
--- | Parses a phone number in E.164 format with a mandatory leading '+'.
-parsePhone :: Text -> Maybe Phone
-parsePhone p =
-  let canonicalPhone = Text.filter (not . isSpace) p
-   in if isValidPhone canonicalPhone
-        then Just $ Phone canonicalPhone
-        else Nothing
-
--- | Checks whether a phone number is valid, i.e. it is in E.164 format
--- with a mandatory leading '+' followed by 10-15 digits.
-isValidPhone :: Text -> Bool
-isValidPhone = either (const False) (const True) . parseOnly e164
-  where
-    e164 = char '+' *> count 8 digit *> count 7 (optional digit) *> endOfInput

From 5f4bc91d4d74fd255972f70de3d31730b1d59372 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 09:07:08 +0000
Subject: [PATCH 03/13] log while searching for phone keys

---
 .../src/PhoneDataMigration/Lib.hs                 | 15 +++++++++++----
 .../src/PhoneDataMigration/Phone.hs               |  1 -
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs b/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
index f0316704fe1..bb3e2306b6f 100644
--- a/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
+++ b/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
@@ -27,20 +27,27 @@ import Imports
 import Options.Applicative
 import PhoneDataMigration.Phone
 import PhoneDataMigration.Types
+import System.Logger.Class (MonadLogger)
 import qualified System.Logger.Class as Log
 import System.Logger.Message ((.=))
 
 pageSize :: Int32
 pageSize = 1000
 
-getKeys :: (MonadClient m) => ConduitM () Phone m ()
-getKeys =
+getKeys :: forall m. (MonadClient m, MonadLogger m) => IORef Int -> ConduitM () Phone m ()
+getKeys counter =
   paginateC cql (paramsP LocalQuorum () pageSize) x5
     .| Conduit.concatMap (mapMaybe (parsePhone . runIdentity))
+    .| Conduit.iterM (const $ logEvery 100000 counter)
   where
     cql :: PrepQuery R () (Identity Text)
     cql = "SELECT key FROM user_keys"
 
+    logEvery :: Int -> IORef Int -> m ()
+    logEvery i ctr = do
+      c <- liftIO $ atomicModifyIORef' ctr (\x -> (x + 1, x + 1))
+      when (c `mod` i == 0) $ Log.info $ "found_phone_keys" .= show c
+
 deleteKey :: (MonadClient m) => Phone -> m IntSum
 deleteKey p = do
   retry x5 $ write cql (params LocalQuorum (Identity $ toText p))
@@ -62,10 +69,10 @@ deleteKeys keys = do
 
 run :: AppT IO ()
 run = do
+  ctr <- liftIO $ newIORef 0
   -- deleting from user_keys while paginating through it is probably not a good idea
   -- therefore we read all keys first and then delete them
-  phoneKeys <- runConduit $ getKeys .| Conduit.sinkList
-  Log.info $ "phone_keys" .= show (length phoneKeys)
+  phoneKeys <- runConduit $ getKeys ctr .| Conduit.sinkList
   result <- runConduit $ deleteKeys phoneKeys .| Conduit.lastDef mempty
   Log.info $ "final_deleted_phone_keys" .= show result
 
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs b/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs
index 8640fa02068..f0e82fc1da4 100644
--- a/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs
+++ b/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs
@@ -1,4 +1,3 @@
-
 module PhoneDataMigration.Phone (Phone, parsePhone, toText) where
 
 import Data.Attoparsec.Text (char, count, digit, endOfInput, parseOnly)

From b896466bbc10756395f5ad223736f1e3c61f1ce8 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 09:30:02 +0000
Subject: [PATCH 04/13] db migration for dropping phone column

---
 cassandra-schema.cql                          |  1 -
 services/brig/brig.cabal                      |  1 +
 services/brig/src/Brig/Schema/Run.hs          |  4 ++-
 .../src/Brig/Schema/V82_DropPhoneColumn.hs    | 35 +++++++++++++++++++
 4 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 services/brig/src/Brig/Schema/V82_DropPhoneColumn.hs

diff --git a/cassandra-schema.cql b/cassandra-schema.cql
index bbeefe5b6e3..0e143beeacf 100644
--- a/cassandra-schema.cql
+++ b/cassandra-schema.cql
@@ -678,7 +678,6 @@ CREATE TABLE brig_test.user (
     managed_by int,
     name text,
     password blob,
-    phone text,
     picture list<blob>,
     provider uuid,
     searchable boolean,
diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal
index 49d45527d52..fd3434fe99d 100644
--- a/services/brig/brig.cabal
+++ b/services/brig/brig.cabal
@@ -188,6 +188,7 @@ library
     Brig.Schema.V79_ConnectionRemoteIndex
     Brig.Schema.V80_KeyPackageCiphersuite
     Brig.Schema.V81_AddFederationRemoteTeams
+    Brig.Schema.V82_DropPhoneColumn
     Brig.Schema.V_FUTUREWORK
     Brig.Team.API
     Brig.Team.DB
diff --git a/services/brig/src/Brig/Schema/Run.hs b/services/brig/src/Brig/Schema/Run.hs
index 049a51e5f5f..be608d28578 100644
--- a/services/brig/src/Brig/Schema/Run.hs
+++ b/services/brig/src/Brig/Schema/Run.hs
@@ -56,6 +56,7 @@ import Brig.Schema.V78_ClientLastActive qualified as V78_ClientLastActive
 import Brig.Schema.V79_ConnectionRemoteIndex qualified as V79_ConnectionRemoteIndex
 import Brig.Schema.V80_KeyPackageCiphersuite qualified as V80_KeyPackageCiphersuite
 import Brig.Schema.V81_AddFederationRemoteTeams qualified as V81_AddFederationRemoteTeams
+import Brig.Schema.V82_DropPhoneColumn qualified as V82_DropPhoneColumn
 import Cassandra.MigrateSchema (migrateSchema)
 import Cassandra.Schema
 import Control.Exception (finally)
@@ -118,7 +119,8 @@ migrations =
     V78_ClientLastActive.migration,
     V79_ConnectionRemoteIndex.migration,
     V80_KeyPackageCiphersuite.migration,
-    V81_AddFederationRemoteTeams.migration
+    V81_AddFederationRemoteTeams.migration,
+    V82_DropPhoneColumn.migration
     -- FUTUREWORK: undo V41 (searchable flag); we stopped using it in
     -- https://github.com/wireapp/wire-server/pull/964
     --
diff --git a/services/brig/src/Brig/Schema/V82_DropPhoneColumn.hs b/services/brig/src/Brig/Schema/V82_DropPhoneColumn.hs
new file mode 100644
index 00000000000..dc086e04a60
--- /dev/null
+++ b/services/brig/src/Brig/Schema/V82_DropPhoneColumn.hs
@@ -0,0 +1,35 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2023 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Brig.Schema.V82_DropPhoneColumn
+  ( migration,
+  )
+where
+
+import Cassandra.Schema
+import Imports
+import Text.RawString.QQ
+
+migration :: Migration
+migration =
+  Migration 82 "Drop phone column from user table" $ do
+    schema'
+      [r|
+        ALTER TABLE user DROP phone
+     |]

From a37e07de03f4103bb1dcdecd7aa3e1c959cb5ed0 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 09:42:22 +0000
Subject: [PATCH 05/13] changelog

---
 changelog.d/5-internal/WPB-10058 | 2 ++
 changelog.d/5-internal/WPB-8709  | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/5-internal/WPB-10058
 delete mode 100644 changelog.d/5-internal/WPB-8709

diff --git a/changelog.d/5-internal/WPB-10058 b/changelog.d/5-internal/WPB-10058
new file mode 100644
index 00000000000..1cdda618893
--- /dev/null
+++ b/changelog.d/5-internal/WPB-10058
@@ -0,0 +1,2 @@
+- DB migration for dropping `phone` column from `user` table
+- Tool for deleting phone keys from `user_keys`
diff --git a/changelog.d/5-internal/WPB-8709 b/changelog.d/5-internal/WPB-8709
deleted file mode 100644
index 7c644d2437b..00000000000
--- a/changelog.d/5-internal/WPB-8709
+++ /dev/null
@@ -1 +0,0 @@
-phone data is removed from DB

From bd4bed45d2337f7f24a09bf58dd4316e893fb442 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 10:06:45 +0000
Subject: [PATCH 06/13] moved to tools/db, renamed to remove-phone-keys

---
 cabal.project                                 |  2 +-
 nix/local-haskell-packages.nix                |  2 +-
 .../remove-phone-keys}/.ormolu                |  0
 tools/db/remove-phone-keys/README.md          | 20 +++++++++++++++++++
 .../remove-phone-keys}/app/Main.hs            |  2 +-
 .../remove-phone-keys}/default.nix            |  4 ++--
 .../remove-phone-keys.cabal}                  | 12 +++++------
 .../src/RemovePhoneKeys}/Lib.hs               |  8 ++++----
 .../src/RemovePhoneKeys}/Phone.hs             |  2 +-
 .../src/RemovePhoneKeys}/Types.hs             |  2 +-
 tools/phone-data-migration/README.md          |  1 -
 11 files changed, 37 insertions(+), 18 deletions(-)
 rename tools/{phone-data-migration => db/remove-phone-keys}/.ormolu (100%)
 create mode 100644 tools/db/remove-phone-keys/README.md
 rename tools/{phone-data-migration => db/remove-phone-keys}/app/Main.hs (94%)
 rename tools/{phone-data-migration => db/remove-phone-keys}/default.nix (91%)
 rename tools/{phone-data-migration/phone-data-migration.cabal => db/remove-phone-keys/remove-phone-keys.cabal} (91%)
 rename tools/{phone-data-migration/src/PhoneDataMigration => db/remove-phone-keys/src/RemovePhoneKeys}/Lib.hs (93%)
 rename tools/{phone-data-migration/src/PhoneDataMigration => db/remove-phone-keys/src/RemovePhoneKeys}/Phone.hs (92%)
 rename tools/{phone-data-migration/src/PhoneDataMigration => db/remove-phone-keys/src/RemovePhoneKeys}/Types.hs (98%)
 delete mode 100644 tools/phone-data-migration/README.md

diff --git a/cabal.project b/cabal.project
index 3c0a1bf2993..c21b4f14834 100644
--- a/cabal.project
+++ b/cabal.project
@@ -48,11 +48,11 @@ packages:
   , tools/db/migrate-sso-feature-flag/
   , tools/db/move-team/
   , tools/db/phone-users/
+  , tools/db/remove-phone-keys
   , tools/db/repair-handles/
   , tools/db/repair-brig-clients-table/
   , tools/db/service-backfill/
   , tools/fedcalls/
-  , tools/phone-data-migration
   , tools/rabbitmq-consumer
   , tools/rex/
   , tools/stern/
diff --git a/nix/local-haskell-packages.nix b/nix/local-haskell-packages.nix
index 14013d63bbf..634b705b44e 100644
--- a/nix/local-haskell-packages.nix
+++ b/nix/local-haskell-packages.nix
@@ -49,12 +49,12 @@
   migrate-sso-feature-flag = hself.callPackage ../tools/db/migrate-sso-feature-flag/default.nix { inherit gitignoreSource; };
   move-team = hself.callPackage ../tools/db/move-team/default.nix { inherit gitignoreSource; };
   phone-users = hself.callPackage ../tools/db/phone-users/default.nix { inherit gitignoreSource; };
+  remove-phone-keys = hself.callPackage ../tools/db/remove-phone-keys/default.nix { inherit gitignoreSource; };
   repair-brig-clients-table = hself.callPackage ../tools/db/repair-brig-clients-table/default.nix { inherit gitignoreSource; };
   repair-handles = hself.callPackage ../tools/db/repair-handles/default.nix { inherit gitignoreSource; };
   service-backfill = hself.callPackage ../tools/db/service-backfill/default.nix { inherit gitignoreSource; };
   fedcalls = hself.callPackage ../tools/fedcalls/default.nix { inherit gitignoreSource; };
   mlsstats = hself.callPackage ../tools/mlsstats/default.nix { inherit gitignoreSource; };
-  phone-data-migration = hself.callPackage ../tools/phone-data-migration/default.nix { inherit gitignoreSource; };
   rabbitmq-consumer = hself.callPackage ../tools/rabbitmq-consumer/default.nix { inherit gitignoreSource; };
   rex = hself.callPackage ../tools/rex/default.nix { inherit gitignoreSource; };
   stern = hself.callPackage ../tools/stern/default.nix { inherit gitignoreSource; };
diff --git a/tools/phone-data-migration/.ormolu b/tools/db/remove-phone-keys/.ormolu
similarity index 100%
rename from tools/phone-data-migration/.ormolu
rename to tools/db/remove-phone-keys/.ormolu
diff --git a/tools/db/remove-phone-keys/README.md b/tools/db/remove-phone-keys/README.md
new file mode 100644
index 00000000000..762702dc5cd
--- /dev/null
+++ b/tools/db/remove-phone-keys/README.md
@@ -0,0 +1,20 @@
+# Remove phone keys
+
+```text
+remove-phone-keys
+
+Usage: remove-phone-keys [--brig-cassandra-host HOST]
+                         [--brig-cassandra-port PORT]
+                         [--brig-cassandra-keyspace STRING]
+
+  Remove phone data from wire-server
+
+Available options:
+  -h,--help                Show this help text
+  --brig-cassandra-host HOST
+                           Cassandra Host for brig (default: "localhost")
+  --brig-cassandra-port PORT
+                           Cassandra Port for brig (default: 9042)
+  --brig-cassandra-keyspace STRING
+                           Cassandra Keyspace for brig (default: "brig_test")
+```
diff --git a/tools/phone-data-migration/app/Main.hs b/tools/db/remove-phone-keys/app/Main.hs
similarity index 94%
rename from tools/phone-data-migration/app/Main.hs
rename to tools/db/remove-phone-keys/app/Main.hs
index 6ba1f3d6873..143a06bae91 100644
--- a/tools/phone-data-migration/app/Main.hs
+++ b/tools/db/remove-phone-keys/app/Main.hs
@@ -17,7 +17,7 @@
 
 module Main where
 
-import qualified PhoneDataMigration.Lib as Lib
+import qualified RemovePhoneKeys.Lib as Lib
 
 main :: IO ()
 main = Lib.main
diff --git a/tools/phone-data-migration/default.nix b/tools/db/remove-phone-keys/default.nix
similarity index 91%
rename from tools/phone-data-migration/default.nix
rename to tools/db/remove-phone-keys/default.nix
index fdada762120..33d4a636df4 100644
--- a/tools/phone-data-migration/default.nix
+++ b/tools/db/remove-phone-keys/default.nix
@@ -17,7 +17,7 @@
 , tinylog
 }:
 mkDerivation {
-  pname = "phone-data-migration";
+  pname = "remove-phone-keys";
   version = "1.0.0";
   src = gitignoreSource ./.;
   isLibrary = true;
@@ -36,5 +36,5 @@ mkDerivation {
   executableHaskellDepends = [ base ];
   description = "remove phone data from wire-server";
   license = lib.licenses.agpl3Only;
-  mainProgram = "phone-data-migration";
+  mainProgram = "remove-phone-keys";
 }
diff --git a/tools/phone-data-migration/phone-data-migration.cabal b/tools/db/remove-phone-keys/remove-phone-keys.cabal
similarity index 91%
rename from tools/phone-data-migration/phone-data-migration.cabal
rename to tools/db/remove-phone-keys/remove-phone-keys.cabal
index daeee0211c5..c2a6203d36e 100644
--- a/tools/phone-data-migration/phone-data-migration.cabal
+++ b/tools/db/remove-phone-keys/remove-phone-keys.cabal
@@ -1,5 +1,5 @@
 cabal-version: 3.0
-name:          phone-data-migration
+name:          remove-phone-keys
 version:       1.0.0
 synopsis:      remove phone data from wire-server
 category:      Network
@@ -12,9 +12,9 @@ build-type:    Simple
 library
   hs-source-dirs:     src
   exposed-modules:
-    PhoneDataMigration.Lib
-    PhoneDataMigration.Phone
-    PhoneDataMigration.Types
+    RemovePhoneKeys.Lib
+    RemovePhoneKeys.Phone
+    RemovePhoneKeys.Types
 
   ghc-options:
     -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates
@@ -78,11 +78,11 @@ library
     UndecidableInstances
     ViewPatterns
 
-executable phone-data-migration
+executable remove-phone-keys
   main-is:          Main.hs
   build-depends:
     , base
-    , phone-data-migration
+    , remove-phone-keys
 
   hs-source-dirs:   app
   default-language: Haskell2010
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Lib.hs
similarity index 93%
rename from tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
rename to tools/db/remove-phone-keys/src/RemovePhoneKeys/Lib.hs
index bb3e2306b6f..94096662de2 100644
--- a/tools/phone-data-migration/src/PhoneDataMigration/Lib.hs
+++ b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Lib.hs
@@ -17,7 +17,7 @@
 -- You should have received a copy of the GNU Affero General Public License along
 -- with this program. If not, see <https://www.gnu.org/licenses/>.
 
-module PhoneDataMigration.Lib where
+module RemovePhoneKeys.Lib where
 
 import Cassandra as C
 import Data.Conduit
@@ -25,8 +25,8 @@ import qualified Data.Conduit.Combinators as Conduit
 import qualified Data.Conduit.List as ConduitList
 import Imports
 import Options.Applicative
-import PhoneDataMigration.Phone
-import PhoneDataMigration.Types
+import RemovePhoneKeys.Phone
+import RemovePhoneKeys.Types
 import System.Logger.Class (MonadLogger)
 import qualified System.Logger.Class as Log
 import System.Logger.Message ((.=))
@@ -82,4 +82,4 @@ main = do
   env <- mkEnv opts
   runReaderT (unAppT run) env
   where
-    desc = header "phone-data-migration" <> progDesc "Remove phone data from wire-server" <> fullDesc
+    desc = header "remove-phone-keys" <> progDesc "Remove phone data from wire-server" <> fullDesc
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Phone.hs
similarity index 92%
rename from tools/phone-data-migration/src/PhoneDataMigration/Phone.hs
rename to tools/db/remove-phone-keys/src/RemovePhoneKeys/Phone.hs
index f0e82fc1da4..cebe74fb86c 100644
--- a/tools/phone-data-migration/src/PhoneDataMigration/Phone.hs
+++ b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Phone.hs
@@ -1,4 +1,4 @@
-module PhoneDataMigration.Phone (Phone, parsePhone, toText) where
+module RemovePhoneKeys.Phone (Phone, parsePhone, toText) where
 
 import Data.Attoparsec.Text (char, count, digit, endOfInput, parseOnly)
 import qualified Data.Text as Text
diff --git a/tools/phone-data-migration/src/PhoneDataMigration/Types.hs b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Types.hs
similarity index 98%
rename from tools/phone-data-migration/src/PhoneDataMigration/Types.hs
rename to tools/db/remove-phone-keys/src/RemovePhoneKeys/Types.hs
index 3e9dd1edf3a..e8d9f42026d 100644
--- a/tools/phone-data-migration/src/PhoneDataMigration/Types.hs
+++ b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Types.hs
@@ -15,7 +15,7 @@
 -- You should have received a copy of the GNU Affero General Public License along
 -- with this program. If not, see <https://www.gnu.org/licenses/>.
 
-module PhoneDataMigration.Types where
+module RemovePhoneKeys.Types where
 
 import Cassandra as C
 import qualified Cassandra.Settings as C
diff --git a/tools/phone-data-migration/README.md b/tools/phone-data-migration/README.md
deleted file mode 100644
index 6f4a9f26e4e..00000000000
--- a/tools/phone-data-migration/README.md
+++ /dev/null
@@ -1 +0,0 @@
-# Phone data migration

From c37dbd030e7467df2f3ce8e0b822bd9b4db059c8 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 11:05:02 +0000
Subject: [PATCH 07/13] removed phone from inconsistencies tool

---
 tools/db/inconsistencies/src/DanglingUserKeys.hs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tools/db/inconsistencies/src/DanglingUserKeys.hs b/tools/db/inconsistencies/src/DanglingUserKeys.hs
index 3d27d4c208c..1242a1a2972 100644
--- a/tools/db/inconsistencies/src/DanglingUserKeys.hs
+++ b/tools/db/inconsistencies/src/DanglingUserKeys.hs
@@ -82,7 +82,6 @@ data Inconsistency = Inconsistency
     time :: Writetime UserId,
     status :: Maybe (WithWritetime AccountStatus),
     userEmail :: Maybe (WithWritetime Email),
-    userPhone :: Maybe (WithWritetime Phone),
     inconsistencyCase :: Text
   }
   deriving (Generic)
@@ -130,13 +129,13 @@ instance Cql EmailKey where
 instance Aeson.ToJSON EmailKey where
   toJSON = Aeson.toJSON . emailKeyUniq
 
-type UserDetailsRow = (Maybe AccountStatus, Maybe (Writetime AccountStatus), Maybe Email, Maybe (Writetime Email), Maybe Phone, Maybe (Writetime Phone))
+type UserDetailsRow = (Maybe AccountStatus, Maybe (Writetime AccountStatus), Maybe Email, Maybe (Writetime Email))
 
 getUserDetails :: UserId -> Client (Maybe UserDetailsRow)
 getUserDetails uid = retry x5 $ query1 cql (params LocalQuorum (Identity uid))
   where
     cql :: PrepQuery R (Identity UserId) UserDetailsRow
-    cql = "SELECT status, writetime(status), email, writetime(email), phone, writetime(phone) from user where id = ?"
+    cql = "SELECT status, writetime(status), email, writetime(email) from user where id = ?"
 
 checkKey :: Logger -> ClientState -> EmailKey -> Bool -> IO (Maybe Inconsistency)
 checkKey l brig key repairData = do
@@ -179,16 +178,14 @@ checkUser l brig key uid time repairData = do
     Nothing -> do
       let status = Nothing
           userEmail = Nothing
-          userPhone = Nothing
           inconsistencyCase = "2."
       when repairData $ -- case 2.
         runClient brig $
           freeEmailKey l key
       pure . Just $ Inconsistency {userId = uid, ..}
-    Just (mStatus, mStatusWriteTime, mEmail, mEmailWriteTime, mPhone, mPhoneWriteTime) -> do
+    Just (mStatus, mStatusWriteTime, mEmail, mEmailWriteTime) -> do
       let status = WithWritetime <$> mStatus <*> mStatusWriteTime
           userEmail = WithWritetime <$> mEmail <*> mEmailWriteTime
-          userPhone = WithWritetime <$> mPhone <*> mPhoneWriteTime
           statusError = case mStatus of
             Nothing -> True
             Just Deleted -> True

From 7f958f7620ac13c808c3ed7250d0f50bf66160ab Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 11:20:31 +0000
Subject: [PATCH 08/13] remove phone from move-team tool

---
 tools/db/move-team/src/Schema.hs |  8 ++++----
 tools/db/move-team/src/Types.hs  | 14 ++++++--------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/tools/db/move-team/src/Schema.hs b/tools/db/move-team/src/Schema.hs
index fdf17e81cae..2249bb2f1a5 100644
--- a/tools/db/move-team/src/Schema.hs
+++ b/tools/db/move-team/src/Schema.hs
@@ -360,10 +360,10 @@ importBrigRichInfo Env {..} path = do
 
 -- brig.user
 
-type RowBrigUser = (Maybe UUID, Maybe [Float], Maybe Int32, Maybe Bool, Maybe [AssetIgnoreData], Maybe Ascii, Maybe Text, Maybe UTCTime, Maybe Text, Maybe Ascii, Maybe Int32, Maybe Text, Maybe Blob, Maybe Text, Maybe [Blob], Maybe UUID, Maybe Bool, Maybe UUID, Maybe Text, Maybe Int32, Maybe UUID)
+type RowBrigUser = (Maybe UUID, Maybe [Float], Maybe Int32, Maybe Bool, Maybe [AssetIgnoreData], Maybe Ascii, Maybe Text, Maybe UTCTime, Maybe Text, Maybe Ascii, Maybe Int32, Maybe Text, Maybe Blob, Maybe [Blob], Maybe UUID, Maybe Bool, Maybe UUID, Maybe Text, Maybe Int32, Maybe UUID)
 
 selectBrigUser :: PrepQuery R (Identity [UserId]) RowBrigUser
-selectBrigUser = "SELECT id, accent, accent_id, activated, assets, country, email, expires, handle, language, managed_by, name, password, phone, picture, provider, searchable, service, sso_id, status, team FROM user WHERE id in ?"
+selectBrigUser = "SELECT id, accent, accent_id, activated, assets, country, email, expires, handle, language, managed_by, name, password, picture, provider, searchable, service, sso_id, status, team FROM user WHERE id in ?"
 
 readBrigUser :: Env -> [UserId] -> ConduitM () [RowBrigUser] IO ()
 readBrigUser Env {..} uids =
@@ -371,7 +371,7 @@ readBrigUser Env {..} uids =
     paginateC selectBrigUser (paramsP LocalQuorum (pure uids) envPageSize) x5
 
 selectBrigUserAll :: PrepQuery R () RowBrigUser
-selectBrigUserAll = "SELECT id, accent, accent_id, activated, assets, country, email, expires, handle, language, managed_by, name, password, phone, picture, provider, searchable, service, sso_id, status, team FROM user"
+selectBrigUserAll = "SELECT id, accent, accent_id, activated, assets, country, email, expires, handle, language, managed_by, name, password, picture, provider, searchable, service, sso_id, status, team FROM user"
 
 readBrigUserAll :: Env -> ConduitM () [RowBrigUser] IO ()
 readBrigUserAll Env {..} =
@@ -388,7 +388,7 @@ exportBrigUserFull env path = do
 
 insertBrigUser :: PrepQuery W RowBrigUser ()
 insertBrigUser =
-  "INSERT INTO user (id, accent, accent_id, activated, assets, country, email, expires, handle, language, managed_by, name, password, phone, picture, provider, searchable, service, sso_id, status, team) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+  "INSERT INTO user (id, accent, accent_id, activated, assets, country, email, expires, handle, language, managed_by, name, password, picture, provider, searchable, service, sso_id, status, team) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
 
 importBrigUser :: Env -> FilePath -> IO ()
 importBrigUser Env {..} path = do
diff --git a/tools/db/move-team/src/Types.hs b/tools/db/move-team/src/Types.hs
index 3f7ef7ebe36..2839331ccb1 100644
--- a/tools/db/move-team/src/Types.hs
+++ b/tools/db/move-team/src/Types.hs
@@ -104,8 +104,8 @@ deriving instance ToJSON TimeUuid
 
 deriving instance FromJSON TimeUuid
 
-instance (ToJSON a, ToJSON b, ToJSON c, ToJSON d, ToJSON e, ToJSON f, ToJSON g, ToJSON h, ToJSON i, ToJSON j, ToJSON k, ToJSON l, ToJSON m, ToJSON n, ToJSON o, ToJSON p, ToJSON q, ToJSON r, ToJSON s, ToJSON t, ToJSON u) => ToJSON ((,,,,,,,,,,,,,,,,,,,,) a b c d e f g h i j k l m n o p q r s t u) where
-  toJSON (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u) =
+instance (ToJSON a, ToJSON b, ToJSON c, ToJSON d, ToJSON e, ToJSON f, ToJSON g, ToJSON h, ToJSON i, ToJSON j, ToJSON k, ToJSON l, ToJSON m, ToJSON n, ToJSON o, ToJSON p, ToJSON q, ToJSON r, ToJSON s, ToJSON t) => ToJSON ((,,,,,,,,,,,,,,,,,,,) a b c d e f g h i j k l m n o p q r s t) where
+  toJSON (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t) =
     Array $
       V.fromList
         [ toJSON a,
@@ -127,14 +127,13 @@ instance (ToJSON a, ToJSON b, ToJSON c, ToJSON d, ToJSON e, ToJSON f, ToJSON g,
           toJSON q,
           toJSON r,
           toJSON s,
-          toJSON t,
-          toJSON u
+          toJSON t
         ]
 
-instance (FromJSON a, FromJSON b, FromJSON c, FromJSON d, FromJSON e, FromJSON f, FromJSON g, FromJSON h, FromJSON i, FromJSON j, FromJSON k, FromJSON l, FromJSON m, FromJSON n, FromJSON o, FromJSON p, FromJSON q, FromJSON r, FromJSON s, FromJSON t, FromJSON u) => FromJSON ((,,,,,,,,,,,,,,,,,,,,) a b c d e f g h i j k l m n o p q r s t u) where
+instance (FromJSON a, FromJSON b, FromJSON c, FromJSON d, FromJSON e, FromJSON f, FromJSON g, FromJSON h, FromJSON i, FromJSON j, FromJSON k, FromJSON l, FromJSON m, FromJSON n, FromJSON o, FromJSON p, FromJSON q, FromJSON r, FromJSON s, FromJSON t) => FromJSON ((,,,,,,,,,,,,,,,,,,,) a b c d e f g h i j k l m n o p q r s t) where
   parseJSON = withArray "Tuple" $ \case
-    (toList -> [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u]) ->
-      (,,,,,,,,,,,,,,,,,,,,)
+    (toList -> [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t]) ->
+      (,,,,,,,,,,,,,,,,,,,)
         <$> parseJSON a
         <*> parseJSON b
         <*> parseJSON c
@@ -155,5 +154,4 @@ instance (FromJSON a, FromJSON b, FromJSON c, FromJSON d, FromJSON e, FromJSON f
         <*> parseJSON r
         <*> parseJSON s
         <*> parseJSON t
-        <*> parseJSON u
     _ -> fail "Expected array of length 21"

From ab132768d20296039e3144b3182cf7a1a90bd272 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Fri, 12 Jul 2024 12:21:52 +0000
Subject: [PATCH 09/13] remove phone from queries

---
 libs/wire-subsystems/src/Wire/StoredUser.hs          | 1 -
 libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/libs/wire-subsystems/src/Wire/StoredUser.hs b/libs/wire-subsystems/src/Wire/StoredUser.hs
index b2ace0784cb..6a716ba04d6 100644
--- a/libs/wire-subsystems/src/Wire/StoredUser.hs
+++ b/libs/wire-subsystems/src/Wire/StoredUser.hs
@@ -21,7 +21,6 @@ data StoredUser = StoredUser
     name :: Name,
     pict :: Maybe Pict,
     email :: Maybe Email,
-    phone :: Maybe Phone,
     ssoId :: Maybe UserSSOId,
     accentId :: ColourId,
     assets :: Maybe [Asset],
diff --git a/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs b/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs
index cba7356f22e..b6662fbac63 100644
--- a/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs
+++ b/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs
@@ -127,7 +127,7 @@ lookupLocaleImpl u = do
 
 selectUser :: PrepQuery R (Identity UserId) (TupleType StoredUser)
 selectUser =
-  "SELECT id, name, picture, email, phone, sso_id, accent_id, assets, \
+  "SELECT id, name, picture, email, sso_id, accent_id, assets, \
   \activated, status, expires, language, country, provider, service, \
   \handle, team, managed_by, supported_protocols \
   \FROM user where id = ?"
@@ -166,7 +166,7 @@ updateUserToTombstone :: PrepQuery W (AccountStatus, Name, ColourId, Pict, [Asse
 updateUserToTombstone =
   "UPDATE user SET status = ?, name = ?,\
   \ accent_id = ?, picture = ?, assets = ?, handle = null, country = null,\
-  \ language = null, email = null, phone = null, sso_id = null WHERE id = ?"
+  \ language = null, email = null, sso_id = null WHERE id = ?"
 
 statusSelect :: PrepQuery R (Identity UserId) (Identity (Maybe AccountStatus))
 statusSelect = "SELECT status FROM user WHERE id = ?"

From 96c15f09e891f5b5dc1565b908610a951441f5a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Dimja=C5=A1evi=C4=87?= <marko.dimjasevic@wire.com>
Date: Mon, 15 Jul 2024 10:57:03 +0200
Subject: [PATCH 10/13] Better split and organize the changelog

---
 changelog.d/0-release-notes/WPB-10058     | 3 +++
 changelog.d/0-release-notes/WPB-10058-5xx | 4 ++++
 changelog.d/2-features/WPB-10058          | 1 +
 changelog.d/5-internal/WPB-10058          | 2 --
 4 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/0-release-notes/WPB-10058
 create mode 100644 changelog.d/0-release-notes/WPB-10058-5xx
 create mode 100644 changelog.d/2-features/WPB-10058
 delete mode 100644 changelog.d/5-internal/WPB-10058

diff --git a/changelog.d/0-release-notes/WPB-10058 b/changelog.d/0-release-notes/WPB-10058
new file mode 100644
index 00000000000..dddc28a74e3
--- /dev/null
+++ b/changelog.d/0-release-notes/WPB-10058
@@ -0,0 +1,3 @@
+Run a tool for deleting phone keys from the `user_keys` table in Brig. It can be
+found in `tools/db/remove-phone-keys`. See the accompanying README for more
+details.
diff --git a/changelog.d/0-release-notes/WPB-10058-5xx b/changelog.d/0-release-notes/WPB-10058-5xx
new file mode 100644
index 00000000000..e884d0434be
--- /dev/null
+++ b/changelog.d/0-release-notes/WPB-10058-5xx
@@ -0,0 +1,4 @@
+Because the `phone` column is deleted from Brig's `user` table in a schema
+migration, temporarily there might be 5xx errors during deployment if Wire
+server 5.4.0 was not deployed previously. To avoid these errors, please deploy
+the Wire server 5.4.0 release first.
diff --git a/changelog.d/2-features/WPB-10058 b/changelog.d/2-features/WPB-10058
new file mode 100644
index 00000000000..02fab832d8c
--- /dev/null
+++ b/changelog.d/2-features/WPB-10058
@@ -0,0 +1 @@
+DB migration for dropping `phone` column from `user` table
diff --git a/changelog.d/5-internal/WPB-10058 b/changelog.d/5-internal/WPB-10058
deleted file mode 100644
index 1cdda618893..00000000000
--- a/changelog.d/5-internal/WPB-10058
+++ /dev/null
@@ -1,2 +0,0 @@
-- DB migration for dropping `phone` column from `user` table
-- Tool for deleting phone keys from `user_keys`

From 370f8ebfbc6749aef734d4b9800796f26a0a4e2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Dimja=C5=A1evi=C4=87?= <marko.dimjasevic@wire.com>
Date: Mon, 15 Jul 2024 11:18:29 +0200
Subject: [PATCH 11/13] Update the README of remove-phone-keys tool

---
 tools/db/remove-phone-keys/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/db/remove-phone-keys/README.md b/tools/db/remove-phone-keys/README.md
index 762702dc5cd..2214a51a1a7 100644
--- a/tools/db/remove-phone-keys/README.md
+++ b/tools/db/remove-phone-keys/README.md
@@ -1,5 +1,8 @@
 # Remove phone keys
 
+This tool removes the phone keys from the `user_keys` table. Assuming `wire-server` is up and running, run the tool as laid out below.
+
+
 ```text
 remove-phone-keys
 

From f7e72fad2ab2f72476cbd8e39c3a69dbf3eaff75 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Mon, 15 Jul 2024 09:30:34 +0000
Subject: [PATCH 12/13] remove db tool

---
 cabal.project                                 |   1 -
 nix/local-haskell-packages.nix                |   1 -
 tools/db/remove-phone-keys/.ormolu            |   1 -
 tools/db/remove-phone-keys/README.md          |  23 ---
 tools/db/remove-phone-keys/app/Main.hs        |  23 ---
 tools/db/remove-phone-keys/default.nix        |  40 ------
 .../remove-phone-keys/remove-phone-keys.cabal |  93 -------------
 .../src/RemovePhoneKeys/Lib.hs                |  85 ------------
 .../src/RemovePhoneKeys/Phone.hs              |  27 ----
 .../src/RemovePhoneKeys/Types.hs              | 131 ------------------
 10 files changed, 425 deletions(-)
 delete mode 120000 tools/db/remove-phone-keys/.ormolu
 delete mode 100644 tools/db/remove-phone-keys/README.md
 delete mode 100644 tools/db/remove-phone-keys/app/Main.hs
 delete mode 100644 tools/db/remove-phone-keys/default.nix
 delete mode 100644 tools/db/remove-phone-keys/remove-phone-keys.cabal
 delete mode 100644 tools/db/remove-phone-keys/src/RemovePhoneKeys/Lib.hs
 delete mode 100644 tools/db/remove-phone-keys/src/RemovePhoneKeys/Phone.hs
 delete mode 100644 tools/db/remove-phone-keys/src/RemovePhoneKeys/Types.hs

diff --git a/cabal.project b/cabal.project
index c21b4f14834..fe2c42af262 100644
--- a/cabal.project
+++ b/cabal.project
@@ -48,7 +48,6 @@ packages:
   , tools/db/migrate-sso-feature-flag/
   , tools/db/move-team/
   , tools/db/phone-users/
-  , tools/db/remove-phone-keys
   , tools/db/repair-handles/
   , tools/db/repair-brig-clients-table/
   , tools/db/service-backfill/
diff --git a/nix/local-haskell-packages.nix b/nix/local-haskell-packages.nix
index 634b705b44e..133fcd9afae 100644
--- a/nix/local-haskell-packages.nix
+++ b/nix/local-haskell-packages.nix
@@ -49,7 +49,6 @@
   migrate-sso-feature-flag = hself.callPackage ../tools/db/migrate-sso-feature-flag/default.nix { inherit gitignoreSource; };
   move-team = hself.callPackage ../tools/db/move-team/default.nix { inherit gitignoreSource; };
   phone-users = hself.callPackage ../tools/db/phone-users/default.nix { inherit gitignoreSource; };
-  remove-phone-keys = hself.callPackage ../tools/db/remove-phone-keys/default.nix { inherit gitignoreSource; };
   repair-brig-clients-table = hself.callPackage ../tools/db/repair-brig-clients-table/default.nix { inherit gitignoreSource; };
   repair-handles = hself.callPackage ../tools/db/repair-handles/default.nix { inherit gitignoreSource; };
   service-backfill = hself.callPackage ../tools/db/service-backfill/default.nix { inherit gitignoreSource; };
diff --git a/tools/db/remove-phone-keys/.ormolu b/tools/db/remove-phone-keys/.ormolu
deleted file mode 120000
index ffc2ca9745e..00000000000
--- a/tools/db/remove-phone-keys/.ormolu
+++ /dev/null
@@ -1 +0,0 @@
-../../../.ormolu
\ No newline at end of file
diff --git a/tools/db/remove-phone-keys/README.md b/tools/db/remove-phone-keys/README.md
deleted file mode 100644
index 2214a51a1a7..00000000000
--- a/tools/db/remove-phone-keys/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-# Remove phone keys
-
-This tool removes the phone keys from the `user_keys` table. Assuming `wire-server` is up and running, run the tool as laid out below.
-
-
-```text
-remove-phone-keys
-
-Usage: remove-phone-keys [--brig-cassandra-host HOST]
-                         [--brig-cassandra-port PORT]
-                         [--brig-cassandra-keyspace STRING]
-
-  Remove phone data from wire-server
-
-Available options:
-  -h,--help                Show this help text
-  --brig-cassandra-host HOST
-                           Cassandra Host for brig (default: "localhost")
-  --brig-cassandra-port PORT
-                           Cassandra Port for brig (default: 9042)
-  --brig-cassandra-keyspace STRING
-                           Cassandra Keyspace for brig (default: "brig_test")
-```
diff --git a/tools/db/remove-phone-keys/app/Main.hs b/tools/db/remove-phone-keys/app/Main.hs
deleted file mode 100644
index 143a06bae91..00000000000
--- a/tools/db/remove-phone-keys/app/Main.hs
+++ /dev/null
@@ -1,23 +0,0 @@
--- This file is part of the Wire Server implementation.
---
--- Copyright (C) 2024 Wire Swiss GmbH <opensource@wire.com>
---
--- This program is free software: you can redistribute it and/or modify it under
--- the terms of the GNU Affero General Public License as published by the Free
--- Software Foundation, either version 3 of the License, or (at your option) any
--- later version.
---
--- This program is distributed in the hope that it will be useful, but WITHOUT
--- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
--- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
--- details.
---
--- You should have received a copy of the GNU Affero General Public License along
--- with this program. If not, see <https://www.gnu.org/licenses/>.
-
-module Main where
-
-import qualified RemovePhoneKeys.Lib as Lib
-
-main :: IO ()
-main = Lib.main
diff --git a/tools/db/remove-phone-keys/default.nix b/tools/db/remove-phone-keys/default.nix
deleted file mode 100644
index 33d4a636df4..00000000000
--- a/tools/db/remove-phone-keys/default.nix
+++ /dev/null
@@ -1,40 +0,0 @@
-# WARNING: GENERATED FILE, DO NOT EDIT.
-# This file is generated by running hack/bin/generate-local-nix-packages.sh and
-# must be regenerated whenever local packages are added or removed, or
-# dependencies are added or removed.
-{ mkDerivation
-, attoparsec
-, base
-, cassandra-util
-, conduit
-, exceptions
-, gitignoreSource
-, imports
-, lens
-, lib
-, optparse-applicative
-, text
-, tinylog
-}:
-mkDerivation {
-  pname = "remove-phone-keys";
-  version = "1.0.0";
-  src = gitignoreSource ./.;
-  isLibrary = true;
-  isExecutable = true;
-  libraryHaskellDepends = [
-    attoparsec
-    cassandra-util
-    conduit
-    exceptions
-    imports
-    lens
-    optparse-applicative
-    text
-    tinylog
-  ];
-  executableHaskellDepends = [ base ];
-  description = "remove phone data from wire-server";
-  license = lib.licenses.agpl3Only;
-  mainProgram = "remove-phone-keys";
-}
diff --git a/tools/db/remove-phone-keys/remove-phone-keys.cabal b/tools/db/remove-phone-keys/remove-phone-keys.cabal
deleted file mode 100644
index c2a6203d36e..00000000000
--- a/tools/db/remove-phone-keys/remove-phone-keys.cabal
+++ /dev/null
@@ -1,93 +0,0 @@
-cabal-version: 3.0
-name:          remove-phone-keys
-version:       1.0.0
-synopsis:      remove phone data from wire-server
-category:      Network
-author:        Wire Swiss GmbH
-maintainer:    Wire Swiss GmbH <backend@wire.com>
-copyright:     (c) 2024 Wire Swiss GmbH
-license:       AGPL-3.0-only
-build-type:    Simple
-
-library
-  hs-source-dirs:     src
-  exposed-modules:
-    RemovePhoneKeys.Lib
-    RemovePhoneKeys.Phone
-    RemovePhoneKeys.Types
-
-  ghc-options:
-    -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates
-    -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path
-    -funbox-strict-fields -threaded -with-rtsopts=-N
-    -Wredundant-constraints -Wunused-packages
-
-  build-depends:
-    , attoparsec
-    , cassandra-util
-    , conduit
-    , exceptions
-    , imports
-    , lens
-    , optparse-applicative
-    , text
-    , tinylog
-
-  default-extensions:
-    AllowAmbiguousTypes
-    BangPatterns
-    ConstraintKinds
-    DataKinds
-    DefaultSignatures
-    DeriveFunctor
-    DeriveGeneric
-    DeriveLift
-    DeriveTraversable
-    DerivingStrategies
-    DerivingVia
-    DuplicateRecordFields
-    EmptyCase
-    FlexibleContexts
-    FlexibleInstances
-    FunctionalDependencies
-    GADTs
-    GeneralizedNewtypeDeriving
-    InstanceSigs
-    KindSignatures
-    LambdaCase
-    MultiParamTypeClasses
-    MultiWayIf
-    NamedFieldPuns
-    NoImplicitPrelude
-    OverloadedLabels
-    OverloadedRecordDot
-    OverloadedStrings
-    PackageImports
-    PatternSynonyms
-    PolyKinds
-    QuasiQuotes
-    RankNTypes
-    RecordWildCards
-    ScopedTypeVariables
-    StandaloneDeriving
-    TupleSections
-    TypeApplications
-    TypeFamilies
-    TypeFamilyDependencies
-    TypeOperators
-    UndecidableInstances
-    ViewPatterns
-
-executable remove-phone-keys
-  main-is:          Main.hs
-  build-depends:
-    , base
-    , remove-phone-keys
-
-  hs-source-dirs:   app
-  default-language: Haskell2010
-  ghc-options:
-    -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates
-    -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path
-    -funbox-strict-fields -threaded -with-rtsopts=-N
-    -Wredundant-constraints -Wunused-packages
diff --git a/tools/db/remove-phone-keys/src/RemovePhoneKeys/Lib.hs b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Lib.hs
deleted file mode 100644
index 94096662de2..00000000000
--- a/tools/db/remove-phone-keys/src/RemovePhoneKeys/Lib.hs
+++ /dev/null
@@ -1,85 +0,0 @@
-{-# LANGUAGE OverloadedStrings #-}
-
--- This file is part of the Wire Server implementation.
---
--- Copyright (C) 2024 Wire Swiss GmbH <opensource@wire.com>
---
--- This program is free software: you can redistribute it and/or modify it under
--- the terms of the GNU Affero General Public License as published by the Free
--- Software Foundation, either version 3 of the License, or (at your option) any
--- later version.
---
--- This program is distributed in the hope that it will be useful, but WITHOUT
--- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
--- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
--- details.
---
--- You should have received a copy of the GNU Affero General Public License along
--- with this program. If not, see <https://www.gnu.org/licenses/>.
-
-module RemovePhoneKeys.Lib where
-
-import Cassandra as C
-import Data.Conduit
-import qualified Data.Conduit.Combinators as Conduit
-import qualified Data.Conduit.List as ConduitList
-import Imports
-import Options.Applicative
-import RemovePhoneKeys.Phone
-import RemovePhoneKeys.Types
-import System.Logger.Class (MonadLogger)
-import qualified System.Logger.Class as Log
-import System.Logger.Message ((.=))
-
-pageSize :: Int32
-pageSize = 1000
-
-getKeys :: forall m. (MonadClient m, MonadLogger m) => IORef Int -> ConduitM () Phone m ()
-getKeys counter =
-  paginateC cql (paramsP LocalQuorum () pageSize) x5
-    .| Conduit.concatMap (mapMaybe (parsePhone . runIdentity))
-    .| Conduit.iterM (const $ logEvery 100000 counter)
-  where
-    cql :: PrepQuery R () (Identity Text)
-    cql = "SELECT key FROM user_keys"
-
-    logEvery :: Int -> IORef Int -> m ()
-    logEvery i ctr = do
-      c <- liftIO $ atomicModifyIORef' ctr (\x -> (x + 1, x + 1))
-      when (c `mod` i == 0) $ Log.info $ "found_phone_keys" .= show c
-
-deleteKey :: (MonadClient m) => Phone -> m IntSum
-deleteKey p = do
-  retry x5 $ write cql (params LocalQuorum (Identity $ toText p))
-  pure 1
-  where
-    cql :: PrepQuery W (Identity Text) ()
-    cql = "DELETE FROM user_keys WHERE key = ?"
-
-deleteKeys :: [Phone] -> ConduitT a IntSum (AppT IO) ()
-deleteKeys keys = do
-  ConduitList.sourceList keys
-    .| Conduit.mapM deleteKey
-    .| Conduit.scanl (<>) mempty
-    .| Conduit.iterM (logEvery 100000)
-  where
-    logEvery :: Int -> IntSum -> AppT IO ()
-    logEvery i s =
-      when (unIntSum s `mod` i == 0) $ Log.info $ "deleted_keys" .= show s
-
-run :: AppT IO ()
-run = do
-  ctr <- liftIO $ newIORef 0
-  -- deleting from user_keys while paginating through it is probably not a good idea
-  -- therefore we read all keys first and then delete them
-  phoneKeys <- runConduit $ getKeys ctr .| Conduit.sinkList
-  result <- runConduit $ deleteKeys phoneKeys .| Conduit.lastDef mempty
-  Log.info $ "final_deleted_phone_keys" .= show result
-
-main :: IO ()
-main = do
-  opts <- execParser (info (helper <*> optsParser) desc)
-  env <- mkEnv opts
-  runReaderT (unAppT run) env
-  where
-    desc = header "remove-phone-keys" <> progDesc "Remove phone data from wire-server" <> fullDesc
diff --git a/tools/db/remove-phone-keys/src/RemovePhoneKeys/Phone.hs b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Phone.hs
deleted file mode 100644
index cebe74fb86c..00000000000
--- a/tools/db/remove-phone-keys/src/RemovePhoneKeys/Phone.hs
+++ /dev/null
@@ -1,27 +0,0 @@
-module RemovePhoneKeys.Phone (Phone, parsePhone, toText) where
-
-import Data.Attoparsec.Text (char, count, digit, endOfInput, parseOnly)
-import qualified Data.Text as Text
-import Imports
-import Options.Applicative
-
-newtype Phone = Phone {fromPhone :: Text}
-  deriving stock (Eq, Ord, Show, Generic)
-
-toText :: Phone -> Text
-toText = fromPhone
-
--- | Parses a phone number in E.164 format with a mandatory leading '+'.
-parsePhone :: Text -> Maybe Phone
-parsePhone p =
-  let canonicalPhone = Text.filter (not . isSpace) p
-   in if isValidPhone canonicalPhone
-        then Just $ Phone canonicalPhone
-        else Nothing
-
--- | Checks whether a phone number is valid, i.e. it is in E.164 format
--- with a mandatory leading '+' followed by 10-15 digits.
-isValidPhone :: Text -> Bool
-isValidPhone = either (const False) (const True) . parseOnly e164
-  where
-    e164 = char '+' *> count 8 digit *> count 7 (optional digit) *> endOfInput
diff --git a/tools/db/remove-phone-keys/src/RemovePhoneKeys/Types.hs b/tools/db/remove-phone-keys/src/RemovePhoneKeys/Types.hs
deleted file mode 100644
index e8d9f42026d..00000000000
--- a/tools/db/remove-phone-keys/src/RemovePhoneKeys/Types.hs
+++ /dev/null
@@ -1,131 +0,0 @@
--- This file is part of the Wire Server implementation.
---
--- Copyright (C) 2024 Wire Swiss GmbH <opensource@wire.com>
---
--- This program is free software: you can redistribute it and/or modify it under
--- the terms of the GNU Affero General Public License as published by the Free
--- Software Foundation, either version 3 of the License, or (at your option) any
--- later version.
---
--- This program is distributed in the hope that it will be useful, but WITHOUT
--- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
--- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
--- details.
---
--- You should have received a copy of the GNU Affero General Public License along
--- with this program. If not, see <https://www.gnu.org/licenses/>.
-
-module RemovePhoneKeys.Types where
-
-import Cassandra as C
-import qualified Cassandra.Settings as C
-import Control.Lens
-import Control.Monad.Catch (MonadThrow)
-import Data.Text.Strict.Lens
-import Imports
-import Options.Applicative
-import qualified System.Logger as Log
-import System.Logger.Class (MonadLogger, log)
-
-data Env = Env
-  { casClient :: C.ClientState,
-    logger :: Log.Logger
-  }
-
-newtype AppT m a = AppT {unAppT :: ReaderT Env m a}
-  deriving
-    ( Functor,
-      Applicative,
-      Monad,
-      MonadIO,
-      MonadThrow,
-      MonadReader Env,
-      MonadUnliftIO
-    )
-
-instance MonadTrans AppT where
-  lift = AppT . lift
-
-instance (MonadIO m, MonadThrow m) => C.MonadClient (AppT m) where
-  liftClient m = do
-    env <- ask
-    lift . C.runClient env.casClient $ m
-  localState f = local (\env -> env {casClient = f $ env.casClient})
-
-instance (MonadIO m) => MonadLogger (AppT m) where
-  log level f = do
-    env <- ask
-    lift $ Log.log (env.logger) level f
-
-mkEnv :: Opts -> IO Env
-mkEnv opts = do
-  logger <- initLogger
-  brigClient <- initCas opts.brigDb logger
-  pure $ Env brigClient logger
-  where
-    initLogger =
-      Log.new
-        . Log.setOutput Log.StdOut
-        . Log.setFormat Nothing
-        . Log.setBufSize 0
-        $ Log.defSettings
-    initCas settings l =
-      C.init
-        . C.setLogger (C.mkLogger l)
-        . C.setContacts settings.host []
-        . C.setPortNumber (fromIntegral settings.port)
-        . C.setKeyspace settings.keyspace
-        . C.setProtocolVersion C.V4
-        $ C.defSettings
-
-data CassandraSettings = CassandraSettings
-  { host :: String,
-    port :: Int,
-    keyspace :: C.Keyspace
-  }
-
-data Opts = Opts
-  { brigDb :: CassandraSettings
-  }
-
-optsParser :: Parser Opts
-optsParser =
-  Opts <$> brigCassandraParser
-
-brigCassandraParser :: Parser CassandraSettings
-brigCassandraParser =
-  CassandraSettings
-    <$> strOption
-      ( long "brig-cassandra-host"
-          <> metavar "HOST"
-          <> help "Cassandra Host for brig"
-          <> value "localhost"
-          <> showDefault
-      )
-    <*> option
-      auto
-      ( long "brig-cassandra-port"
-          <> metavar "PORT"
-          <> help "Cassandra Port for brig"
-          <> value 9042
-          <> showDefault
-      )
-    <*> ( C.Keyspace
-            . view packed
-            <$> strOption
-              ( long "brig-cassandra-keyspace"
-                  <> metavar "STRING"
-                  <> help "Cassandra Keyspace for brig"
-                  <> value "brig_test"
-                  <> showDefault
-              )
-        )
-
-newtype IntSum = IntSum {unIntSum :: Int}
-  deriving newtype (Num, Show)
-
-instance Semigroup IntSum where
-  IntSum a <> IntSum b = IntSum (a + b)
-
-instance Monoid IntSum where
-  mempty = IntSum 0

From cc1e2e10a0b6abe1f35bfa702b78b369921d16f9 Mon Sep 17 00:00:00 2001
From: Leif Battermann <leif.battermann@wire.com>
Date: Mon, 15 Jul 2024 09:37:06 +0000
Subject: [PATCH 13/13] updated changelog

---
 changelog.d/0-release-notes/WPB-10058 | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/changelog.d/0-release-notes/WPB-10058 b/changelog.d/0-release-notes/WPB-10058
index dddc28a74e3..8f9c066875b 100644
--- a/changelog.d/0-release-notes/WPB-10058
+++ b/changelog.d/0-release-notes/WPB-10058
@@ -1,3 +1 @@
-Run a tool for deleting phone keys from the `user_keys` table in Brig. It can be
-found in `tools/db/remove-phone-keys`. See the accompanying README for more
-details.
+To remove phone keys from brig's `user_keys` table an ad hoc data-migration can be run. See PR https://github.com/wireapp/wire-server/pull/4146 which contains the implementation.