From 65d8b724e28a49c4c41708cb7823f9cddf3fcc59 Mon Sep 17 00:00:00 2001 From: Stefan Matting Date: Mon, 17 Jul 2023 15:45:12 +0200 Subject: [PATCH 1/2] Remove billing-team-member-backfill tool --- cabal.project | 3 - nix/local-haskell-packages.nix | 1 - nix/wire-server.nix | 2 - tools/db/billing-team-member-backfill/.ormolu | 1 - .../db/billing-team-member-backfill/README.md | 14 ---- .../billing-team-member-backfill.cabal | 81 ------------------- .../billing-team-member-backfill/default.nix | 42 ---------- .../billing-team-member-backfill/src/Main.hs | 57 ------------- .../src/Options.hs | 75 ----------------- .../billing-team-member-backfill/src/Work.hs | 81 ------------------- 10 files changed, 357 deletions(-) delete mode 120000 tools/db/billing-team-member-backfill/.ormolu delete mode 100644 tools/db/billing-team-member-backfill/README.md delete mode 100644 tools/db/billing-team-member-backfill/billing-team-member-backfill.cabal delete mode 100644 tools/db/billing-team-member-backfill/default.nix delete mode 100644 tools/db/billing-team-member-backfill/src/Main.hs delete mode 100644 tools/db/billing-team-member-backfill/src/Options.hs delete mode 100644 tools/db/billing-team-member-backfill/src/Work.hs diff --git a/cabal.project b/cabal.project index a6334c5db4..30e1a2b832 100644 --- a/cabal.project +++ b/cabal.project @@ -40,7 +40,6 @@ packages: , services/spar/ , tools/db/assets/ , tools/db/auto-whitelist/ - , tools/db/billing-team-member-backfill/ , tools/db/find-undead/ , tools/db/inconsistencies/ , tools/db/migrate-sso-feature-flag/ @@ -62,8 +61,6 @@ package background-worker ghc-options: -Werror package bilge ghc-options: -Werror -package billing-team-member-backfill - ghc-options: -Werror package brig ghc-options: -Werror package brig-types diff --git a/nix/local-haskell-packages.nix b/nix/local-haskell-packages.nix index 0770af43b6..ef57fc6614 100644 --- a/nix/local-haskell-packages.nix +++ b/nix/local-haskell-packages.nix @@ -44,7 +44,6 @@ spar = hself.callPackage ../services/spar/default.nix { inherit gitignoreSource; }; assets = hself.callPackage ../tools/db/assets/default.nix { inherit gitignoreSource; }; auto-whitelist = hself.callPackage ../tools/db/auto-whitelist/default.nix { inherit gitignoreSource; }; - billing-team-member-backfill = hself.callPackage ../tools/db/billing-team-member-backfill/default.nix { inherit gitignoreSource; }; find-undead = hself.callPackage ../tools/db/find-undead/default.nix { inherit gitignoreSource; }; inconsistencies = hself.callPackage ../tools/db/inconsistencies/default.nix { inherit gitignoreSource; }; migrate-sso-feature-flag = hself.callPackage ../tools/db/migrate-sso-feature-flag/default.nix { inherit gitignoreSource; }; diff --git a/nix/wire-server.nix b/nix/wire-server.nix index 5dc2f125cc..39a2348f97 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -80,8 +80,6 @@ let proxy = [ "proxy" ]; spar = [ "spar" "spar-integration" "spar-schema" "spar-migrate-data" ]; stern = [ "stern" "stern-integration" ]; - - billing-team-member-backfill = [ "billing-team-member-backfill" ]; inconsistencies = [ "inconsistencies" ]; zauth = [ "zauth" ]; background-worker = [ "background-worker" ]; diff --git a/tools/db/billing-team-member-backfill/.ormolu b/tools/db/billing-team-member-backfill/.ormolu deleted file mode 120000 index ffc2ca9745..0000000000 --- a/tools/db/billing-team-member-backfill/.ormolu +++ /dev/null @@ -1 +0,0 @@ -../../../.ormolu \ No newline at end of file diff --git a/tools/db/billing-team-member-backfill/README.md b/tools/db/billing-team-member-backfill/README.md deleted file mode 100644 index 5d72bd0e6a..0000000000 --- a/tools/db/billing-team-member-backfill/README.md +++ /dev/null @@ -1,14 +0,0 @@ -## billing_team_member backfill - -A tool for filling table `billing_team_member` from existing data. - -### How to run this - -```sh -export GALLEY_HOST=... # ip address of galley cassandra DB node -export GALLEY_KEYSPACE=galley - -ssh -v -f ubuntu@${GALLEY_HOST} -L 2021:${GALLEY_HOST}:9042 -N - -./dist/billing-team-member-backfill --cassandra-host-galley=localhost --cassandra-port-galley=2021 --cassandra-keyspace-galley=${GALLEY_KEYSPACE} -``` diff --git a/tools/db/billing-team-member-backfill/billing-team-member-backfill.cabal b/tools/db/billing-team-member-backfill/billing-team-member-backfill.cabal deleted file mode 100644 index b7d9befc03..0000000000 --- a/tools/db/billing-team-member-backfill/billing-team-member-backfill.cabal +++ /dev/null @@ -1,81 +0,0 @@ -cabal-version: 1.12 -name: billing-team-member-backfill -version: 1.0.0 -synopsis: Backfill billing_team_member table -category: Network -author: Wire Swiss GmbH -maintainer: Wire Swiss GmbH -copyright: (c) 2020 Wire Swiss GmbH -license: AGPL-3 -build-type: Simple - -executable billing-team-member-backfill - main-is: Main.hs - other-modules: - Options - Paths_billing_team_member_backfill - Work - - hs-source-dirs: src - default-extensions: - NoImplicitPrelude - AllowAmbiguousTypes - BangPatterns - ConstraintKinds - DataKinds - DefaultSignatures - DeriveFunctor - DeriveGeneric - DeriveLift - DeriveTraversable - DerivingStrategies - DerivingVia - DuplicateRecordFields - EmptyCase - FlexibleContexts - FlexibleInstances - FunctionalDependencies - GADTs - InstanceSigs - KindSignatures - LambdaCase - MultiParamTypeClasses - MultiWayIf - NamedFieldPuns - OverloadedRecordDot - OverloadedStrings - PackageImports - PatternSynonyms - PolyKinds - QuasiQuotes - RankNTypes - ScopedTypeVariables - StandaloneDeriving - TupleSections - TypeApplications - TypeFamilies - TypeFamilyDependencies - TypeOperators - UndecidableInstances - ViewPatterns - - ghc-options: - -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates - -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path - -funbox-strict-fields -threaded -with-rtsopts=-N -with-rtsopts=-T - -rtsopts -Wredundant-constraints -Wunused-packages - - build-depends: - base - , cassandra-util - , conduit - , containers - , imports - , lens - , optparse-applicative - , text - , tinylog - , types-common - , wire-api - - default-language: Haskell2010 diff --git a/tools/db/billing-team-member-backfill/default.nix b/tools/db/billing-team-member-backfill/default.nix deleted file mode 100644 index 18f60f04e7..0000000000 --- a/tools/db/billing-team-member-backfill/default.nix +++ /dev/null @@ -1,42 +0,0 @@ -# WARNING: GENERATED FILE, DO NOT EDIT. -# This file is generated by running hack/bin/generate-local-nix-packages.sh and -# must be regenerated whenever local packages are added or removed, or -# dependencies are added or removed. -{ mkDerivation -, base -, cassandra-util -, conduit -, containers -, gitignoreSource -, imports -, lens -, lib -, optparse-applicative -, text -, tinylog -, types-common -, wire-api -}: -mkDerivation { - pname = "billing-team-member-backfill"; - version = "1.0.0"; - src = gitignoreSource ./.; - isLibrary = false; - isExecutable = true; - executableHaskellDepends = [ - base - cassandra-util - conduit - containers - imports - lens - optparse-applicative - text - tinylog - types-common - wire-api - ]; - description = "Backfill billing_team_member table"; - license = lib.licenses.agpl3Only; - mainProgram = "billing-team-member-backfill"; -} diff --git a/tools/db/billing-team-member-backfill/src/Main.hs b/tools/db/billing-team-member-backfill/src/Main.hs deleted file mode 100644 index 6811cf3154..0000000000 --- a/tools/db/billing-team-member-backfill/src/Main.hs +++ /dev/null @@ -1,57 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Main - ( main, - ) -where - -import Cassandra as C -import Cassandra.Settings as C -import Imports -import Options as O -import Options.Applicative -import qualified System.Logger as Log -import Work - -main :: IO () -main = do - s <- execParser (info (helper <*> settingsParser) desc) - lgr <- initLogger - gc <- initCas (setCasGalley s) lgr - runCommand lgr gc - where - desc = - header "billing-team-member-backfill" - <> progDesc "Backfill billing_team_member table" - <> fullDesc - initLogger = - Log.new - . Log.setOutput Log.StdOut - . Log.setFormat Nothing - . Log.setBufSize 0 - $ Log.defSettings - initCas cas l = - C.init - . C.setLogger (C.mkLogger l) - . C.setContacts (cHosts cas) [] - . C.setPortNumber (fromIntegral $ cPort cas) - . C.setKeyspace (cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings diff --git a/tools/db/billing-team-member-backfill/src/Options.hs b/tools/db/billing-team-member-backfill/src/Options.hs deleted file mode 100644 index ca3afd0802..0000000000 --- a/tools/db/billing-team-member-backfill/src/Options.hs +++ /dev/null @@ -1,75 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Options - ( setCasGalley, - cHosts, - cPort, - cKeyspace, - settingsParser, - ) -where - -import qualified Cassandra as C -import qualified Data.Text as Text -import Imports -import Options.Applicative - -newtype MigratorSettings = MigratorSettings {setCasGalley :: CassandraSettings} - deriving (Show) - -data CassandraSettings = CassandraSettings - { cHosts :: !String, - cPort :: !Word16, - cKeyspace :: !C.Keyspace - } - deriving (Show) - -settingsParser :: Parser MigratorSettings -settingsParser = - MigratorSettings - <$> cassandraSettingsParser "galley" - -cassandraSettingsParser :: String -> Parser CassandraSettings -cassandraSettingsParser ks = - CassandraSettings - <$> strOption - ( long ("cassandra-host-" ++ ks) - <> metavar "HOST" - <> help ("Cassandra Host for: " ++ ks) - <> value "localhost" - <> showDefault - ) - <*> option - auto - ( long ("cassandra-port-" ++ ks) - <> metavar "PORT" - <> help ("Cassandra Port for: " ++ ks) - <> value 9042 - <> showDefault - ) - <*> ( C.Keyspace . Text.pack - <$> strOption - ( long ("cassandra-keyspace-" ++ ks) - <> metavar "STRING" - <> help ("Cassandra Keyspace for: " ++ ks) - <> value (ks ++ "_test") - <> showDefault - ) - ) diff --git a/tools/db/billing-team-member-backfill/src/Work.hs b/tools/db/billing-team-member-backfill/src/Work.hs deleted file mode 100644 index 84b8b1d604..0000000000 --- a/tools/db/billing-team-member-backfill/src/Work.hs +++ /dev/null @@ -1,81 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE ScopedTypeVariables #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Work where - -import Cassandra -import Conduit -import Control.Lens (view) -import Data.Conduit.Internal (zipSources) -import qualified Data.Conduit.List as C -import Data.Id -import qualified Data.Set as Set -import Imports -import System.Logger (Logger) -import qualified System.Logger as Log -import Wire.API.Team.Permission - -runCommand :: Logger -> ClientState -> IO () -runCommand l galley = - runConduit $ - zipSources - (C.sourceList [(1 :: Int32) ..]) - (transPipe (runClient galley) getTeamMembers) - .| C.mapM - ( \(i, p) -> - Log.info l (Log.field "team members" (show (i * pageSize))) - >> pure p - ) - .| C.concatMap (filter isOwner) - .| C.map (\(t, u, _) -> (t, u)) - .| C.chunksOf 50 - .| C.mapM - ( \x -> - Log.info l (Log.field "writing billing team members" (show (length x))) - >> pure x - ) - .| C.mapM_ (runClient galley . createBillingTeamMembers) - -pageSize :: Int32 -pageSize = 1000 - ----------------------------------------------------------------------------- --- Queries - --- | Get team members from Galley -getTeamMembers :: ConduitM () [(TeamId, UserId, Maybe Permissions)] Client () -getTeamMembers = paginateC cql (paramsP LocalQuorum () pageSize) x5 - where - cql :: PrepQuery R () (TeamId, UserId, Maybe Permissions) - cql = "SELECT team, user, perms FROM team_member" - -createBillingTeamMembers :: [(TeamId, UserId)] -> Client () -createBillingTeamMembers pairs = - retry x5 . batch $ do - setType BatchLogged - setConsistency LocalQuorum - mapM_ (addPrepQuery cql) pairs - where - cql :: PrepQuery W (TeamId, UserId) () - cql = "INSERT INTO billing_team_member (team, user) values (?, ?)" - -isOwner :: (TeamId, UserId, Maybe Permissions) -> Bool -isOwner (_, _, Just p) = SetBilling `Set.member` view self p -isOwner _ = False From 8988fa5f674669630ad5a3e5fc5cce9c5d55433b Mon Sep 17 00:00:00 2001 From: Stefan Matting Date: Mon, 17 Jul 2023 16:16:07 +0200 Subject: [PATCH 2/2] hi ci