diff --git a/changelog.d/5-internal/sts-expiry-metrics b/changelog.d/5-internal/sts-expiry-metrics index 9416e009f3..556f1b7c8b 100644 --- a/changelog.d/5-internal/sts-expiry-metrics +++ b/changelog.d/5-internal/sts-expiry-metrics @@ -1 +1 @@ -Add AWS security token metrics to brig +Add AWS security token metrics to all services diff --git a/libs/metrics-core/metrics-core.cabal b/libs/metrics-core/metrics-core.cabal index b4191471ba..12ef8c9ddd 100644 --- a/libs/metrics-core/metrics-core.cabal +++ b/libs/metrics-core/metrics-core.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 94e1bf0c93057c9abbeafa85bd4d465c71e20c3bda5a8962cfaad8431861ddd5 +-- hash: ecdf5dcfd0edfbffa2ff3490ea07fc1a5fac46874617b280fe981ac942852ff2 name: metrics-core version: 0.3.2 @@ -20,6 +20,7 @@ build-type: Simple library exposed-modules: Data.Metrics + Data.Metrics.AWS Data.Metrics.GC other-modules: Paths_metrics_core @@ -73,5 +74,6 @@ library , imports , prometheus-client , text >=0.11 + , time , unordered-containers >=0.2 default-language: Haskell2010 diff --git a/libs/metrics-core/package.yaml b/libs/metrics-core/package.yaml index abba84394a..d1e316f5db 100644 --- a/libs/metrics-core/package.yaml +++ b/libs/metrics-core/package.yaml @@ -19,4 +19,5 @@ library: - prometheus-client - unordered-containers >=0.2 - text >=0.11 + - time - immortal diff --git a/libs/metrics-core/src/Data/Metrics/AWS.hs b/libs/metrics-core/src/Data/Metrics/AWS.hs new file mode 100644 index 0000000000..7ff710f229 --- /dev/null +++ b/libs/metrics-core/src/Data/Metrics/AWS.hs @@ -0,0 +1,29 @@ +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Data.Metrics.AWS (gaugeTokenRemaing) where + +import Data.Metrics (Metrics, gaugeSet, path) +import Data.Time +import Imports + +gaugeTokenRemaing :: Metrics -> Maybe NominalDiffTime -> IO () +gaugeTokenRemaing m mbRemaining = do + let t = toSeconds (fromMaybe 0 mbRemaining) + gaugeSet t (path "aws_auth.token_secs_remaining") m + where + toSeconds :: NominalDiffTime -> Double + toSeconds = fromRational . toRational diff --git a/libs/types-common-aws/package.yaml b/libs/types-common-aws/package.yaml index db9fd5c09f..9132ef5096 100644 --- a/libs/types-common-aws/package.yaml +++ b/libs/types-common-aws/package.yaml @@ -24,6 +24,7 @@ dependencies: - tasty - tasty-hunit - text >=0.11 +- time library: source-dirs: src ghc-prof-options: -fprof-auto-exported diff --git a/libs/types-common-aws/src/AWS/Util.hs b/libs/types-common-aws/src/AWS/Util.hs new file mode 100644 index 0000000000..dac97a3e6e --- /dev/null +++ b/libs/types-common-aws/src/AWS/Util.hs @@ -0,0 +1,32 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2022 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module AWS.Util where + +import qualified Amazonka as AWS +import Data.Time +import Imports + +readAuthExpiration :: AWS.Env -> IO (Maybe NominalDiffTime) +readAuthExpiration env = do + authEnv <- + case runIdentity (AWS.envAuth env) of + AWS.Auth authEnv -> pure authEnv + AWS.Ref _ ref -> do + readIORef ref + now <- getCurrentTime + pure $ ((`diffUTCTime` now) . AWS.fromTime) <$> (AWS._authExpiration authEnv) diff --git a/libs/types-common-aws/types-common-aws.cabal b/libs/types-common-aws/types-common-aws.cabal index 80df15ca8f..9a7e344696 100644 --- a/libs/types-common-aws/types-common-aws.cabal +++ b/libs/types-common-aws/types-common-aws.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: e21810d1081ee50f1d775ea6112131f62a9fb270d33d2c8a44024afa384a34ff +-- hash: 329c0cd9a1a42c313141a132db82bbfc665b36ff4066dbe6d1b5c1706fc4cb01 name: types-common-aws version: 0.16.0 @@ -30,6 +30,7 @@ flag protobuf library exposed-modules: + AWS.Util Util.Test.SQS other-modules: Paths_types_common_aws @@ -91,6 +92,7 @@ library , tasty , tasty-hunit , text >=0.11 + , time if impl(ghc >=8) ghc-options: -fno-warn-redundant-constraints if flag(protobuf) diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index d9d923a334..dd3a691b03 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -270,6 +270,7 @@ library , tinylog >=0.10 , transformers >=0.3 , types-common >=0.16 + , types-common-aws , types-common-journal >=0.1 , unliftio >=0.2 , unordered-containers >=0.2 diff --git a/services/brig/package.yaml b/services/brig/package.yaml index 7e7e48d29f..cf1f95d94f 100644 --- a/services/brig/package.yaml +++ b/services/brig/package.yaml @@ -120,6 +120,7 @@ library: - tinylog >=0.10 - transformers >=0.3 - types-common >=0.16 + - types-common-aws - types-common-journal >=0.1 - unliftio >=0.2 - unordered-containers >=0.2 diff --git a/services/brig/src/Brig/AWS.hs b/services/brig/src/Brig/AWS.hs index 529d3d88bb..85c081a514 100644 --- a/services/brig/src/Brig/AWS.hs +++ b/services/brig/src/Brig/AWS.hs @@ -42,8 +42,6 @@ module Brig.AWS -- * AWS exec, execCatch, - readAuthExpiration, - isAuthARef, ) where @@ -64,7 +62,6 @@ import Data.ByteString.Builder (toLazyByteString) import qualified Data.ByteString.Lazy as BL import qualified Data.Text as Text import qualified Data.Text.Encoding as Text -import Data.Time import Data.UUID hiding (null) import Imports hiding (group) import Network.HTTP.Client (HttpException (..), HttpExceptionContent (..), Manager) @@ -273,19 +270,3 @@ canRetry (Left e) = case e of retry5x :: (Monad m) => RetryPolicyM m retry5x = limitRetries 5 <> exponentialBackoff 100000 - -readAuthExpiration :: AWS.Env -> IO (Maybe NominalDiffTime) -readAuthExpiration env = do - authEnv <- - case runIdentity (AWS.envAuth env) of - AWS.Auth authEnv -> pure authEnv - AWS.Ref _ ref -> do - readIORef ref - now <- getCurrentTime - pure $ ((`diffUTCTime` now) . AWS.fromTime) <$> (AWS._authExpiration authEnv) - -isAuthARef :: AWS.Env -> Bool -isAuthARef env = - case runIdentity (AWS.envAuth env) of - AWS.Auth _ -> False - AWS.Ref _ _ -> True diff --git a/services/brig/src/Brig/Run.hs b/services/brig/src/Brig/Run.hs index 606fe74210..9b09534586 100644 --- a/services/brig/src/Brig/Run.hs +++ b/services/brig/src/Brig/Run.hs @@ -23,6 +23,7 @@ module Brig.Run ) where +import AWS.Util (readAuthExpiration) import Brig.API (sitemap) import Brig.API.Federation import Brig.API.Handler @@ -49,12 +50,11 @@ import Control.Monad.Random (randomRIO) import qualified Data.Aeson as Aeson import Data.Default (Default (def)) import Data.Id (RequestId (..)) -import Data.Metrics (gaugeSet, path) +import Data.Metrics.AWS (gaugeTokenRemaing) import qualified Data.Metrics.Servant as Metrics import Data.Proxy (Proxy (Proxy)) import Data.String.Conversions (cs) import Data.Text (unpack) -import Data.Time (NominalDiffTime) import Imports hiding (head) import qualified Network.HTTP.Media as HTTPMedia import qualified Network.HTTP.Types as HTTP @@ -239,12 +239,8 @@ collectAuthMetrics :: forall r. AppT r () collectAuthMetrics = do m <- view metrics env <- view (awsEnv . amazonkaEnv) - - forever $ do - t <- toSeconds . fromMaybe 0 <$$> liftIO $ AWS.readAuthExpiration env - gaugeSet t (path "aws_auth.token_secs_remaining") m - gaugeSet (if AWS.isAuthARef env then 1.0 else 0.0) (path "aws_auth.token_is_reference") m - liftIO $ threadDelay 1_000_000 - where - toSeconds :: NominalDiffTime -> Double - toSeconds = fromRational . toRational + liftIO $ + forever $ do + mbRemaining <- readAuthExpiration env + gaugeTokenRemaing m mbRemaining + threadDelay 1_000_000 diff --git a/services/cargohold/cargohold.cabal b/services/cargohold/cargohold.cabal index 078308f0a1..283c836020 100644 --- a/services/cargohold/cargohold.cabal +++ b/services/cargohold/cargohold.cabal @@ -113,6 +113,7 @@ library , imports , kan-extensions , lens >=4.1 + , metrics-core , metrics-wai >=0.4 , mime >=0.4 , optparse-applicative >=0.10 @@ -126,6 +127,8 @@ library , time >=1.4 , tinylog >=0.10 , types-common >=0.16 + , types-common-aws + , unliftio , unordered-containers >=0.2 , uri-bytestring >=0.2 , uuid >=1.3.5 diff --git a/services/cargohold/package.yaml b/services/cargohold/package.yaml index d59cff100a..8341787379 100644 --- a/services/cargohold/package.yaml +++ b/services/cargohold/package.yaml @@ -47,6 +47,7 @@ library: - http-client-openssl >=0.2 - lens >=4.1 - metrics-wai >=0.4 + - metrics-core - optparse-applicative >=0.10 - retry >=0.5 - resourcet >=1.1 @@ -56,6 +57,8 @@ library: - time >=1.4 - tinylog >=0.10 - types-common >=0.16 + - types-common-aws + - unliftio - unordered-containers >=0.2 - uri-bytestring >=0.2 - uuid >=1.3.5 diff --git a/services/cargohold/src/CargoHold/Run.hs b/services/cargohold/src/CargoHold/Run.hs index 21538bd999..7f0b3d73d1 100644 --- a/services/cargohold/src/CargoHold/Run.hs +++ b/services/cargohold/src/CargoHold/Run.hs @@ -14,6 +14,7 @@ -- -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . +{-# LANGUAGE NumericUnderscores #-} module CargoHold.Run ( run, @@ -21,8 +22,11 @@ module CargoHold.Run ) where +import AWS.Util (readAuthExpiration) +import qualified Amazonka as AWS import CargoHold.API.Federation import CargoHold.API.Public +import CargoHold.AWS (amazonkaEnv) import CargoHold.App import CargoHold.Options import Control.Exception (bracket) @@ -30,6 +34,8 @@ import Control.Lens (set, (^.)) import Control.Monad.Codensity import Data.Default import Data.Id +import Data.Metrics (Metrics) +import Data.Metrics.AWS (gaugeTokenRemaing) import Data.Metrics.Servant import Data.Proxy import Data.Text (unpack) @@ -42,6 +48,7 @@ import qualified Network.Wai.Utilities.Server as Server import qualified Servant import Servant.API import Servant.Server hiding (Handler, runHandler) +import qualified UnliftIO.Async as Async import Util.Options import Wire.API.Routes.API import Wire.API.Routes.Internal.Cargohold @@ -53,6 +60,7 @@ type CombinedAPI = FederationAPI :<|> ServantAPI :<|> InternalAPI run :: Opts -> IO () run o = lowerCodensity $ do (app, e) <- mkApp o + void $ Codensity $ Async.withAsync (collectAuthMetrics (e ^. metrics) (e ^. aws . amazonkaEnv)) liftIO $ do s <- Server.newSettings $ @@ -88,3 +96,11 @@ mkApp o = Codensity $ \k -> toServantHandler :: Env -> Handler a -> Servant.Handler a toServantHandler env = liftIO . runHandler env + +collectAuthMetrics :: MonadIO m => Metrics -> AWS.Env -> m () +collectAuthMetrics m env = do + liftIO $ + forever $ do + mbRemaining <- readAuthExpiration env + gaugeTokenRemaing m mbRemaining + threadDelay 1_000_000 diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index bfbc9b5723..b9f7ecc89b 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -213,6 +213,7 @@ library , kan-extensions , lens >=4.4 , memory + , metrics-core , metrics-wai >=0.4 , mtl >=2.2 , optparse-applicative >=0.10 @@ -250,6 +251,7 @@ library , tls >=1.3.10 , transformers , types-common >=0.16 + , types-common-aws , types-common-journal >=0.1 , unliftio >=0.2 , unordered-containers >=0.2 diff --git a/services/galley/package.yaml b/services/galley/package.yaml index 7a8b3f658d..a9495416d6 100644 --- a/services/galley/package.yaml +++ b/services/galley/package.yaml @@ -65,6 +65,7 @@ library: - lens >=4.4 - memory - metrics-wai >=0.4 + - metrics-core - mtl >=2.2 - optparse-applicative >=0.10 - pem @@ -97,6 +98,7 @@ library: - tls >=1.3.10 - types-common >=0.16 - types-common-journal >=0.1 + - types-common-aws - unliftio >=0.2 - unordered-containers >=0.2 - uri-bytestring >=0.2 diff --git a/services/galley/src/Galley/Run.hs b/services/galley/src/Galley/Run.hs index 21148ef78c..7f4b245563 100644 --- a/services/galley/src/Galley/Run.hs +++ b/services/galley/src/Galley/Run.hs @@ -14,6 +14,7 @@ -- -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . +{-# LANGUAGE NumericUnderscores #-} module Galley.Run ( run, @@ -21,6 +22,8 @@ module Galley.Run ) where +import AWS.Util (readAuthExpiration) +import qualified Amazonka as AWS import Bilge.Request (requestIdName) import Cassandra (runClient, shutdown) import Cassandra.Schema (versionCheck) @@ -31,6 +34,8 @@ import Control.Monad.Codensity import qualified Data.Aeson as Aeson import Data.Default import Data.Id +import Data.Metrics (Metrics) +import Data.Metrics.AWS (gaugeTokenRemaing) import qualified Data.Metrics.Middleware as M import Data.Metrics.Servant (servantPlusWAIPrometheusMiddleware) import Data.Misc (portNumber) @@ -41,6 +46,7 @@ import Galley.API.Federation (FederationAPI, federationSitemap) import Galley.API.Internal import Galley.App import qualified Galley.App as App +import Galley.Aws (awsEnv) import Galley.Cassandra import Galley.Monad import Galley.Options @@ -71,6 +77,8 @@ run opts = lowerCodensity $ do (env ^. App.applog) (env ^. monitor) + forM_ (env ^. aEnv) $ \aws -> + void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv) void $ Codensity $ Async.withAsync $ runApp env deleteLoop void $ Codensity $ Async.withAsync $ runApp env refreshMetrics lift $ finally (runSettingsWithShutdown settings app 5) (shutdown (env ^. cstate)) @@ -152,3 +160,11 @@ refreshMetrics = do n <- Q.len q M.gaugeSet (fromIntegral n) (M.path "galley.deletequeue.len") m threadDelay 1000000 + +collectAuthMetrics :: MonadIO m => Metrics -> AWS.Env -> m () +collectAuthMetrics m env = do + liftIO $ + forever $ do + mbRemaining <- readAuthExpiration env + gaugeTokenRemaing m mbRemaining + threadDelay 1_000_000 diff --git a/services/gundeck/gundeck.cabal b/services/gundeck/gundeck.cabal index 43f175afd4..4f67b2c36c 100644 --- a/services/gundeck/gundeck.cabal +++ b/services/gundeck/gundeck.cabal @@ -140,6 +140,7 @@ library , tinylog >=0.10 , tls >=1.3.4 , types-common >=0.16 + , types-common-aws , unliftio >=0.2 , unordered-containers >=0.2 , uuid >=1.3 diff --git a/services/gundeck/package.yaml b/services/gundeck/package.yaml index 93a3e783db..e2b0d94749 100644 --- a/services/gundeck/package.yaml +++ b/services/gundeck/package.yaml @@ -57,6 +57,7 @@ library: - tinylog >=0.10 - tls >=1.3.4 - types-common >=0.16 + - types-common-aws - unliftio >=0.2 - unordered-containers >=0.2 - uuid >=1.3 diff --git a/services/gundeck/src/Gundeck/Aws.hs b/services/gundeck/src/Gundeck/Aws.hs index 0694f07923..989a09964e 100644 --- a/services/gundeck/src/Gundeck/Aws.hs +++ b/services/gundeck/src/Gundeck/Aws.hs @@ -20,7 +20,7 @@ module Gundeck.Aws ( -- * Monad - Env, + Env (..), mkEnv, Amazon, execute, diff --git a/services/gundeck/src/Gundeck/Run.hs b/services/gundeck/src/Gundeck/Run.hs index cbc9a1eae9..51742bac5f 100644 --- a/services/gundeck/src/Gundeck/Run.hs +++ b/services/gundeck/src/Gundeck/Run.hs @@ -14,13 +14,18 @@ -- -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . +{-# LANGUAGE NumericUnderscores #-} module Gundeck.Run where +import AWS.Util (readAuthExpiration) +import qualified Amazonka as AWS import Cassandra (runClient, shutdown) import Cassandra.Schema (versionCheck) import Control.Exception (finally) import Control.Lens hiding (enum) +import Data.Metrics (Metrics) +import Data.Metrics.AWS (gaugeTokenRemaing) import Data.Metrics.Middleware (metrics) import Data.Metrics.Middleware.Prometheus (waiPrometheusMiddleware) import Data.Text (unpack) @@ -28,6 +33,7 @@ import qualified Database.Redis as Redis import Gundeck.API (sitemap) import qualified Gundeck.Aws as Aws import Gundeck.Env +import qualified Gundeck.Env as Env import Gundeck.Monad import Gundeck.Options import Gundeck.React @@ -53,10 +59,12 @@ run o = do let throttleMillis = fromMaybe defSqsThrottleMillis $ o ^. (optSettings . setSqsThrottleMillis) lst <- Async.async $ Aws.execute (e ^. awsEnv) (Aws.listen throttleMillis (runDirect e . onEvent)) wtbs <- forM (e ^. threadBudgetState) $ \tbs -> Async.async $ runDirect e $ watchThreadBudgetState m tbs 10 + wCollectAuth <- Async.async (collectAuthMetrics m (Aws._awsEnv (Env._awsEnv e))) runSettingsWithShutdown s (middleware e $ mkApp e) 5 `finally` do Log.info l $ Log.msg (Log.val "Shutting down ...") shutdown (e ^. cstate) Async.cancel lst + Async.cancel wCollectAuth forM_ wtbs Async.cancel Redis.disconnect (e ^. rstate) Log.close (e ^. applog) @@ -73,3 +81,11 @@ mkApp :: Env -> Wai.Application mkApp e r k = runGundeck e r (route routes r k) where routes = compile sitemap + +collectAuthMetrics :: MonadIO m => Metrics -> AWS.Env -> m () +collectAuthMetrics m env = do + liftIO $ + forever $ do + mbRemaining <- readAuthExpiration env + gaugeTokenRemaing m mbRemaining + threadDelay 1_000_000