Add aws_auth.token_secs_remaining to gundeck, cargohold, galley

changelog.d/5-internal/sts-expiry-metrics

@@ -1 +1 @@
-Add AWS security token metrics to brig
+Add AWS security token metrics to all services

libs/metrics-core/metrics-core.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 94e1bf0c93057c9abbeafa85bd4d465c71e20c3bda5a8962cfaad8431861ddd5
+-- hash: ecdf5dcfd0edfbffa2ff3490ea07fc1a5fac46874617b280fe981ac942852ff2
 
 name:           metrics-core
 version:        0.3.2
@@ -20,6 +20,7 @@ build-type:     Simple
 library
   exposed-modules:
       Data.Metrics
+      Data.Metrics.AWS
       Data.Metrics.GC
   other-modules:
       Paths_metrics_core
@@ -73,5 +74,6 @@ library
     , imports
     , prometheus-client
     , text >=0.11
+    , time
     , unordered-containers >=0.2
   default-language: Haskell2010

libs/metrics-core/package.yaml

@@ -19,4 +19,5 @@ library:
   - prometheus-client
   - unordered-containers >=0.2
   - text >=0.11
+  - time
   - immortal

libs/metrics-core/src/Data/Metrics/AWS.hs

@@ -0,0 +1,29 @@
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Data.Metrics.AWS (gaugeTokenRemaing) where
+
+import Data.Metrics (Metrics, gaugeSet, path)
+import Data.Time
+import Imports
+
+gaugeTokenRemaing :: Metrics -> Maybe NominalDiffTime -> IO ()
+gaugeTokenRemaing m mbRemaining = do
+  let t = toSeconds (fromMaybe 0 mbRemaining)
+  gaugeSet t (path "aws_auth.token_secs_remaining") m
+  where
+    toSeconds :: NominalDiffTime -> Double
+    toSeconds = fromRational . toRational

libs/types-common-aws/package.yaml

@@ -24,6 +24,7 @@ dependencies:
 - tasty
 - tasty-hunit
 - text >=0.11
+- time
 library:
   source-dirs: src
   ghc-prof-options: -fprof-auto-exported

libs/types-common-aws/src/AWS/Util.hs

@@ -0,0 +1,32 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2022 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module AWS.Util where
+
+import qualified Amazonka as AWS
+import Data.Time
+import Imports
+
+readAuthExpiration :: AWS.Env -> IO (Maybe NominalDiffTime)
+readAuthExpiration env = do
+  authEnv <-
+    case runIdentity (AWS.envAuth env) of
+      AWS.Auth authEnv -> pure authEnv
+      AWS.Ref _ ref -> do
+        readIORef ref
+  now <- getCurrentTime
+  pure $ ((`diffUTCTime` now) . AWS.fromTime) <$> (AWS._authExpiration authEnv)

libs/types-common-aws/types-common-aws.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: e21810d1081ee50f1d775ea6112131f62a9fb270d33d2c8a44024afa384a34ff
+-- hash: 329c0cd9a1a42c313141a132db82bbfc665b36ff4066dbe6d1b5c1706fc4cb01
 
 name:           types-common-aws
 version:        0.16.0
@@ -30,6 +30,7 @@ flag protobuf
 
 library
   exposed-modules:
+      AWS.Util
       Util.Test.SQS
   other-modules:
       Paths_types_common_aws
@@ -91,6 +92,7 @@ library
     , tasty
     , tasty-hunit
     , text >=0.11
+    , time
   if impl(ghc >=8)
     ghc-options: -fno-warn-redundant-constraints
   if flag(protobuf)

services/brig/brig.cabal

@@ -270,6 +270,7 @@ library
     , tinylog >=0.10
     , transformers >=0.3
     , types-common >=0.16
+    , types-common-aws
     , types-common-journal >=0.1
     , unliftio >=0.2
     , unordered-containers >=0.2

services/brig/package.yaml

@@ -120,6 +120,7 @@ library:
   - tinylog >=0.10
   - transformers >=0.3
   - types-common >=0.16
+  - types-common-aws
   - types-common-journal >=0.1
   - unliftio >=0.2
   - unordered-containers >=0.2

services/brig/src/Brig/AWS.hs

@@ -42,8 +42,6 @@ module Brig.AWS
     -- * AWS
     exec,
     execCatch,
-    readAuthExpiration,
-    isAuthARef,
   )
 where
 
@@ -64,7 +62,6 @@ import Data.ByteString.Builder (toLazyByteString)
 import qualified Data.ByteString.Lazy as BL
 import qualified Data.Text as Text
 import qualified Data.Text.Encoding as Text
-import Data.Time
 import Data.UUID hiding (null)
 import Imports hiding (group)
 import Network.HTTP.Client (HttpException (..), HttpExceptionContent (..), Manager)
@@ -273,19 +270,3 @@ canRetry (Left e) = case e of
 
 retry5x :: (Monad m) => RetryPolicyM m
 retry5x = limitRetries 5 <> exponentialBackoff 100000
-
-readAuthExpiration :: AWS.Env -> IO (Maybe NominalDiffTime)
-readAuthExpiration env = do
-  authEnv <-
-    case runIdentity (AWS.envAuth env) of
-      AWS.Auth authEnv -> pure authEnv
-      AWS.Ref _ ref -> do
-        readIORef ref
-  now <- getCurrentTime
-  pure $ ((`diffUTCTime` now) . AWS.fromTime) <$> (AWS._authExpiration authEnv)
-
-isAuthARef :: AWS.Env -> Bool
-isAuthARef env =
-  case runIdentity (AWS.envAuth env) of
-    AWS.Auth _ -> False
-    AWS.Ref _ _ -> True

services/brig/src/Brig/Run.hs

@@ -23,6 +23,7 @@ module Brig.Run
   )
 where
 
+import AWS.Util (readAuthExpiration)
 import Brig.API (sitemap)
 import Brig.API.Federation
 import Brig.API.Handler
@@ -49,12 +50,11 @@ import Control.Monad.Random (randomRIO)
 import qualified Data.Aeson as Aeson
 import Data.Default (Default (def))
 import Data.Id (RequestId (..))
-import Data.Metrics (gaugeSet, path)
+import Data.Metrics.AWS (gaugeTokenRemaing)
 import qualified Data.Metrics.Servant as Metrics
 import Data.Proxy (Proxy (Proxy))
 import Data.String.Conversions (cs)
 import Data.Text (unpack)
-import Data.Time (NominalDiffTime)
 import Imports hiding (head)
 import qualified Network.HTTP.Media as HTTPMedia
 import qualified Network.HTTP.Types as HTTP
@@ -239,12 +239,8 @@ collectAuthMetrics :: forall r. AppT r ()
 collectAuthMetrics = do
   m <- view metrics
   env <- view (awsEnv . amazonkaEnv)
-
-  forever $ do
-    t <- toSeconds . fromMaybe 0 <$$> liftIO $ AWS.readAuthExpiration env
-    gaugeSet t (path "aws_auth.token_secs_remaining") m
-    gaugeSet (if AWS.isAuthARef env then 1.0 else 0.0) (path "aws_auth.token_is_reference") m
-    liftIO $ threadDelay 1_000_000
-  where
-    toSeconds :: NominalDiffTime -> Double
-    toSeconds = fromRational . toRational
+  liftIO $
+    forever $ do
+      mbRemaining <- readAuthExpiration env
+      gaugeTokenRemaing m mbRemaining
+      threadDelay 1_000_000

services/cargohold/cargohold.cabal

@@ -113,6 +113,7 @@ library
     , imports
     , kan-extensions
     , lens >=4.1
+    , metrics-core
     , metrics-wai >=0.4
     , mime >=0.4
     , optparse-applicative >=0.10
@@ -126,6 +127,8 @@ library
     , time >=1.4
     , tinylog >=0.10
     , types-common >=0.16
+    , types-common-aws
+    , unliftio
     , unordered-containers >=0.2
     , uri-bytestring >=0.2
     , uuid >=1.3.5

services/cargohold/package.yaml

@@ -47,6 +47,7 @@ library:
   - http-client-openssl >=0.2
   - lens >=4.1
   - metrics-wai >=0.4
+  - metrics-core
   - optparse-applicative >=0.10
   - retry >=0.5
   - resourcet >=1.1
@@ -56,6 +57,8 @@ library:
   - time >=1.4
   - tinylog >=0.10
   - types-common >=0.16
+  - types-common-aws
+  - unliftio
   - unordered-containers >=0.2
   - uri-bytestring >=0.2
   - uuid >=1.3.5

services/cargohold/src/CargoHold/Run.hs

@@ -14,22 +14,28 @@
 --
 -- You should have received a copy of the GNU Affero General Public License along
 -- with this program. If not, see <https://www.gnu.org/licenses/>.
+{-# LANGUAGE NumericUnderscores #-}
 
 module CargoHold.Run
   ( run,
     mkApp,
   )
 where
 
+import AWS.Util (readAuthExpiration)
+import qualified Amazonka as AWS
 import CargoHold.API.Federation
 import CargoHold.API.Public
+import CargoHold.AWS (amazonkaEnv)
 import CargoHold.App
 import CargoHold.Options
 import Control.Exception (bracket)
 import Control.Lens (set, (^.))
 import Control.Monad.Codensity
 import Data.Default
 import Data.Id
+import Data.Metrics (Metrics)
+import Data.Metrics.AWS (gaugeTokenRemaing)
 import Data.Metrics.Servant
 import Data.Proxy
 import Data.Text (unpack)
@@ -42,6 +48,7 @@ import qualified Network.Wai.Utilities.Server as Server
 import qualified Servant
 import Servant.API
 import Servant.Server hiding (Handler, runHandler)
+import qualified UnliftIO.Async as Async
 import Util.Options
 import Wire.API.Routes.API
 import Wire.API.Routes.Internal.Cargohold
@@ -53,6 +60,7 @@ type CombinedAPI = FederationAPI :<|> ServantAPI :<|> InternalAPI
 run :: Opts -> IO ()
 run o = lowerCodensity $ do
   (app, e) <- mkApp o
+  void $ Codensity $ Async.withAsync (collectAuthMetrics (e ^. metrics) (e ^. aws . amazonkaEnv))
   liftIO $ do
     s <-
       Server.newSettings $
@@ -88,3 +96,11 @@ mkApp o = Codensity $ \k ->
 
 toServantHandler :: Env -> Handler a -> Servant.Handler a
 toServantHandler env = liftIO . runHandler env
+
+collectAuthMetrics :: MonadIO m => Metrics -> AWS.Env -> m ()
+collectAuthMetrics m env = do
+  liftIO $
+    forever $ do
+      mbRemaining <- readAuthExpiration env
+      gaugeTokenRemaing m mbRemaining
+      threadDelay 1_000_000

services/galley/galley.cabal

@@ -213,6 +213,7 @@ library
     , kan-extensions
     , lens >=4.4
     , memory
+    , metrics-core
     , metrics-wai >=0.4
     , mtl >=2.2
     , optparse-applicative >=0.10
@@ -250,6 +251,7 @@ library
     , tls >=1.3.10
     , transformers
     , types-common >=0.16
+    , types-common-aws
     , types-common-journal >=0.1
     , unliftio >=0.2
     , unordered-containers >=0.2

services/galley/package.yaml

@@ -65,6 +65,7 @@ library:
   - lens >=4.4
   - memory
   - metrics-wai >=0.4
+  - metrics-core
   - mtl >=2.2
   - optparse-applicative >=0.10
   - pem
@@ -97,6 +98,7 @@ library:
   - tls >=1.3.10
   - types-common >=0.16
   - types-common-journal >=0.1
+  - types-common-aws
   - unliftio >=0.2
   - unordered-containers >=0.2
   - uri-bytestring >=0.2

services/galley/src/Galley/Run.hs

@@ -14,13 +14,16 @@
 --
 -- You should have received a copy of the GNU Affero General Public License along
 -- with this program. If not, see <https://www.gnu.org/licenses/>.
+{-# LANGUAGE NumericUnderscores #-}
 
 module Galley.Run
   ( run,
     mkApp,
   )
 where
 
+import AWS.Util (readAuthExpiration)
+import qualified Amazonka as AWS
 import Bilge.Request (requestIdName)
 import Cassandra (runClient, shutdown)
 import Cassandra.Schema (versionCheck)
@@ -31,6 +34,8 @@ import Control.Monad.Codensity
 import qualified Data.Aeson as Aeson
 import Data.Default
 import Data.Id
+import Data.Metrics (Metrics)
+import Data.Metrics.AWS (gaugeTokenRemaing)
 import qualified Data.Metrics.Middleware as M
 import Data.Metrics.Servant (servantPlusWAIPrometheusMiddleware)
 import Data.Misc (portNumber)
@@ -41,6 +46,7 @@ import Galley.API.Federation (FederationAPI, federationSitemap)
 import Galley.API.Internal
 import Galley.App
 import qualified Galley.App as App
+import Galley.Aws (awsEnv)
 import Galley.Cassandra
 import Galley.Monad
 import Galley.Options
@@ -71,6 +77,8 @@ run opts = lowerCodensity $ do
           (env ^. App.applog)
           (env ^. monitor)
 
+  forM_ (env ^. aEnv) $ \aws ->
+    void $ Codensity $ Async.withAsync $ collectAuthMetrics (env ^. monitor) (aws ^. awsEnv)
   void $ Codensity $ Async.withAsync $ runApp env deleteLoop
   void $ Codensity $ Async.withAsync $ runApp env refreshMetrics
   lift $ finally (runSettingsWithShutdown settings app 5) (shutdown (env ^. cstate))
@@ -152,3 +160,11 @@ refreshMetrics = do
     n <- Q.len q
     M.gaugeSet (fromIntegral n) (M.path "galley.deletequeue.len") m
     threadDelay 1000000
+
+collectAuthMetrics :: MonadIO m => Metrics -> AWS.Env -> m ()
+collectAuthMetrics m env = do
+  liftIO $
+    forever $ do
+      mbRemaining <- readAuthExpiration env
+      gaugeTokenRemaing m mbRemaining
+      threadDelay 1_000_000