diff --git a/.changeset/rude-eggs-pull.md b/.changeset/rude-eggs-pull.md new file mode 100644 index 0000000000000..08a76b14723e8 --- /dev/null +++ b/.changeset/rude-eggs-pull.md @@ -0,0 +1,9 @@ +--- +'@eth-optimism/common-ts': minor +'@eth-optimism/drippie-mon': minor +'@eth-optimism/fault-detector': minor +'@eth-optimism/message-relayer': minor +'@eth-optimism/replica-healthcheck': minor +--- + +BaseServiceV2 exposes service name and version as standard synthetic metric diff --git a/packages/common-ts/src/base-service/base-service-v2.ts b/packages/common-ts/src/base-service/base-service-v2.ts index c7075c0bbd812..896b62320c5c6 100644 --- a/packages/common-ts/src/base-service/base-service-v2.ts +++ b/packages/common-ts/src/base-service/base-service-v2.ts @@ -13,7 +13,7 @@ import bodyParser from 'body-parser' import morgan from 'morgan' import { Logger } from '../common/logger' -import { Metric } from './metrics' +import { Metric, Gauge, Counter } from './metrics' import { validators } from './validators' export type Options = { @@ -31,6 +31,7 @@ export type OptionsSpec = { validator: (spec?: Spec) => ValidatorSpec desc: string default?: TOptions[P] + secret?: boolean } } @@ -38,6 +39,11 @@ export type MetricsV2 = { [key: string]: Metric } +export type StandardMetrics = { + metadata: Gauge + unhandledErrors: Counter +} + export type MetricsSpec = { [P in keyof Required]: { type: new (configuration: any) => TMetrics[P] @@ -99,7 +105,7 @@ export abstract class BaseServiceV2< /** * Metrics. */ - protected readonly metrics: TMetrics + protected readonly metrics: TMetrics & StandardMetrics /** * Registry for prometheus metrics. @@ -137,6 +143,7 @@ export abstract class BaseServiceV2< */ constructor(params: { name: string + version: string optionsSpec: OptionsSpec metricsSpec: MetricsSpec options?: Partial @@ -170,6 +177,31 @@ export abstract class BaseServiceV2< }, } + // List of options that can safely be logged. + const publicOptionNames = Object.entries(params.optionsSpec) + .filter(([, spec]) => { + return spec.secret !== true + }) + .map(([key]) => { + return key + }) + + // Add default metrics to metrics spec. + ;(params.metricsSpec as any) = { + ...(params.metricsSpec || {}), + + // Users cannot set these options. + metadata: { + type: Gauge, + desc: 'Service metadata', + labels: ['name', 'version'].concat(publicOptionNames), + }, + unhandledErrors: { + type: Counter, + desc: 'Unhandled errors', + }, + } + /** * Special snake_case function which accounts for the common strings "L1" and "L2" which would * normally be split into "L_1" and "L_2" by the snake_case function. @@ -274,7 +306,7 @@ export abstract class BaseServiceV2< labelNames: spec.labels || [], }) return acc - }, {}) as TMetrics + }, {}) as TMetrics & StandardMetrics // Create the metrics server. this.metricsRegistry = prometheus.register @@ -309,6 +341,19 @@ export abstract class BaseServiceV2< // Handle stop signals. process.on('SIGTERM', stop) process.on('SIGINT', stop) + + // Set metadata synthetic metric. + this.metrics.metadata.set( + { + name: params.name, + version: params.version, + ...publicOptionNames.reduce((acc, key) => { + acc[key] = config.str(key) + return acc + }, {}), + }, + 1 + ) } /** @@ -330,21 +375,7 @@ export abstract class BaseServiceV2< app.use(bodyParser.urlencoded({ extended: true })) // Logging. - app.use( - morgan((tokens, req, res) => { - return [ - tokens.method(req, res), - tokens.url(req, res), - tokens.status(req, res), - JSON.stringify(req.body), - '\n', - tokens.res(req, res, 'content-length'), - '-', - tokens['response-time'](req, res), - 'ms', - ].join(' ') - }) - ) + app.use(morgan('short')) // Metrics. // Will expose a /metrics endpoint by default. @@ -397,6 +428,7 @@ export abstract class BaseServiceV2< try { await this.main() } catch (err) { + this.metrics.unhandledErrors.inc() this.logger.error('caught an unhandled exception', { message: err.message, stack: err.stack, diff --git a/packages/drippie-mon/src/service.ts b/packages/drippie-mon/src/service.ts index 4958483a84efa..b1286ebb45c51 100644 --- a/packages/drippie-mon/src/service.ts +++ b/packages/drippie-mon/src/service.ts @@ -14,7 +14,6 @@ type DrippieMonOptions = { } type DrippieMonMetrics = { - metadata: Gauge isExecutable: Gauge executedDripCount: Gauge unexpectedRpcErrors: Counter @@ -31,6 +30,8 @@ export class DrippieMonService extends BaseServiceV2< > { constructor(options?: Partial) { super({ + // eslint-disable-next-line @typescript-eslint/no-var-requires + version: require('../package.json').version, name: 'drippie-mon', loop: true, loopIntervalMs: 60_000, @@ -39,6 +40,7 @@ export class DrippieMonService extends BaseServiceV2< rpc: { validator: validators.provider, desc: 'Provider for network where Drippie is deployed', + secret: true, }, drippieAddress: { validator: validators.str, @@ -46,11 +48,6 @@ export class DrippieMonService extends BaseServiceV2< }, }, metricsSpec: { - metadata: { - type: Gauge, - desc: 'Drippie Monitor metadata', - labels: ['version', 'address'], - }, isExecutable: { type: Gauge, desc: 'Whether or not the drip is currently executable', @@ -76,15 +73,6 @@ export class DrippieMonService extends BaseServiceV2< DrippieArtifact.abi, this.options.rpc ) - - this.metrics.metadata.set( - { - // eslint-disable-next-line @typescript-eslint/no-var-requires - version: require('../package.json').version, - address: this.options.drippieAddress, - }, - 1 - ) } protected async main(): Promise { diff --git a/packages/fault-detector/src/service.ts b/packages/fault-detector/src/service.ts index c859a2485850e..496eb1263ee7b 100644 --- a/packages/fault-detector/src/service.ts +++ b/packages/fault-detector/src/service.ts @@ -32,6 +32,8 @@ type State = { export class FaultDetector extends BaseServiceV2 { constructor(options?: Partial) { super({ + // eslint-disable-next-line @typescript-eslint/no-var-requires + version: require('../package.json').version, name: 'fault-detector', loop: true, loopIntervalMs: 1000, @@ -40,10 +42,12 @@ export class FaultDetector extends BaseServiceV2 { l1RpcProvider: { validator: validators.provider, desc: 'Provider for interacting with L1', + secret: true, }, l2RpcProvider: { validator: validators.provider, desc: 'Provider for interacting with L2', + secret: true, }, startBatchIndex: { validator: validators.num, diff --git a/packages/message-relayer/src/service.ts b/packages/message-relayer/src/service.ts index 2a1e692eca9c9..d5ddd68376029 100644 --- a/packages/message-relayer/src/service.ts +++ b/packages/message-relayer/src/service.ts @@ -37,20 +37,25 @@ export class MessageRelayerService extends BaseServiceV2< > { constructor(options?: Partial) { super({ - name: 'Message_Relayer', + // eslint-disable-next-line @typescript-eslint/no-var-requires + version: require('../package.json').version, + name: 'message-relayer', options, optionsSpec: { l1RpcProvider: { validator: validators.provider, desc: 'Provider for interacting with L1.', + secret: true, }, l2RpcProvider: { validator: validators.provider, desc: 'Provider for interacting with L2.', + secret: true, }, l1Wallet: { validator: validators.wallet, desc: 'Wallet used to interact with L1.', + secret: true, }, fromL2TransactionIndex: { validator: validators.num, diff --git a/packages/replica-healthcheck/src/service.ts b/packages/replica-healthcheck/src/service.ts index 69927d9564857..980748e1ab753 100644 --- a/packages/replica-healthcheck/src/service.ts +++ b/packages/replica-healthcheck/src/service.ts @@ -32,17 +32,21 @@ export class HealthcheckService extends BaseServiceV2< > { constructor(options?: Partial) { super({ - name: 'Healthcheck', + // eslint-disable-next-line @typescript-eslint/no-var-requires + version: require('../package.json').version, + name: 'healthcheck', loopIntervalMs: 5000, options, optionsSpec: { referenceRpcProvider: { validator: validators.provider, desc: 'Provider for interacting with L1', + secret: true, }, targetRpcProvider: { validator: validators.provider, desc: 'Provider for interacting with L2', + secret: true, }, onDivergenceWaitMs: { validator: validators.num,