diff --git a/lib/instrumentation/index.spec.ts b/lib/instrumentation/index.spec.ts index 4cf986cdcf3..d8482d0490a 100644 --- a/lib/instrumentation/index.spec.ts +++ b/lib/instrumentation/index.spec.ts @@ -4,6 +4,7 @@ import { NodeTracerProvider, SimpleSpanProcessor, } from '@opentelemetry/sdk-trace-node'; +import { GetDatasourceReleasesSpanProcessor } from '../modules/datasource/span-processor.ts'; import { GitOperationSpanProcessor } from '../util/git/span-processor.ts'; import { disableInstrumentations, @@ -57,13 +58,14 @@ describe('instrumentation/index', () => { _activeSpanProcessor: { _spanProcessors: [ new GitOperationSpanProcessor(), + new GetDatasourceReleasesSpanProcessor(), expect.any(SimpleSpanProcessor), ], }, }); }); - it('registers GitOperationSpanProcessor regardless of tracing being enabled', () => { + it('registers GitOperationSpanProcessor, GetDatasourceReleasesSpanProcessor regardless of tracing being enabled', () => { // intentionally don't set it delete process.env.RENOVATE_TRACING_CONSOLE_EXPORTER; delete process.env.OTEL_EXPORTER_OTLP_ENDPOINT; @@ -77,6 +79,7 @@ describe('instrumentation/index', () => { _activeSpanProcessor: { _spanProcessors: expect.arrayContaining([ new GitOperationSpanProcessor(), + new GetDatasourceReleasesSpanProcessor(), ]), }, }); @@ -96,6 +99,7 @@ describe('instrumentation/index', () => { _activeSpanProcessor: { _spanProcessors: [ new GitOperationSpanProcessor(), + new GetDatasourceReleasesSpanProcessor(), { _exporter: { _delegate: { @@ -129,6 +133,7 @@ describe('instrumentation/index', () => { _activeSpanProcessor: { _spanProcessors: [ new GitOperationSpanProcessor(), + new GetDatasourceReleasesSpanProcessor(), { _exporter: {} }, { _exporter: { diff --git a/lib/instrumentation/index.ts b/lib/instrumentation/index.ts index 90e4208451e..a72ff13f3c5 100644 --- a/lib/instrumentation/index.ts +++ b/lib/instrumentation/index.ts @@ -26,6 +26,7 @@ import { } from '@opentelemetry/semantic-conventions'; import { isPromise } from '@sindresorhus/is'; import { pkg } from '../expose.ts'; +import { GetDatasourceReleasesSpanProcessor } from '../modules/datasource/span-processor.ts'; import { GitOperationSpanProcessor } from '../util/git/span-processor.ts'; import { getResourceDetectors } from './detectors.ts'; import type { RenovateSpanOptions } from './types.ts'; @@ -39,7 +40,10 @@ import { let instrumentations: Instrumentation[] = []; export function init(): void { - const spanProcessors: SpanProcessor[] = [new GitOperationSpanProcessor()]; + const spanProcessors: SpanProcessor[] = [ + new GitOperationSpanProcessor(), + new GetDatasourceReleasesSpanProcessor(), + ]; if (!isTracingEnabled()) { const traceProvider = new NodeTracerProvider({ spanProcessors }); diff --git a/lib/instrumentation/types.ts b/lib/instrumentation/types.ts index 794049a3279..d7825b05ef6 100644 --- a/lib/instrumentation/types.ts +++ b/lib/instrumentation/types.ts @@ -1,4 +1,5 @@ import type { Attributes, SpanKind, SpanOptions } from '@opentelemetry/api'; +import type { ATTR_CODE_FUNCTION_NAME } from '@opentelemetry/semantic-conventions'; import type { RenovateSplit } from '../config/types.ts'; import type { BunyanRecord } from '../logger/types.ts'; import type { PackageFile } from '../modules/manager/types.ts'; @@ -12,6 +13,10 @@ export type RenovateSpanOptions = { export type RenovateSpanAttributes = { [ATTR_RENOVATE_SPLIT]?: RenovateSplit; [ATTR_VCS_GIT_OPERATION_TYPE]?: GitOperationType; + [ATTR_CODE_FUNCTION_NAME]?: string; + [ATTR_RENOVATE_DATASOURCE]?: string; + [ATTR_RENOVATE_REGISTRY_URL]?: string; + [ATTR_RENOVATE_PACKAGE_NAME]?: string; } & Attributes; /** @@ -68,6 +73,21 @@ export interface DependencyStatus { export const ATTR_RENOVATE_SPLIT = 'renovate.split'; +/** + * The name of a Renovate datasource (ex: `github-tags`, `npm`, `docker`, etc). + */ +export const ATTR_RENOVATE_DATASOURCE = 'renovate.datasource'; + +/** + * The registry URL of a registry URL as might be used with a datasource and package name. + */ +export const ATTR_RENOVATE_REGISTRY_URL = 'renovate.registryUrl'; + +/** + * The package name of a package. + */ +export const ATTR_RENOVATE_PACKAGE_NAME = 'renovate.packageName'; + /** * the Git Version Control System (VCS)'s Operation Type * diff --git a/lib/modules/datasource/index.ts b/lib/modules/datasource/index.ts index fb76da30004..b9ff61026a1 100644 --- a/lib/modules/datasource/index.ts +++ b/lib/modules/datasource/index.ts @@ -1,7 +1,14 @@ +import { ATTR_CODE_FUNCTION_NAME } from '@opentelemetry/semantic-conventions'; import { isFunction, isNonEmptyArray, isString } from '@sindresorhus/is'; import { dequal } from 'dequal'; import { GlobalConfig } from '../../config/global.ts'; import { HOST_DISABLED } from '../../constants/error-messages.ts'; +import { instrument } from '../../instrumentation/index.ts'; +import { + ATTR_RENOVATE_DATASOURCE, + ATTR_RENOVATE_PACKAGE_NAME, + ATTR_RENOVATE_REGISTRY_URL, +} from '../../instrumentation/types.ts'; import { logger } from '../../logger/index.ts'; import { ExternalHostError } from '../../types/errors/external-host-error.ts'; import { coerceArray } from '../../util/array.ts'; @@ -86,7 +93,18 @@ async function getRegistryReleases( DatasourceCacheStats.miss(datasource.id, registryUrl, config.packageName); } - const res = await datasource.getReleases({ ...config, registryUrl }); + const res = await instrument( + 'getReleases', + () => datasource.getReleases({ ...config, registryUrl }), + { + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'getReleases', + [ATTR_RENOVATE_DATASOURCE]: datasource.id, + [ATTR_RENOVATE_REGISTRY_URL]: registryUrl, + [ATTR_RENOVATE_PACKAGE_NAME]: config.packageName, + }, + }, + ); if (res?.releases.length) { res.registryUrl ??= registryUrl; } @@ -371,7 +389,18 @@ async function fetchReleases( dep = await mergeRegistries(config, datasource, registryUrls); } } else { - dep = await datasource.getReleases(config); + dep = await instrument( + 'getReleases', + () => datasource.getReleases(config), + { + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'getReleases', + [ATTR_RENOVATE_DATASOURCE]: datasource.id, + [ATTR_RENOVATE_REGISTRY_URL]: config.registryUrl ?? '', + [ATTR_RENOVATE_PACKAGE_NAME]: config.packageName, + }, + }, + ); } } catch (err) { if (err.message === HOST_DISABLED || err.err?.message === HOST_DISABLED) { diff --git a/lib/modules/datasource/span-processor.spec.ts b/lib/modules/datasource/span-processor.spec.ts new file mode 100644 index 00000000000..e89c3dd5694 --- /dev/null +++ b/lib/modules/datasource/span-processor.spec.ts @@ -0,0 +1,138 @@ +import type { ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { ATTR_CODE_FUNCTION_NAME } from '@opentelemetry/semantic-conventions'; +import { partial } from '~test/util.ts'; +import { + ATTR_RENOVATE_DATASOURCE, + ATTR_RENOVATE_PACKAGE_NAME, + ATTR_RENOVATE_REGISTRY_URL, +} from '../../instrumentation/types.ts'; +import * as stats from '../../util/stats.ts'; +import { GetDatasourceReleasesSpanProcessor } from './span-processor.ts'; + +vi.mock('../../util/stats.ts'); + +describe('modules/datasource/span-processor', () => { + describe('GetDatasourceReleasesSpanProcessor', () => { + it('creates an instance', async () => { + const processor = new GetDatasourceReleasesSpanProcessor(); + expect(processor).toBeInstanceOf(GetDatasourceReleasesSpanProcessor); + await expect(processor.forceFlush()).resolves.toBeUndefined(); + expect(processor.onStart(partial(), partial())).toBeUndefined(); + await expect(processor.shutdown()).resolves.toBeUndefined(); + }); + + it('writes span datapoints to GetDatasourceReleasesStats', () => { + const writeMock = vi.mocked(stats.GetDatasourceReleasesStats.write); + + const processor = new GetDatasourceReleasesSpanProcessor(); + processor.onEnd( + partial({ + ended: true, + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'getReleases', + [ATTR_RENOVATE_DATASOURCE]: 'npm', + [ATTR_RENOVATE_REGISTRY_URL]: 'https://registry.npmjs.org', + [ATTR_RENOVATE_PACKAGE_NAME]: 'lodash', + }, + duration: [1, 500_123_000], // 1.500123 seconds = 1500.123ms + }), + ); + + expect(writeMock).toHaveBeenCalledOnce(); + expect(writeMock).toHaveBeenCalledWith( + 'npm', + 'https://registry.npmjs.org', + 'lodash', + 1500.123, + ); + }); + + it('defaults registryUrl to an empty string if not provided', () => { + const writeMock = vi.mocked(stats.GetDatasourceReleasesStats.write); + + const processor = new GetDatasourceReleasesSpanProcessor(); + processor.onEnd( + partial({ + ended: true, + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'getReleases', + [ATTR_RENOVATE_DATASOURCE]: 'npm', + [ATTR_RENOVATE_PACKAGE_NAME]: 'lodash', + }, + duration: [1, 0], + }), + ); + + expect(writeMock).toHaveBeenCalledOnce(); + expect(writeMock).toHaveBeenCalledWith('npm', '', 'lodash', 1000); + }); + + interface NoWriteTestCase { + name: string; + span: Partial; + } + + const noWriteTestCases: NoWriteTestCase[] = [ + { + name: 'span is not ended', + span: { + ended: false, + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'getReleases', + [ATTR_RENOVATE_DATASOURCE]: 'npm', + [ATTR_RENOVATE_REGISTRY_URL]: 'https://registry.npmjs.org', + [ATTR_RENOVATE_PACKAGE_NAME]: 'lodash', + }, + duration: [1, 0], + }, + }, + { + name: 'function name is not getReleases', + span: { + ended: true, + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'somethingElse', + [ATTR_RENOVATE_DATASOURCE]: 'npm', + [ATTR_RENOVATE_REGISTRY_URL]: 'https://registry.npmjs.org', + [ATTR_RENOVATE_PACKAGE_NAME]: 'lodash', + }, + duration: [1, 0], + }, + }, + { + name: 'datasource is not provided', + span: { + ended: true, + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'getReleases', + [ATTR_RENOVATE_REGISTRY_URL]: 'https://registry.npmjs.org', + [ATTR_RENOVATE_PACKAGE_NAME]: 'lodash', + }, + duration: [1, 0], + }, + }, + { + name: 'package name is not provided', + span: { + ended: true, + attributes: { + [ATTR_CODE_FUNCTION_NAME]: 'getReleases', + [ATTR_RENOVATE_DATASOURCE]: 'npm', + [ATTR_RENOVATE_REGISTRY_URL]: 'https://registry.npmjs.org', + }, + duration: [1, 0], + }, + }, + ]; + + test.each(noWriteTestCases)( + 'does not write span datapoints to GetDatasourceReleasesStats if $name', + ({ span }) => { + const writeMock = vi.mocked(stats.GetDatasourceReleasesStats.write); + const processor = new GetDatasourceReleasesSpanProcessor(); + processor.onEnd(partial(span)); + expect(writeMock).not.toHaveBeenCalled(); + }, + ); + }); +}); diff --git a/lib/modules/datasource/span-processor.ts b/lib/modules/datasource/span-processor.ts new file mode 100644 index 00000000000..879824a6e53 --- /dev/null +++ b/lib/modules/datasource/span-processor.ts @@ -0,0 +1,56 @@ +import type { Context } from '@opentelemetry/api'; +import type { + ReadableSpan, + Span, + SpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import { ATTR_CODE_FUNCTION_NAME } from '@opentelemetry/semantic-conventions'; +import { + ATTR_RENOVATE_DATASOURCE, + ATTR_RENOVATE_PACKAGE_NAME, + ATTR_RENOVATE_REGISTRY_URL, +} from '../../instrumentation/types.ts'; +import { GetDatasourceReleasesStats } from '../../util/stats.ts'; + +export class GetDatasourceReleasesSpanProcessor implements SpanProcessor { + forceFlush(): Promise { + return Promise.resolve(); + } + + onStart(_span: Span, _parentContext: Context): void { + // no implementation + } + + onEnd(span: ReadableSpan): void { + if (!span.ended) { + return; + } + + if (span.attributes[ATTR_CODE_FUNCTION_NAME] !== 'getReleases') { + return; + } + + const datasource = span.attributes[ATTR_RENOVATE_DATASOURCE] as string; + const registryUrl = (span.attributes[ATTR_RENOVATE_REGISTRY_URL] ?? + '') as string; + const packageName = span.attributes[ATTR_RENOVATE_PACKAGE_NAME] as string; + + if (!datasource || !packageName) { + return; + } + + // duration[0] is seconds, duration[1] is nanoseconds. + const durationMs = span.duration[0] * 1000 + span.duration[1] / 1_000_000; + + GetDatasourceReleasesStats.write( + datasource, + registryUrl, + packageName, + durationMs, + ); + } + + shutdown(): Promise { + return Promise.resolve(); + } +} diff --git a/lib/util/stats.spec.ts b/lib/util/stats.spec.ts index b4bbe846b4d..e91ebed4d6d 100644 --- a/lib/util/stats.spec.ts +++ b/lib/util/stats.spec.ts @@ -3,6 +3,7 @@ import * as memCache from './cache/memory/index.ts'; import { AbandonedPackageStats, DatasourceCacheStats, + GetDatasourceReleasesStats, GitOperationStats, HttpCacheStats, HttpStats, @@ -139,6 +140,436 @@ describe('util/stats', () => { }); }); + describe('GetDatasourceReleasesStats', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('returns empty report', () => { + const res = GetDatasourceReleasesStats.getReport(); + expect(res).toEqual({ + stats: { + avgMs: 0, + count: 0, + maxMs: 0, + medianMs: 0, + totalMs: 0, + }, + datasources: {}, + }); + }); + + it('writes data points', () => { + GetDatasourceReleasesStats.write( + 'npm', + 'https://registry.npmjs.org', + 'lodash', + 100, + ); + GetDatasourceReleasesStats.write( + 'npm', + 'https://registry.npmjs.org', + 'lodash', + 200, + ); + GetDatasourceReleasesStats.write( + 'npm', + 'https://jfrog.company.com/artifactory', + 'foo', + 400, + ); + GetDatasourceReleasesStats.write( + 'docker', + 'https://registry.docker.com', + 'alpine', + 1000, + ); + GetDatasourceReleasesStats.write( + 'docker', + 'https://registry.docker.com', + 'memcached', + 2000, + ); + GetDatasourceReleasesStats.write( + 'docker', + 'https://registry.docker.com', + 'istio/pilot', + 3000, + ); + + const res = GetDatasourceReleasesStats.getReport(); + expect(res).toEqual({ + stats: { + avgMs: 1117, + count: 6, + maxMs: 3000, + medianMs: 1000, + totalMs: 6700, + }, + datasources: { + npm: { + stats: { + avgMs: 233, + count: 3, + maxMs: 400, + medianMs: 200, + totalMs: 700, + }, + registryUrls: { + 'https://registry.npmjs.org': { + stats: { + avgMs: 150, + count: 2, + maxMs: 200, + medianMs: 200, + totalMs: 300, + }, + packages: { + lodash: { + avgMs: 150, + count: 2, + maxMs: 200, + medianMs: 200, + totalMs: 300, + }, + }, + }, + 'https://jfrog.company.com/artifactory': { + stats: { + avgMs: 400, + count: 1, + maxMs: 400, + medianMs: 400, + totalMs: 400, + }, + packages: { + foo: { + avgMs: 400, + count: 1, + maxMs: 400, + medianMs: 400, + totalMs: 400, + }, + }, + }, + }, + }, + docker: { + stats: { + avgMs: 2000, + count: 3, + maxMs: 3000, + medianMs: 2000, + totalMs: 6000, + }, + registryUrls: { + 'https://registry.docker.com': { + stats: { + avgMs: 2000, + count: 3, + maxMs: 3000, + medianMs: 2000, + totalMs: 6000, + }, + packages: { + alpine: { + avgMs: 1000, + count: 1, + maxMs: 1000, + medianMs: 1000, + totalMs: 1000, + }, + memcached: { + avgMs: 2000, + count: 1, + maxMs: 2000, + medianMs: 2000, + totalMs: 2000, + }, + 'istio/pilot': { + avgMs: 3000, + count: 1, + maxMs: 3000, + medianMs: 3000, + totalMs: 3000, + }, + }, + }, + }, + }, + }, + }); + }); + + it('wraps a function', async () => { + const res = await GetDatasourceReleasesStats.wrap( + 'npm', + 'https://registry.npmjs.org', + 'lodash', + () => { + vi.advanceTimersByTime(100); + return Promise.resolve('foo'); + }, + ); + + expect(res).toBe('foo'); + expect(GetDatasourceReleasesStats.getReport()).toEqual({ + stats: { + avgMs: 100, + count: 1, + maxMs: 100, + medianMs: 100, + totalMs: 100, + }, + datasources: { + npm: { + stats: { + avgMs: 100, + count: 1, + maxMs: 100, + medianMs: 100, + totalMs: 100, + }, + registryUrls: { + 'https://registry.npmjs.org': { + stats: { + avgMs: 100, + count: 1, + maxMs: 100, + medianMs: 100, + totalMs: 100, + }, + packages: { + lodash: { + avgMs: 100, + count: 1, + maxMs: 100, + medianMs: 100, + totalMs: 100, + }, + }, + }, + }, + }, + }, + }); + }); + + it('logs report', () => { + GetDatasourceReleasesStats.write( + 'npm', + 'https://registry.npmjs.org', + 'lodash', + 100, + ); + GetDatasourceReleasesStats.write( + 'npm', + 'https://registry.npmjs.org', + 'lodash', + 200, + ); + GetDatasourceReleasesStats.write( + 'npm', + 'https://jfrog.company.com/artifactory', + 'foo', + 400, + ); + GetDatasourceReleasesStats.write( + 'docker', + 'https://registry.docker.com', + 'alpine', + 1000, + ); + GetDatasourceReleasesStats.write( + 'docker', + 'https://registry.docker.com', + 'memcached', + 2000, + ); + GetDatasourceReleasesStats.write( + 'docker', + 'https://registry.docker.com', + 'istio/pilot', + 3000, + ); + + GetDatasourceReleasesStats.report(); + + expect(logger.logger.trace).toHaveBeenCalledTimes(1); + const [traceData, traceMsg] = logger.logger.trace.mock.calls[0]; + expect(traceMsg).toBe('getReleases statistics with packages'); + expect(traceData).toEqual({ + stats: { + avgMs: 1117, + count: 6, + maxMs: 3000, + medianMs: 1000, + totalMs: 6700, + }, + datasources: { + npm: { + stats: { + avgMs: 233, + count: 3, + maxMs: 400, + medianMs: 200, + totalMs: 700, + }, + registryUrls: { + 'https://registry.npmjs.org': { + stats: { + avgMs: 150, + count: 2, + maxMs: 200, + medianMs: 200, + totalMs: 300, + }, + packages: { + lodash: { + avgMs: 150, + count: 2, + maxMs: 200, + medianMs: 200, + totalMs: 300, + }, + }, + }, + 'https://jfrog.company.com/artifactory': { + stats: { + avgMs: 400, + count: 1, + maxMs: 400, + medianMs: 400, + totalMs: 400, + }, + packages: { + foo: { + avgMs: 400, + count: 1, + maxMs: 400, + medianMs: 400, + totalMs: 400, + }, + }, + }, + }, + }, + docker: { + stats: { + avgMs: 2000, + count: 3, + maxMs: 3000, + medianMs: 2000, + totalMs: 6000, + }, + registryUrls: { + 'https://registry.docker.com': { + stats: { + avgMs: 2000, + count: 3, + maxMs: 3000, + medianMs: 2000, + totalMs: 6000, + }, + packages: { + alpine: { + avgMs: 1000, + count: 1, + maxMs: 1000, + medianMs: 1000, + totalMs: 1000, + }, + memcached: { + avgMs: 2000, + count: 1, + maxMs: 2000, + medianMs: 2000, + totalMs: 2000, + }, + 'istio/pilot': { + avgMs: 3000, + count: 1, + maxMs: 3000, + medianMs: 3000, + totalMs: 3000, + }, + }, + }, + }, + }, + }, + }); + + expect(logger.logger.debug).toHaveBeenCalledTimes(1); + const [debugData, debugMsg] = logger.logger.debug.mock.calls[0]; + expect(debugMsg).toBe('getReleases statistics summary'); + expect(debugData).toEqual({ + stats: { + avgMs: 1117, + count: 6, + maxMs: 3000, + medianMs: 1000, + totalMs: 6700, + }, + datasources: { + npm: { + stats: { + avgMs: 233, + count: 3, + maxMs: 400, + medianMs: 200, + totalMs: 700, + }, + registryUrls: { + 'https://registry.npmjs.org': { + stats: { + avgMs: 150, + count: 2, + maxMs: 200, + medianMs: 200, + totalMs: 300, + }, + }, + 'https://jfrog.company.com/artifactory': { + stats: { + avgMs: 400, + count: 1, + maxMs: 400, + medianMs: 400, + totalMs: 400, + }, + }, + }, + }, + docker: { + stats: { + avgMs: 2000, + count: 3, + maxMs: 3000, + medianMs: 2000, + totalMs: 6000, + }, + registryUrls: { + 'https://registry.docker.com': { + stats: { + avgMs: 2000, + count: 3, + maxMs: 3000, + medianMs: 2000, + totalMs: 6000, + }, + }, + }, + }, + }, + }); + }); + }); + describe('PackageCacheStats', () => { beforeEach(() => { vi.useFakeTimers(); diff --git a/lib/util/stats.ts b/lib/util/stats.ts index e6e5b05b998..3bfc4dbd83c 100644 --- a/lib/util/stats.ts +++ b/lib/util/stats.ts @@ -57,6 +57,168 @@ export class LookupStats { } } +interface GetReleasesDataPoint { + datasource: string; + registryUrl: string; + packageName: string; + duration: number; +} + +interface getReleaseStatsInternalPackages { + stats: T; + packages: Record; +} + +/** + * Internal structure that represents the hierarchical structure of the data. We use this + * to handle the duration datapoints, and then convert to the final report structure. + */ +interface getReleaseStatsInternal { + // Overall stats + stats: T; + datasources: Record< + string, + { + // Datasource stats. + stats: T; + registryUrls: Record< + string, + [P] extends [never] + ? Omit, 'packages'> + : getReleaseStatsInternalPackages + >; + } + >; +} + +export type GetReleaseStatsReport = getReleaseStatsInternal; + +// Short report does not include package stats. +export type GetReleaseStatsReportShort = getReleaseStatsInternal< + TimingStatsReport, + never +>; + +export class GetDatasourceReleasesStats { + static write( + datasource: string, + registryUrl: string, + packageName: string, + duration: number, + ): void { + const data = + memCache.get('get-releases-stats') ?? []; + data.push({ datasource, registryUrl, packageName, duration }); + memCache.set('get-releases-stats', data); + } + + static async wrap( + datasource: string, + registryUrl: string, + packageName: string, + callback: () => Promise, + ): Promise { + const start = Date.now(); + const result = await callback(); + const duration = Date.now() - start; + this.write(datasource, registryUrl, packageName, duration); + return result; + } + + static getReport(): GetReleaseStatsReport { + const data = + memCache.get('get-releases-stats') ?? []; + + // Process all datapoints into a hierarchical structure of datasource, registry url, and package name. + const durationData: getReleaseStatsInternal = { + stats: [], + datasources: {}, + }; + for (const { datasource, registryUrl, packageName, duration } of data) { + durationData.stats.push(duration); + + durationData.datasources[datasource] ??= { stats: [], registryUrls: {} }; + durationData.datasources[datasource].stats.push(duration); + + durationData.datasources[datasource].registryUrls[registryUrl] ??= { + stats: [], + packages: {}, + }; + durationData.datasources[datasource].registryUrls[registryUrl].stats.push( + duration, + ); + + durationData.datasources[datasource].registryUrls[registryUrl].packages[ + packageName + ] ??= []; + durationData.datasources[datasource].registryUrls[registryUrl].packages[ + packageName + ].push(duration); + } + + const report: GetReleaseStatsReport = { + stats: makeTimingReport(durationData.stats), + datasources: {}, + }; + + for (const [datasource, datasourceData] of Object.entries( + durationData.datasources, + )) { + report.datasources[datasource] = { + stats: makeTimingReport(datasourceData.stats), + registryUrls: {}, + }; + + for (const [registryUrl, registryUrlData] of Object.entries( + datasourceData.registryUrls, + )) { + report.datasources[datasource].registryUrls[registryUrl] = { + stats: makeTimingReport(registryUrlData.stats), + packages: {}, + }; + + for (const [packageName, packageNameData] of Object.entries( + registryUrlData.packages, + )) { + report.datasources[datasource].registryUrls[registryUrl].packages[ + packageName + ] = makeTimingReport(packageNameData); + } + } + } + + return report; + } + + static report(): void { + const report = this.getReport(); + + const shortReport: GetReleaseStatsReportShort = { + stats: report.stats, + datasources: {}, + }; + + for (const [datasource, datasourceData] of Object.entries( + report.datasources, + )) { + shortReport.datasources[datasource] = { + stats: datasourceData.stats, + registryUrls: {}, + }; + for (const [registryUrl, registryUrlData] of Object.entries( + datasourceData.registryUrls, + )) { + shortReport.datasources[datasource].registryUrls[registryUrl] = { + stats: registryUrlData.stats, + }; + } + } + + logger.trace(report, 'getReleases statistics with packages'); + logger.debug(shortReport, 'getReleases statistics summary'); + } +} + type PackageCacheData = number[]; export class PackageCacheStats { diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index ca2939fed93..56e0ee93be9 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -24,6 +24,7 @@ import { addSplit, getSplits, splitInit } from '../../util/split.ts'; import { AbandonedPackageStats, DatasourceCacheStats, + GetDatasourceReleasesStats, GitOperationStats, HttpCacheStats, HttpStats, @@ -207,6 +208,7 @@ export async function renovateRepository( HttpStats.report(); HttpCacheStats.report(); LookupStats.report(); + GetDatasourceReleasesStats.report(); ObsoleteCacheHitLogger.report(); AbandonedPackageStats.report(); GitOperationStats.report();