diff --git a/lib/modules/datasource/maven/cache.spec.ts b/lib/modules/datasource/maven/cache.spec.ts new file mode 100644 index 00000000000..e8e7b9887bd --- /dev/null +++ b/lib/modules/datasource/maven/cache.spec.ts @@ -0,0 +1,275 @@ +import { codeBlock } from 'common-tags'; +import { mockDeep } from 'vitest-mock-extended'; +import { XmlDocument } from 'xmldoc'; +import { Fixtures } from '~test/fixtures.ts'; +import * as httpMock from '~test/http-mock.ts'; +import * as _packageCache from '../../../util/cache/package/index.ts'; +import type { HttpCache } from '../../../util/http/cache/schema.ts'; +import { id as versioning } from '../../versioning/maven/index.ts'; +import { getPkgReleases } from '../index.ts'; +import { MavenDatasource } from './index.ts'; +import { CachedMavenXml } from './schema.ts'; + +vi.mock('../../../util/cache/package/index.ts', () => mockDeep()); + +const packageCache = vi.mocked(_packageCache); + +const packageName = 'org.example:package'; +const registryUrl = 'https://repo.maven.apache.org/maven2'; +const metadataUrl = + 'https://repo.maven.apache.org/maven2/org/example/package/maven-metadata.xml'; +const pomUrl = + 'https://repo.maven.apache.org/maven2/org/example/package/2.0.0/package-2.0.0.pom'; + +describe('modules/datasource/maven/cache', () => { + let cache: Record; + + beforeEach(() => { + vi.resetAllMocks(); + cache = {}; + + packageCache.get.mockImplementation((_namespace, key) => + Promise.resolve(cache[key] as never), + ); + packageCache.getCacheType.mockReturnValue(undefined); + packageCache.setWithRawTtl.mockImplementation((_namespace, key, value) => { + cache[key] = value as HttpCache; + return Promise.resolve(null as never); + }); + }); + + it('persists trimmed metadata and pom bodies', async () => { + httpMock + .scope(registryUrl) + .get('/org/example/package/maven-metadata.xml') + .reply(200, Fixtures.get('metadata.xml')) + .get('/org/example/package/2.0.0/package-2.0.0.pom') + .reply(200, Fixtures.get('pom.xml')); + + const result = await getPkgReleases({ + datasource: MavenDatasource.id, + packageName, + registryUrls: [registryUrl], + versioning, + }); + + expect(result).toMatchObject({ + homepage: 'https://package.example.org/about', + packageScope: 'org.example', + tags: { + latest: '2.0.0', + release: '2.0.0', + }, + }); + + const metadataCache = cache[metadataUrl]!; + const metadata = new XmlDocument( + (metadataCache.httpResponse as { body: string }).body, + ); + expect(metadata.valueWithPath('groupId')).toBeUndefined(); + expect(metadata.valueWithPath('artifactId')).toBeUndefined(); + expect( + metadata.descendantWithPath('versioning.lastUpdated'), + ).toBeUndefined(); + expect(metadata.valueWithPath('versioning.latest')).toBe('2.0.0'); + expect(metadata.valueWithPath('versioning.release')).toBe('2.0.0'); + + const pomCache = cache[pomUrl]!; + const pom = new XmlDocument( + (pomCache.httpResponse as { body: string }).body, + ); + expect(pom.valueWithPath('groupId')).toBe('org.example'); + expect(pom.valueWithPath('url')).toBe('https://package.example.org/about'); + expect(pom.valueWithPath('name')).toBeUndefined(); + expect(pom.valueWithPath('description')).toBeUndefined(); + }); + + it('serves cached trimmed XML without refetching', async () => { + const timestamp = new Date().toISOString(); + cache[metadataUrl] = { + etag: 'etag', + httpResponse: { + statusCode: 200, + body: CachedMavenXml.parse(Fixtures.get('metadata.xml')), + }, + timestamp, + }; + cache[pomUrl] = { + etag: 'etag', + httpResponse: { + statusCode: 200, + body: CachedMavenXml.parse(Fixtures.get('pom.xml')), + }, + timestamp, + }; + + const result = await getPkgReleases({ + datasource: MavenDatasource.id, + packageName, + registryUrls: [registryUrl], + versioning, + }); + + expect(result).toMatchObject({ + homepage: 'https://package.example.org/about', + packageScope: 'org.example', + tags: { + latest: '2.0.0', + release: '2.0.0', + }, + }); + expect(httpMock.getTrace()).toEqual([]); + expect(packageCache.setWithRawTtl).not.toHaveBeenCalled(); + }); + + it('preserves empty relocation markers on cache hits', async () => { + const pomWithEmptyRelocation = codeBlock` + + + + + + `; + const timestamp = new Date().toISOString(); + + cache[metadataUrl] = { + etag: 'etag', + httpResponse: { + statusCode: 200, + body: CachedMavenXml.parse(Fixtures.get('metadata.xml')), + }, + timestamp, + }; + cache[pomUrl] = { + etag: 'etag', + httpResponse: { + statusCode: 200, + body: CachedMavenXml.parse(pomWithEmptyRelocation), + }, + timestamp, + }; + + const result = await getPkgReleases({ + datasource: MavenDatasource.id, + packageName, + registryUrls: [registryUrl], + versioning, + }); + + expect(result).toMatchObject({ + replacementName: 'org.example:package', + replacementVersion: '2.0.0', + }); + expect(httpMock.getTrace()).toEqual([]); + }); + + it('revalidates trimmed cached XML after 304 responses', async () => { + const staleTimestamp = '2024-01-01T00:00:00.000Z'; + + cache[metadataUrl] = { + etag: 'metadata-etag', + lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT', + httpResponse: { + statusCode: 200, + headers: { etag: 'metadata-etag' }, + body: CachedMavenXml.parse(Fixtures.get('metadata.xml')), + }, + timestamp: staleTimestamp, + }; + cache[pomUrl] = { + etag: 'pom-etag', + lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT', + httpResponse: { + statusCode: 200, + headers: { etag: 'pom-etag' }, + body: CachedMavenXml.parse(Fixtures.get('pom.xml')), + }, + timestamp: staleTimestamp, + }; + + httpMock + .scope(registryUrl) + .get('/org/example/package/maven-metadata.xml') + .reply(304) + .get('/org/example/package/2.0.0/package-2.0.0.pom') + .reply(304); + + const result = await getPkgReleases({ + datasource: MavenDatasource.id, + packageName, + registryUrls: [registryUrl], + versioning, + }); + + expect(result).toMatchObject({ + homepage: 'https://package.example.org/about', + packageScope: 'org.example', + tags: { + latest: '2.0.0', + release: '2.0.0', + }, + }); + expect(packageCache.setWithRawTtl).toHaveBeenCalledTimes(2); + expect(cache[metadataUrl].timestamp).not.toBe(staleTimestamp); + expect(cache[pomUrl].timestamp).not.toBe(staleTimestamp); + }); + + it('serves cached trimmed snapshot XML without refetching', async () => { + const timestamp = new Date().toISOString(); + const snapshotMetadataUrl = + 'https://repo.maven.apache.org/maven2/org/example/package/1.0.3-SNAPSHOT/maven-metadata.xml'; + const snapshotPomUrl = + 'https://repo.maven.apache.org/maven2/org/example/package/1.0.3-SNAPSHOT/package-1.0.3-20200101.010003-3.pom'; + + cache[metadataUrl] = { + etag: 'etag', + httpResponse: { + statusCode: 200, + body: CachedMavenXml.parse(Fixtures.get('metadata-snapshot-only.xml')), + }, + timestamp, + }; + cache[snapshotMetadataUrl] = { + etag: 'etag', + httpResponse: { + statusCode: 200, + body: CachedMavenXml.parse( + Fixtures.get('metadata-snapshot-version.xml'), + ), + }, + timestamp, + }; + cache[snapshotPomUrl] = { + etag: 'etag', + httpResponse: { + statusCode: 200, + body: CachedMavenXml.parse(Fixtures.get('pom.xml')), + }, + timestamp, + }; + + const result = await getPkgReleases({ + datasource: MavenDatasource.id, + packageName, + registryUrls: [registryUrl], + versioning, + }); + + expect(result).toEqual({ + display: 'org.example:package', + group: 'org.example', + homepage: 'https://package.example.org/about', + name: 'package', + packageScope: 'org.example', + registryUrl, + releases: [{ version: '1.0.3-SNAPSHOT' }], + respectLatest: false, + tags: { + latest: '1.0.3-SNAPSHOT', + release: '1.0.3-SNAPSHOT', + }, + }); + expect(httpMock.getTrace()).toEqual([]); + expect(packageCache.setWithRawTtl).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/modules/datasource/maven/schema.spec.ts b/lib/modules/datasource/maven/schema.spec.ts new file mode 100644 index 00000000000..2e0e6ca97db --- /dev/null +++ b/lib/modules/datasource/maven/schema.spec.ts @@ -0,0 +1,145 @@ +import { codeBlock } from 'common-tags'; +import { Fixtures } from '~test/fixtures.ts'; +import { CachedMavenXml } from './schema.ts'; + +describe('modules/datasource/maven/schema', () => { + it('trims release metadata to the fields used by Renovate', () => { + const input = Fixtures.get('metadata.xml'); + + expect(CachedMavenXml.parse(input)).toEqual(codeBlock` + + + + 2.0.0 + 2.0.0 + + 0.0.1 + 1.0.0 + 1.0.1 + 1.0.2 + 1.0.3-SNAPSHOT + 1.0.4-SNAPSHOT + 1.0.5-SNAPSHOT + 2.0.0 + + + + `); + }); + + it('trims snapshot metadata to the fields used by Renovate', () => { + const input = Fixtures.get('metadata-snapshot-version.xml'); + + expect(CachedMavenXml.parse(input)).toEqual(codeBlock` + + + 1.0.3-SNAPSHOT + + + 20200101.010003 + 3 + + + + `); + }); + + it('trims pom files to the fields used by Renovate', () => { + const input = codeBlock` + + org.example + package + Package Name + Package description + https://package.example.org/about + + scm:git:https://github.com/example/package + + + + org.relocated + package-new + 2.0.0 + Moved + + + + org.parent + package-parent + 1.2.3 + + + `; + + expect(CachedMavenXml.parse(input)).toEqual(codeBlock` + + + org.example + https://package.example.org/about + + scm:git:https://github.com/example/package + + + + org.relocated + package-new + 2.0.0 + Moved + + + + org.parent + package-parent + 1.2.3 + + + `); + }); + + it('preserves empty relocation tags', () => { + const input = codeBlock` + + package + Package Name + + + + + `; + + expect(CachedMavenXml.parse(input)).toEqual(codeBlock` + + + + + + + `); + }); + + it('passes through unknown XML unchanged', () => { + const input = 'test'; + expect(CachedMavenXml.parse(input)).toBe(input); + }); + + it('passes through prefixed pom XML unchanged', () => { + const input = + 'https://package.example.org/about'; + expect(CachedMavenXml.parse(input)).toBe(input); + }); + + it('passes through pom XML when no retained fields are present', () => { + const input = 'package'; + expect(CachedMavenXml.parse(input)).toBe(input); + }); + + it('passes through metadata XML when no retained fields are present', () => { + const input = 'org.example'; + expect(CachedMavenXml.parse(input)).toBe(input); + }); + + it('passes through invalid XML unchanged', () => { + const input = ''; + expect(CachedMavenXml.parse(input)).toBe(input); + }); +}); diff --git a/lib/modules/datasource/maven/schema.ts b/lib/modules/datasource/maven/schema.ts new file mode 100644 index 00000000000..309efa46eaa --- /dev/null +++ b/lib/modules/datasource/maven/schema.ts @@ -0,0 +1,194 @@ +import { XmlDocument, type XmlElement } from 'xmldoc'; +import { z } from 'zod/v3'; + +const xmlHeader = ''; + +function escapeXml(value: string): string { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>'); +} + +class XmlWriter { + private lines: string[] = []; + private level: number; + + constructor(level = 0) { + this.level = level; + } + + value(name: string, value: string | undefined): void { + if (value === undefined) { + return; + } + + this.lines.push(`${this.indent()}<${name}>${escapeXml(value)}`); + } + + node(name: string, renderChildren: (xml: XmlWriter) => void): void { + this.renderNode(name, renderChildren, false); + } + + nodeOrEmpty(name: string, renderChildren: (xml: XmlWriter) => void): void { + this.renderNode(name, renderChildren, true); + } + + hasContent(): boolean { + return this.lines.length > 0; + } + + toString(): string { + return this.lines.join('\n'); + } + + private renderNode( + name: string, + renderChildren: (xml: XmlWriter) => void, + preserveEmpty: boolean, + ): void { + const contentStart = this.lines.length; + this.level += 1; + renderChildren(this); + this.level -= 1; + + const content = this.lines.splice(contentStart); + + if (!content.length) { + if (preserveEmpty) { + this.lines.push(`${this.indent()}<${name} />`); + } + + return; + } + + this.lines.push( + `${this.indent()}<${name}>`, + ...content, + `${this.indent()}`, + ); + } + + private indent(): string { + return ' '.repeat(this.level); + } +} + +function shrinkToUsefulSize(original: string, trimmed: string): string { + if (trimmed.length >= original.length) { + return original; + } + + return trimmed; +} + +function renderRelocationNode( + xml: XmlWriter, + relocation: XmlElement | undefined, +): void { + if (!relocation) { + return; + } + + xml.nodeOrEmpty('relocation', () => { + xml.value('groupId', relocation.valueWithPath('groupId')); + xml.value('artifactId', relocation.valueWithPath('artifactId')); + xml.value('version', relocation.valueWithPath('version')); + xml.value('message', relocation.valueWithPath('message')); + }); +} + +function trimMetadataXml(metadata: XmlDocument, input: string): string { + const version = metadata.descendantWithPath('version')?.val; + const latest = metadata.descendantWithPath('versioning.latest')?.val; + const release = metadata.descendantWithPath('versioning.release')?.val; + const versions = + metadata + .descendantWithPath('versioning.versions') + ?.childrenNamed('version') + .map((child) => child.val) ?? []; + const snapshot = metadata.descendantWithPath('versioning.snapshot'); + const timestamp = snapshot?.childNamed('timestamp')?.val; + const buildNumber = snapshot?.childNamed('buildNumber')?.val; + + const xml = new XmlWriter(); + xml.node('metadata', () => { + xml.value('version', version); + xml.node('versioning', () => { + xml.value('latest', latest); + xml.value('release', release); + xml.node('versions', () => { + for (const trimmedVersion of versions) { + xml.value('version', trimmedVersion); + } + }); + xml.node('snapshot', () => { + xml.value('timestamp', timestamp); + xml.value('buildNumber', buildNumber); + }); + }); + }); + + if (!xml.hasContent()) { + return input; + } + + return shrinkToUsefulSize(input, [xmlHeader, xml.toString()].join('\n')); +} + +function trimPomXml(project: XmlDocument, input: string): string { + const homepage = project.valueWithPath('url'); + const sourceUrl = project.valueWithPath('scm.url'); + const groupId = project.valueWithPath('groupId'); + const relocation = project.descendantWithPath( + 'distributionManagement.relocation', + ); + const parent = project.childNamed('parent'); + + const xml = new XmlWriter(); + xml.node('project', () => { + xml.value('groupId', groupId); + xml.value('url', homepage); + xml.node('scm', () => { + xml.value('url', sourceUrl); + }); + xml.node('distributionManagement', () => { + renderRelocationNode(xml, relocation); + }); + xml.node('parent', () => { + xml.value('groupId', parent?.valueWithPath('groupId')); + xml.value('artifactId', parent?.valueWithPath('artifactId')); + xml.value('version', parent?.valueWithPath('version')); + }); + }); + + if (!xml.hasContent()) { + return input; + } + + return shrinkToUsefulSize(input, [xmlHeader, xml.toString()].join('\n')); +} + +export function trimMavenXml(input: string): string { + let parsed: XmlDocument; + try { + parsed = new XmlDocument(input); + } catch { + return input; + } + + if (parsed.name.includes(':')) { + return input; + } + + switch (parsed.name) { + case 'metadata': + return trimMetadataXml(parsed, input); + case 'project': + return trimPomXml(parsed, input); + default: + return input; + } +} + +export const CachedMavenXml = z.string().transform(trimMavenXml); diff --git a/lib/modules/datasource/maven/util.spec.ts b/lib/modules/datasource/maven/util.spec.ts index f6b0fdf306d..059fa01c128 100644 --- a/lib/modules/datasource/maven/util.spec.ts +++ b/lib/modules/datasource/maven/util.spec.ts @@ -7,6 +7,7 @@ import { Http, HttpError } from '../../../util/http/index.ts'; import { MAVEN_REPO } from './common.ts'; import type { MavenFetchError } from './types.ts'; import { + downloadHttpContent, downloadHttpProtocol, downloadMavenXml, downloadS3Protocol, @@ -76,6 +77,23 @@ describe('modules/datasource/maven/util', () => { }); }); + describe('downloadHttpContent', () => { + it('returns the downloaded text body', async () => { + const http = partial({ + getText: () => + Promise.resolve({ + statusCode: 200, + body: 'pom text', + headers: {}, + }), + }); + + await expect( + downloadHttpContent(http, 'https://example.com/'), + ).resolves.toBe('pom text'); + }); + }); + describe('downloadS3Protocol', () => { it('returns error for non-S3 URLs', async () => { const res = await downloadS3Protocol(new URL('http://not-s3.com/')); diff --git a/lib/modules/datasource/maven/util.ts b/lib/modules/datasource/maven/util.ts index 40c4e9eaff6..7ee690591dd 100644 --- a/lib/modules/datasource/maven/util.ts +++ b/lib/modules/datasource/maven/util.ts @@ -16,6 +16,7 @@ import { asTimestamp } from '../../../util/timestamp.ts'; import { ensureTrailingSlash, isHttpUrl, parseUrl } from '../../../util/url.ts'; import { getGoogleAuthToken } from '../util.ts'; import { MAVEN_REPO } from './common.ts'; +import { CachedMavenXml } from './schema.ts'; import type { DependencyInfo, MavenDependency, @@ -70,6 +71,7 @@ const cacheProvider = new PackageHttpCacheProvider({ softTtlMinutes: 15, checkAuthorizationHeader: true, checkCacheControlHeader: false, // Maven doesn't respond with `cache-control` headers + writeSchema: CachedMavenXml, }); export async function downloadHttpProtocol(