From 4efe66d122fe8a012bbb8ab15f3a66ee5869e1a7 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 6 Jun 2024 21:10:47 +0545 Subject: [PATCH 1/8] massage and validate hostRules.matchHost --- .../custom/host-rules-migration.spec.ts | 2 + .../migrations/custom/host-rules-migration.ts | 2 + lib/config/validation.spec.ts | 38 +++++++++++++++++++ lib/config/validation.ts | 19 ++++++++++ 4 files changed, 61 insertions(+) diff --git a/lib/config/migrations/custom/host-rules-migration.spec.ts b/lib/config/migrations/custom/host-rules-migration.spec.ts index fdc77fa14086cd..c51be008651e71 100644 --- a/lib/config/migrations/custom/host-rules-migration.spec.ts +++ b/lib/config/migrations/custom/host-rules-migration.spec.ts @@ -27,6 +27,7 @@ describe('config/migrations/custom/host-rules-migration', () => { { hostName: 'some.domain.com', token: '123test' }, { endpoint: 'domain.com/', token: '123test' }, { host: 'some.domain.com', token: '123test' }, + { host: 'some.domain.com:8080', token: '123test' }, ], } as any, { @@ -58,6 +59,7 @@ describe('config/migrations/custom/host-rules-migration', () => { { matchHost: 'some.domain.com', token: '123test' }, { matchHost: 'https://domain.com/', token: '123test' }, { matchHost: 'some.domain.com', token: '123test' }, + { matchHost: 'https://some.domain.com:8080', token: '123test' }, ], }, ); diff --git a/lib/config/migrations/custom/host-rules-migration.ts b/lib/config/migrations/custom/host-rules-migration.ts index b9d1d5af86b49f..9bd7888253e792 100644 --- a/lib/config/migrations/custom/host-rules-migration.ts +++ b/lib/config/migrations/custom/host-rules-migration.ts @@ -94,6 +94,8 @@ function validateHostRule(rule: LegacyHostRule & HostRule): void { function massageUrl(url: string): string { if (!url.includes('://') && url.includes('/')) { return 'https://' + url; + } else if (!url.includes('://') && url.includes(':')) { + return 'https://' + url; } else { return url; } diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts index ee8e4d5a5fa040..ab844360af7b1e 100644 --- a/lib/config/validation.spec.ts +++ b/lib/config/validation.spec.ts @@ -1134,6 +1134,44 @@ describe('config/validation', () => { ]); }); + it('errors if invalid matchHost values in hostRules', async () => { + GlobalConfig.set({ allowedHeaders: ['X-*'] }); + + const config = { + hostRules: [ + { + matchHost: '://', + token: 'token', + }, + { + matchHost: '', + token: 'token', + }, + { + matchHost: undefined, + token: 'token', + }, + ], + }; + const { errors } = await configValidation.validateConfig('repo', config); + expect(errors).toMatchObject([ + { + topic: 'Configuration Error', + message: + 'Configuration option `hostRules[2].matchHost` should be a string', + }, + { + topic: 'Configuration Error', + message: + 'Invalid value for hostRules matchHost. It cannot be an empty string.', + }, + { + topic: 'Configuration Error', + message: 'hostRules matchHost `://` is not a valid URL.', + }, + ]); + }); + it('errors if forbidden header in hostRules', async () => { GlobalConfig.set({ allowedHeaders: ['X-*'] }); diff --git a/lib/config/validation.ts b/lib/config/validation.ts index 15b87c3b144022..2196e1936538ea 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -814,6 +814,25 @@ export async function validateConfig( ? (config.allowedHeaders as string[]) ?? [] : GlobalConfig.get('allowedHeaders', []); for (const rule of val as HostRule[]) { + if (is.nonEmptyString(rule.matchHost)) { + if (rule.matchHost.includes('://')) { + try { + new URL(rule.matchHost); + } catch (err) { + errors.push({ + topic: 'Configuration Error', + message: `hostRules matchHost \`${rule.matchHost}\` is not a valid URL.`, + }); + } + } + } else if (is.emptyString(rule.matchHost)) { + errors.push({ + topic: 'Configuration Error', + message: + 'Invalid value for hostRules matchHost. It cannot be an empty string.', + }); + } + if (!rule.headers) { continue; } From 6a75be1245fcaa17df8de24ccc61697ff6c28df7 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 6 Jun 2024 22:16:42 +0545 Subject: [PATCH 2/8] apply suggestions --- lib/config/validation.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts index ab844360af7b1e..976c86b8c80f40 100644 --- a/lib/config/validation.spec.ts +++ b/lib/config/validation.spec.ts @@ -1151,6 +1151,10 @@ describe('config/validation', () => { matchHost: undefined, token: 'token', }, + { + hostType: 'github', + token: 'token', + }, ], }; const { errors } = await configValidation.validateConfig('repo', config); From d1767a992a54beec47f79c657ca72b513c30ca2e Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 6 Jun 2024 18:44:09 +0200 Subject: [PATCH 3/8] Update lib/config/migrations/custom/host-rules-migration.spec.ts --- lib/config/migrations/custom/host-rules-migration.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/migrations/custom/host-rules-migration.spec.ts b/lib/config/migrations/custom/host-rules-migration.spec.ts index c51be008651e71..67c1331db2f487 100644 --- a/lib/config/migrations/custom/host-rules-migration.spec.ts +++ b/lib/config/migrations/custom/host-rules-migration.spec.ts @@ -27,7 +27,7 @@ describe('config/migrations/custom/host-rules-migration', () => { { hostName: 'some.domain.com', token: '123test' }, { endpoint: 'domain.com/', token: '123test' }, { host: 'some.domain.com', token: '123test' }, - { host: 'some.domain.com:8080', token: '123test' }, + { matchHost: 'some.domain.com:8080', token: '123test' }, ], } as any, { From 46ca16412d7ef349bd428651148d69a658c4ca47 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 6 Jun 2024 23:32:01 +0545 Subject: [PATCH 4/8] use parseUrl --- lib/config/validation.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/config/validation.ts b/lib/config/validation.ts index 2196e1936538ea..57fae958ca5bdc 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -15,6 +15,7 @@ import { matchRegexOrGlobList, } from '../util/string-match'; import * as template from '../util/template'; +import { parseUrl } from '../util/url'; import { hasValidSchedule, hasValidTimezone, @@ -816,9 +817,7 @@ export async function validateConfig( for (const rule of val as HostRule[]) { if (is.nonEmptyString(rule.matchHost)) { if (rule.matchHost.includes('://')) { - try { - new URL(rule.matchHost); - } catch (err) { + if (is.null_(parseUrl(rule.matchHost))) { errors.push({ topic: 'Configuration Error', message: `hostRules matchHost \`${rule.matchHost}\` is not a valid URL.`, From 9d8ba7b0808bfce8329e4b614ede8c6d3717f7e7 Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Wed, 19 Jun 2024 09:35:58 +0530 Subject: [PATCH 5/8] Update lib/config/validation.ts --- lib/config/validation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/validation.ts b/lib/config/validation.ts index 57fae958ca5bdc..dede421eff834d 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -817,7 +817,7 @@ export async function validateConfig( for (const rule of val as HostRule[]) { if (is.nonEmptyString(rule.matchHost)) { if (rule.matchHost.includes('://')) { - if (is.null_(parseUrl(rule.matchHost))) { + if (parseUrl(rule.matchHost) === null) { errors.push({ topic: 'Configuration Error', message: `hostRules matchHost \`${rule.matchHost}\` is not a valid URL.`, From 80172990bf278cf569de8ef90e54c4fbf7663740 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sat, 22 Jun 2024 20:35:20 +0530 Subject: [PATCH 6/8] massage host in hostRules.add method --- lib/util/host-rules.spec.ts | 33 ++++++++++++++++++++++----------- lib/util/host-rules.ts | 14 ++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/lib/util/host-rules.spec.ts b/lib/util/host-rules.spec.ts index f5fa0cbf239a53..9caa4a3657051a 100644 --- a/lib/util/host-rules.spec.ts +++ b/lib/util/host-rules.spec.ts @@ -56,6 +56,27 @@ describe('util/host-rules', () => { username: 'user1', }); }); + + it('massages host url', () => { + add({ + matchHost: 'some.domain.com:8080', + username: 'user1', + password: 'pass1', + }); + add({ + matchHost: 'domain.com/', + username: 'user2', + password: 'pass2', + }); + expect(find({ url: 'https://some.domain.com:8080' })).toEqual({ + password: 'pass1', + username: 'user1', + }); + expect(find({ url: 'https://domain.com/' })).toEqual({ + password: 'pass2', + username: 'user2', + }); + }); }); describe('find()', () => { @@ -125,7 +146,7 @@ describe('util/host-rules', () => { }); it('matches on specific path', () => { - // Initialized platform holst rule + // Initialized platform host rule add({ hostType: 'github', matchHost: 'https://api.github.com', @@ -244,16 +265,6 @@ describe('util/host-rules', () => { expect(find({ url: 'httpsdomain.com' }).token).toBeUndefined(); }); - it('host with port is interpreted as empty', () => { - add({ - matchHost: 'domain.com:9118', - token: 'def', - }); - expect(find({ url: 'https://domain.com:9118' }).token).toBe('def'); - expect(find({ url: 'https://domain.com' }).token).toBe('def'); - expect(find({ url: 'httpsdomain.com' }).token).toBe('def'); - }); - it('matches on hostType and endpoint', () => { add({ hostType: NugetDatasource.id, diff --git a/lib/util/host-rules.ts b/lib/util/host-rules.ts index 1da59fcd9e3c49..6449d09545364e 100644 --- a/lib/util/host-rules.ts +++ b/lib/util/host-rules.ts @@ -38,11 +38,25 @@ export function migrateRule(rule: LegacyHostRule & HostRule): HostRule { return result; } +/** + * prefix https:// to hosts with port or path + */ +function massageHostUrl(url: string): string { + if (!url.includes('://') && url.includes('/')) { + return 'https://' + url; + } else if (!url.includes('://') && url.includes(':')) { + return 'https://' + url; + } + + return url; +} + export function add(params: HostRule): void { const rule = migrateRule(params); const confidentialFields: (keyof HostRule)[] = ['password', 'token']; if (rule.matchHost) { + rule.matchHost = massageHostUrl(rule.matchHost); const parsedUrl = parseUrl(rule.matchHost); rule.resolvedHost = parsedUrl?.hostname ?? rule.matchHost; confidentialFields.forEach((field) => { From 92b95c046a0b5868f9514b5dd493f64d0afae448 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 23 Jun 2024 23:45:57 +0530 Subject: [PATCH 7/8] use single fn --- .../migrations/custom/host-rules-migration.ts | 16 +++------------- lib/util/host-rules.ts | 2 +- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/lib/config/migrations/custom/host-rules-migration.ts b/lib/config/migrations/custom/host-rules-migration.ts index 9bd7888253e792..3a6e614d1c1f3a 100644 --- a/lib/config/migrations/custom/host-rules-migration.ts +++ b/lib/config/migrations/custom/host-rules-migration.ts @@ -2,7 +2,7 @@ import is from '@sindresorhus/is'; import { CONFIG_VALIDATION } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import type { HostRule } from '../../../types'; -import type { LegacyHostRule } from '../../../util/host-rules'; +import { type LegacyHostRule, massageHostUrl } from '../../../util/host-rules'; import { AbstractMigration } from '../base/abstract-migration'; import { migrateDatasource } from './datasource-migration'; @@ -25,7 +25,7 @@ export class HostRulesMigration extends AbstractMigration { if (key === 'matchHost') { if (is.string(value)) { - newRule.matchHost ??= massageUrl(value); + newRule.matchHost ??= massageHostUrl(value); } continue; } @@ -45,7 +45,7 @@ export class HostRulesMigration extends AbstractMigration { key === 'domainName' ) { if (is.string(value)) { - newRule.matchHost ??= massageUrl(value); + newRule.matchHost ??= massageHostUrl(value); } continue; } @@ -91,16 +91,6 @@ function validateHostRule(rule: LegacyHostRule & HostRule): void { } } -function massageUrl(url: string): string { - if (!url.includes('://') && url.includes('/')) { - return 'https://' + url; - } else if (!url.includes('://') && url.includes(':')) { - return 'https://' + url; - } else { - return url; - } -} - function removeUndefinedFields( obj: Record, ): Record { diff --git a/lib/util/host-rules.ts b/lib/util/host-rules.ts index 6449d09545364e..787e59869cd24d 100644 --- a/lib/util/host-rules.ts +++ b/lib/util/host-rules.ts @@ -41,7 +41,7 @@ export function migrateRule(rule: LegacyHostRule & HostRule): HostRule { /** * prefix https:// to hosts with port or path */ -function massageHostUrl(url: string): string { +export function massageHostUrl(url: string): string { if (!url.includes('://') && url.includes('/')) { return 'https://' + url; } else if (!url.includes('://') && url.includes(':')) { From a4b12fe6fb856e1f39a525d9928535ad64644e4f Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Mon, 24 Jun 2024 02:36:53 +0530 Subject: [PATCH 8/8] refactor --- .../migrations/custom/host-rules-migration.ts | 3 ++- lib/util/host-rules.ts | 15 +-------------- lib/util/url.spec.ts | 10 ++++++++++ lib/util/url.ts | 13 +++++++++++++ 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/lib/config/migrations/custom/host-rules-migration.ts b/lib/config/migrations/custom/host-rules-migration.ts index 3a6e614d1c1f3a..838cf93d48f175 100644 --- a/lib/config/migrations/custom/host-rules-migration.ts +++ b/lib/config/migrations/custom/host-rules-migration.ts @@ -2,7 +2,8 @@ import is from '@sindresorhus/is'; import { CONFIG_VALIDATION } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import type { HostRule } from '../../../types'; -import { type LegacyHostRule, massageHostUrl } from '../../../util/host-rules'; +import type { LegacyHostRule } from '../../../util/host-rules'; +import { massageHostUrl } from '../../../util/url'; import { AbstractMigration } from '../base/abstract-migration'; import { migrateDatasource } from './datasource-migration'; diff --git a/lib/util/host-rules.ts b/lib/util/host-rules.ts index 787e59869cd24d..55d5c4d85a7e87 100644 --- a/lib/util/host-rules.ts +++ b/lib/util/host-rules.ts @@ -4,7 +4,7 @@ import type { CombinedHostRule, HostRule } from '../types'; import { clone } from './clone'; import * as sanitize from './sanitize'; import { toBase64 } from './string'; -import { isHttpUrl, parseUrl } from './url'; +import { isHttpUrl, massageHostUrl, parseUrl } from './url'; let hostRules: HostRule[] = []; @@ -38,19 +38,6 @@ export function migrateRule(rule: LegacyHostRule & HostRule): HostRule { return result; } -/** - * prefix https:// to hosts with port or path - */ -export function massageHostUrl(url: string): string { - if (!url.includes('://') && url.includes('/')) { - return 'https://' + url; - } else if (!url.includes('://') && url.includes(':')) { - return 'https://' + url; - } - - return url; -} - export function add(params: HostRule): void { const rule = migrateRule(params); diff --git a/lib/util/url.spec.ts b/lib/util/url.spec.ts index a0c65cb7ae27b1..0ed4143cb69c55 100644 --- a/lib/util/url.spec.ts +++ b/lib/util/url.spec.ts @@ -5,6 +5,7 @@ import { getQueryString, isHttpUrl, joinUrlParts, + massageHostUrl, parseLinkHeader, parseUrl, replaceUrlPath, @@ -214,4 +215,13 @@ describe('util/url', () => { }, }); }); + + it('massageHostUrl', () => { + expect(massageHostUrl('domain.com')).toBe('domain.com'); + expect(massageHostUrl('domain.com:8080')).toBe('https://domain.com:8080'); + expect(massageHostUrl('domain.com/some/path')).toBe( + 'https://domain.com/some/path', + ); + expect(massageHostUrl('https://domain.com')).toBe('https://domain.com'); + }); }); diff --git a/lib/util/url.ts b/lib/util/url.ts index 9ac957233127ff..26853a013e061b 100644 --- a/lib/util/url.ts +++ b/lib/util/url.ts @@ -132,3 +132,16 @@ export function parseLinkHeader( } return _parseLinkHeader(linkHeader); } + +/** + * prefix https:// to hosts with port or path + */ +export function massageHostUrl(url: string): string { + if (!url.includes('://') && url.includes('/')) { + return 'https://' + url; + } else if (!url.includes('://') && url.includes(':')) { + return 'https://' + url; + } else { + return url; + } +}