From f64e360947e8167fd66261ffd0de67530a2c7b36 Mon Sep 17 00:00:00 2001 From: Florian Scholz Date: Fri, 26 Jul 2024 12:14:03 +0200 Subject: [PATCH 1/3] Validate spec_urls based on webref ids --- lint/linter/test-spec-urls.ts | 50 ++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/lint/linter/test-spec-urls.ts b/lint/linter/test-spec-urls.ts index d8609e139c6e00..6f483fd01917c1 100644 --- a/lint/linter/test-spec-urls.ts +++ b/lint/linter/test-spec-urls.ts @@ -2,7 +2,6 @@ * See LICENSE file for more information. */ import chalk from 'chalk-template'; -import specData from 'web-specs' assert { type: 'json' }; import { Linter, Logger, LinterData } from '../utils.js'; import { CompatStatement } from '../../types/types.js'; @@ -41,19 +40,37 @@ const specsExceptions = [ 'https://github.com/WebAssembly/multi-memory/blob/main/proposals', ]; -const allowedSpecURLs = [ - ...(specData - .filter((spec) => spec.standing == 'good') - .map((spec) => [ - spec.url, - spec.nightly?.url, - ...(spec.nightly ? spec.nightly.alternateUrls : []), - spec.series.nightlyUrl, - ]) - .flat() - .filter((url) => !!url) as string[]), - ...specsExceptions, -]; +/** + * Get valid specification URLs from webref ids + * @returns array of valid spec urls (including fragment id) + */ +const getValidSpecURLs = async (): Promise => { + const indexFile = await fetch( + 'https://raw.githubusercontent.com/w3c/webref/main/ed/index.json', + ); + const index = JSON.parse(await indexFile.text()); + + const ids: string[] = []; + index.results.forEach((result) => { + if (result.ids) { + ids.push(result.ids); + } + }); + + const specIDs: string[] = []; + await Promise.all( + ids.map(async (id) => { + const idFile = await fetch( + `https://raw.githubusercontent.com/w3c/webref/main/ed/${id}`, + ); + const idResponse = JSON.parse(await idFile.text()); + specIDs.push(...idResponse.ids); + }), + ); + return specIDs; +}; + +const validSpecURLs = await getValidSpecURLs(); /** * Process the data for spec URL errors @@ -70,7 +87,10 @@ const processData = (data: CompatStatement, logger: Logger): void => { : [data.spec_url]; for (const specURL of featureSpecURLs) { - if (!allowedSpecURLs.some((prefix) => specURL.startsWith(prefix))) { + if ( + !validSpecURLs.includes(specURL) && + !specsExceptions.some((host) => specURL.startsWith(host)) + ) { logger.error( chalk`Invalid specification URL found: {bold ${specURL}}. Check if: - there is a more current specification URL From 7116139a95760a1cfca00886114081e9ce7b4113 Mon Sep 17 00:00:00 2001 From: Florian Scholz Date: Sat, 27 Jul 2024 10:37:21 +0200 Subject: [PATCH 2/3] =?UTF-8?q?Address=20feedback=20from=20Fran=C3=A7ois?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lint/linter/test-spec-urls.ts | 65 +++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/lint/linter/test-spec-urls.ts b/lint/linter/test-spec-urls.ts index 6f483fd01917c1..6f75708bc6f25d 100644 --- a/lint/linter/test-spec-urls.ts +++ b/lint/linter/test-spec-urls.ts @@ -40,6 +40,13 @@ const specsExceptions = [ 'https://github.com/WebAssembly/multi-memory/blob/main/proposals', ]; +interface ValidSpecHosts { + url: string; + alternateUrl: string; +} + +const validSpecHosts: ValidSpecHosts[] = []; + /** * Get valid specification URLs from webref ids * @returns array of valid spec urls (including fragment id) @@ -49,28 +56,41 @@ const getValidSpecURLs = async (): Promise => { 'https://raw.githubusercontent.com/w3c/webref/main/ed/index.json', ); const index = JSON.parse(await indexFile.text()); + const specIDs: string[] = []; - const ids: string[] = []; - index.results.forEach((result) => { - if (result.ids) { - ids.push(result.ids); + index.results.forEach((spec) => { + if (spec.standing === 'good') { + if (spec.shortname === 'webnn') { + validSpecHosts.push({ + url: spec.series.releaseUrl, + alternateUrl: spec.nightly?.url, + }); + } else { + validSpecHosts.push({ + url: spec.series.nightlyUrl, + alternateUrl: spec.nightly?.url, + }); + } + if (spec.ids) { + specIDs.push(spec.ids); + } } }); - const specIDs: string[] = []; + const specURLsWithFragments: string[] = []; await Promise.all( - ids.map(async (id) => { + specIDs.map(async (id) => { const idFile = await fetch( `https://raw.githubusercontent.com/w3c/webref/main/ed/${id}`, ); const idResponse = JSON.parse(await idFile.text()); - specIDs.push(...idResponse.ids); + specURLsWithFragments.push(...idResponse.ids); }), ); - return specIDs; + return specURLsWithFragments; }; -const validSpecURLs = await getValidSpecURLs(); +const validSpecURLsWithFragments = await getValidSpecURLs(); /** * Process the data for spec URL errors @@ -87,8 +107,31 @@ const processData = (data: CompatStatement, logger: Logger): void => { : [data.spec_url]; for (const specURL of featureSpecURLs) { - if ( - !validSpecURLs.includes(specURL) && + if (specURL.includes('#')) { + const hasSpec = validSpecURLsWithFragments.includes(encodeURI(specURL)); + + const alternateSpecURLs = validSpecHosts.filter( + (spec) => spec.url === specURL.split('#')[0], + ); + + const hasAlternateSpec = alternateSpecURLs.some((altSpecURL) => { + const specToLookup = encodeURI( + altSpecURL.alternateUrl + '#' + specURL.split('#')[1], + ); + if (validSpecURLsWithFragments.includes(specToLookup)) { + return true; + } + return false; + }); + + if (!hasSpec && !hasAlternateSpec) { + logger.error( + chalk`Invalid specification fragment found: {bold ${specURL}}.`, + ); + } + } else if ( + !validSpecHosts.some((host) => specURL.startsWith(host.url)) && + !validSpecHosts.some((host) => specURL.startsWith(host.alternateUrl)) && !specsExceptions.some((host) => specURL.startsWith(host)) ) { logger.error( From b0c01497b6d4979923c9b8db2bff079c69e31024 Mon Sep 17 00:00:00 2001 From: Florian Scholz Date: Sat, 27 Jul 2024 11:31:22 +0200 Subject: [PATCH 3/3] decode hashes as suggested --- lint/linter/test-spec-urls.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lint/linter/test-spec-urls.ts b/lint/linter/test-spec-urls.ts index 6f75708bc6f25d..babdbf49615ae5 100644 --- a/lint/linter/test-spec-urls.ts +++ b/lint/linter/test-spec-urls.ts @@ -84,7 +84,12 @@ const getValidSpecURLs = async (): Promise => { `https://raw.githubusercontent.com/w3c/webref/main/ed/${id}`, ); const idResponse = JSON.parse(await idFile.text()); - specURLsWithFragments.push(...idResponse.ids); + specURLsWithFragments.push( + ...idResponse.ids.map((id) => { + const url = new URL(id); + return url.origin + url.pathname + decodeURIComponent(url.hash); + }), + ); }), ); return specURLsWithFragments; @@ -108,16 +113,15 @@ const processData = (data: CompatStatement, logger: Logger): void => { for (const specURL of featureSpecURLs) { if (specURL.includes('#')) { - const hasSpec = validSpecURLsWithFragments.includes(encodeURI(specURL)); + const hasSpec = validSpecURLsWithFragments.includes(specURL); const alternateSpecURLs = validSpecHosts.filter( (spec) => spec.url === specURL.split('#')[0], ); const hasAlternateSpec = alternateSpecURLs.some((altSpecURL) => { - const specToLookup = encodeURI( - altSpecURL.alternateUrl + '#' + specURL.split('#')[1], - ); + const specToLookup = + altSpecURL.alternateUrl + '#' + specURL.split('#')[1]; if (validSpecURLsWithFragments.includes(specToLookup)) { return true; }