Validate spec_urls based on webref ids

lint/linter/test-spec-urls.ts

@@ -2,7 +2,6 @@
  * See LICENSE file for more information. */
 
 import chalk from 'chalk-template';
-import specData from 'web-specs' assert { type: 'json' };
 
 import { Linter, Logger, LinterData } from '../utils.js';
 import { CompatStatement } from '../../types/types.js';
@@ -41,19 +40,62 @@ const specsExceptions = [
   'https://github.com/WebAssembly/multi-memory/blob/main/proposals',
 ];
 
-const allowedSpecURLs = [
-  ...(specData
-    .filter((spec) => spec.standing == 'good')
-    .map((spec) => [
-      spec.url,
-      spec.nightly?.url,
-      ...(spec.nightly ? spec.nightly.alternateUrls : []),
-      spec.series.nightlyUrl,
-    ])
-    .flat()
-    .filter((url) => !!url) as string[]),
-  ...specsExceptions,
-];
+interface ValidSpecHosts {
+  url: string;
+  alternateUrl: string;
+}
+
+const validSpecHosts: ValidSpecHosts[] = [];
+
+/**
+ * Get valid specification URLs from webref ids
+ * @returns array of valid spec urls (including fragment id)
+ */
+const getValidSpecURLs = async (): Promise<string[]> => {
+  const indexFile = await fetch(
+    'https://raw.githubusercontent.com/w3c/webref/main/ed/index.json',
+  );
+  const index = JSON.parse(await indexFile.text());
+  const specIDs: string[] = [];
+
+  index.results.forEach((spec) => {
+    if (spec.standing === 'good') {
+      if (spec.shortname === 'webnn') {
+        validSpecHosts.push({
+          url: spec.series.releaseUrl,
+          alternateUrl: spec.nightly?.url,
+        });
+      } else {
+        validSpecHosts.push({
+          url: spec.series.nightlyUrl,
+          alternateUrl: spec.nightly?.url,
+        });
+      }
+      if (spec.ids) {
+        specIDs.push(spec.ids);
+      }
+    }
+  });
+
+  const specURLsWithFragments: string[] = [];
+  await Promise.all(
+    specIDs.map(async (id) => {
+      const idFile = await fetch(
+        `https://raw.githubusercontent.com/w3c/webref/main/ed/${id}`,
+      );
+      const idResponse = JSON.parse(await idFile.text());
+      specURLsWithFragments.push(
+        ...idResponse.ids.map((id) => {
+          const url = new URL(id);
+          return url.origin + url.pathname + decodeURIComponent(url.hash);
+        }),
+      );
+    }),
+  );
+  return specURLsWithFragments;
+};
+
+const validSpecURLsWithFragments = await getValidSpecURLs();
 
 /**
  * Process the data for spec URL errors
@@ -70,7 +112,32 @@ const processData = (data: CompatStatement, logger: Logger): void => {
     : [data.spec_url];
 
   for (const specURL of featureSpecURLs) {
-    if (!allowedSpecURLs.some((prefix) => specURL.startsWith(prefix))) {
+    if (specURL.includes('#')) {
+      const hasSpec = validSpecURLsWithFragments.includes(specURL);
+
+      const alternateSpecURLs = validSpecHosts.filter(
+        (spec) => spec.url === specURL.split('#')[0],
+      );
+
+      const hasAlternateSpec = alternateSpecURLs.some((altSpecURL) => {
+        const specToLookup =
+          altSpecURL.alternateUrl + '#' + specURL.split('#')[1];
+        if (validSpecURLsWithFragments.includes(specToLookup)) {
+          return true;
+        }
+        return false;
+      });
+
+      if (!hasSpec && !hasAlternateSpec) {
+        logger.error(
+          chalk`Invalid specification fragment found: {bold ${specURL}}.`,
+        );
+      }
+    } else if (
+      !validSpecHosts.some((host) => specURL.startsWith(host.url)) &&
+      !validSpecHosts.some((host) => specURL.startsWith(host.alternateUrl)) &&
+      !specsExceptions.some((host) => specURL.startsWith(host))
+    ) {
       logger.error(
         chalk`Invalid specification URL found: {bold ${specURL}}. Check if:
          - there is a more current specification URL