Localize all the 'register that a biblio reference exists' code, and make section links use it. Fixes #2280

Author: tabatkins

bikeshed/Spec.py

@@ -244,8 +244,8 @@ def processDocument(self):
         u.formatArgumentdefTables(self)
         u.formatElementdefTables(self)
         u.processAutolinks(self)
+        u.fixInterDocumentReferences(self)
         biblio.dedupBiblioReferences(self)
-        u.verifyUsageOfAllLocalBiblios(self)
         caniuse.addCanIUsePanels(self)
         boilerplate.addIndexSection(self)
         boilerplate.addExplicitIndexes(self)
@@ -269,6 +269,7 @@ def processDocument(self):
         fingerprinting.addTrackingVector(self)
         u.fixIntraDocumentReferences(self)
         u.fixInterDocumentReferences(self)
+        u.verifyUsageOfAllLocalBiblios(self)
         u.removeMultipleLinks(self)
         u.forceCrossorigin(self)
         lint.brokenLinks(self)

bikeshed/biblio.py

@@ -525,4 +525,4 @@ def isShepherdRef(ref):
         if keys["shepherd"] in doc.externalRefsUsed:
             for k, v in list(doc.externalRefsUsed[keys["shepherd"]].items()):
                 doc.externalRefsUsed[keys["specref"]][k] = v
-        del doc.externalRefsUsed[keys["shepherd"]]
+            del doc.externalRefsUsed[keys["shepherd"]]

bikeshed/unsortedJunk.py

@@ -445,30 +445,31 @@ def fixInterDocumentReferences(doc: "t.SpecType"):
         )
 
 
-def fillInterDocumentReferenceFromShepherd(doc: "t.SpecType", el, spec, section):
-    specData = doc.refs.fetchHeadings(spec)
-    if section in specData:
-        heading = specData[section]
+def fillInterDocumentReferenceFromShepherd(doc: "t.SpecType", el, specName, section):
+    headingData = doc.refs.fetchHeadings(specName)
+    if section in headingData:
+        heading = headingData[section]
     else:
         m.die(
-            f"Couldn't find section '{section}' in spec '{spec}':\n{h.outerHTML(el)}",
+            f"Couldn't find section '{section}' in spec '{specName}':\n{h.outerHTML(el)}",
             el=el,
         )
         return
     if isinstance(heading, list):
         # Multipage spec
         if len(heading) == 1:
             # only one heading of this name, no worries
-            heading = specData[heading[0]]
+            heading = headingData[heading[0]]
         else:
             # multiple headings of this id, user needs to disambiguate
             m.die(
-                f"Multiple headings with id '{section}' for spec '{spec}'. Please specify:\n"
-                + "\n".join("  [[{}]]".format(spec + x) for x in heading),
+                f"Multiple headings with id '{section}' for spec '{specName}'. Please specify:\n"
+                + "\n".join(f"  [[{specName + x}]]" for x in heading),
                 el=el,
             )
             return
     if doc.md.status == "current":
+        # FIXME: doc.md.status is not these values
         if "current" in heading:
             heading = heading["current"]
         else:
@@ -485,6 +486,11 @@ def fillInterDocumentReferenceFromShepherd(doc: "t.SpecType", el, spec, section)
         h.appendChild(el, " §\u202f{number} {text}".format(**heading))
     h.removeAttr(el, "data-link-spec", "spec-section")
 
+    # Mark this as a used biblio ref
+    specData = doc.refs.specs[specName]
+    bib = biblio.SpecBasedBiblioEntry(specData)
+    registerBiblioUsage(doc, bib, el=el)
+
 
 def fillInterDocumentReferenceFromSpecref(doc: "t.SpecType", el, spec, section):
     bib = doc.refs.getBiblioRef(spec)
@@ -497,6 +503,9 @@ def fillInterDocumentReferenceFromSpecref(doc: "t.SpecType", el, spec, section):
         el.text = bib.title + " §\u202f" + section[1:]
     h.removeAttr(el, "data-link-spec", "spec-section")
 
+    # Mark this as a used biblio ref
+    registerBiblioUsage(doc, bib, el=el)
+
 
 def processDfns(doc: "t.SpecType"):
     dfns = h.findAll(config.dfnElementsSelector, doc)
@@ -767,17 +776,6 @@ def classifyLink(el):
 def processBiblioLinks(doc: "t.SpecType"):
     biblioLinks = h.findAll("a[data-link-type='biblio']", doc)
     for el in biblioLinks:
-        biblioType = el.get("data-biblio-type")
-        if biblioType == "normative":
-            storage = doc.normativeRefs
-        elif biblioType == "informative":
-            storage = doc.informativeRefs
-        else:
-            m.die(
-                f"Unknown data-biblio-type value '{biblioType}' on {h.outerHTML(el)}. Only 'normative' and 'informative' allowed.",
-                el=el,
-            )
-            continue
 
         linkText = determineLinkText(el)
         if linkText[0] == "[" and linkText[-1] == "]":
@@ -828,7 +826,7 @@ def processBiblioLinks(doc: "t.SpecType"):
             doc.refs.preferredBiblioNames[ref.linkText] = linkText
             # Use it on the current ref. Future ones will use the preferred name automatically.
             ref.linkText = linkText
-        storage[ref.linkText] = ref
+        registerBiblioUsage(doc, ref, el=el, type=el.get("data-biblio-type"))
 
         id = config.simplifyText(ref.linkText)
         el.set("href", "#biblio-" + id)
@@ -853,8 +851,8 @@ def verifyUsageOfAllLocalBiblios(doc: "t.SpecType"):
     were used in the spec,
     so you can remove entries when they're no longer necessary.
     """
-    usedBiblioKeys = {x.lower() for x in list(doc.normativeRefs.keys()) + list(doc.informativeRefs.keys())}
-    localBiblios = [b["linkText"].lower() for bs in doc.refs.biblios.values() for b in bs if b["order"] == 1]
+    usedBiblioKeys = {x.upper() for x in list(doc.normativeRefs.keys()) + list(doc.informativeRefs.keys())}
+    localBiblios = [b["linkText"].upper() for bs in doc.refs.biblios.values() for b in bs if b["order"] == 1]
     unusedBiblioKeys = []
     for b in localBiblios:
         if b not in usedBiblioKeys:
@@ -927,15 +925,10 @@ def processAutolinks(doc: "t.SpecType"):
         # rather than checking `status == "local"`, as "local" refs include
         # those defined in `<pre class="anchor">` datablocks, which we do
         # want to capture here.
-        if ref and ref.spec and doc.refs.spec and ref.spec.lower() != doc.refs.spec.lower():
-            spec = ref.spec.lower()
+        if ref and ref.spec and doc.refs.spec and ref.spec.upper() != doc.refs.spec.upper():
+            spec = ref.spec.upper()
             key = ref.for_[0] if ref.for_ else ""
 
-            if h.isNormative(el, doc):
-                biblioStorage = doc.normativeRefs
-            else:
-                biblioStorage = doc.informativeRefs
-
             # If the ref is from an anchor block, it knows what it's doing.
             # Don't follow obsoletion chains.
             allowObsolete = ref.status == "anchor-block"
@@ -948,8 +941,8 @@ def processAutolinks(doc: "t.SpecType"):
                 allowObsolete=allowObsolete,
             )
             if biblioRef:
-                biblioStorage[biblioRef.linkText] = biblioRef
-                spec = biblioRef.linkText.lower()
+                spec = biblioRef.linkText.upper()
+                registerBiblioUsage(doc, biblioRef, el=el)
                 doc.externalRefsUsed[spec]["_biblio"] = biblioRef
             doc.externalRefsUsed[spec][ref.text][key] = ref
 
@@ -967,6 +960,24 @@ def processAutolinks(doc: "t.SpecType"):
     h.dedupIDs(doc)
 
 
+def registerBiblioUsage(
+    doc: "t.SpecType", ref: biblio.BiblioEntry, el: t.ElementT, type: t.Optional[str] = None
+) -> None:
+    if type is None:
+        if h.isNormative(el, doc):
+            type = "normative"
+        else:
+            type = "informative"
+    if type == "normative":
+        biblioStorage = doc.normativeRefs
+    elif type == "informative":
+        biblioStorage = doc.informativeRefs
+    else:
+        m.die(f"Unknown biblio type {type}.", el=el)
+        return
+    biblioStorage[ref.linkText.upper()] = ref
+
+
 def decorateAutolink(doc: "t.SpecType", el, linkType, linkText, ref):
     # Add additional effects to autolinks.
     if doc.md.slimBuildArtifact:

tests/github/WICG/aom/spec/custom-element-semantics.html

@@ -1108,13 +1108,13 @@ <h3 class="no-num no-ref heading settled" id="index-defined-here"><span class="c
   <h3 class="no-num no-ref heading settled" id="index-defined-elsewhere"><span class="content">Terms defined by reference</span><a class="self-link" href="#index-defined-elsewhere"></a></h3>
   <ul class="index">
    <li>
-    <a data-link-type="biblio">[aria]</a> defines the following terms:
+    <a data-link-type="biblio">[ARIA]</a> defines the following terms:
     <ul>
      <li><span class="dfn-paneled" id="term-for-idl-def-accessibilityrole">AccessibilityRole</span>
      <li><span class="dfn-paneled" id="term-for-idl-def-ariaattributes">AriaAttributes</span>
     </ul>
    <li>
-    <a data-link-type="biblio">[core-aam]</a> defines the following terms:
+    <a data-link-type="biblio">[CORE-AAM]</a> defines the following terms:
     <ul>
      <li><span class="dfn-paneled" id="term-for-dfn-accessibility-tree">accessibility tree</span>
      <li><span class="dfn-paneled" id="term-for-dfn-accessible-object">accessible object</span>
@@ -1139,7 +1139,7 @@ <h3 class="no-num no-ref heading settled" id="index-defined-elsewhere"><span cla
      <li><span class="dfn-paneled" id="term-for-custom-element-definition①">defined</span>
     </ul>
    <li>
-    <a data-link-type="biblio">[html-aam]</a> defines the following terms:
+    <a data-link-type="biblio">[HTML-AAM]</a> defines the following terms:
     <ul>
      <li><span class="dfn-paneled" id="term-for-details-id-17">role of button</span>
     </ul>
@@ -1161,6 +1161,8 @@ <h3 class="no-num no-ref heading settled" id="normative"><span class="content">N
    <dd>Anne van Kesteren. <a href="https://dom.spec.whatwg.org/"><cite>DOM Standard</cite></a>. Living Standard. URL: <a href="https://dom.spec.whatwg.org/">https://dom.spec.whatwg.org/</a>
    <dt id="biblio-html">[HTML]
    <dd>Anne van Kesteren; et al. <a href="https://html.spec.whatwg.org/multipage/"><cite>HTML Standard</cite></a>. Living Standard. URL: <a href="https://html.spec.whatwg.org/multipage/">https://html.spec.whatwg.org/multipage/</a>
+   <dt id="biblio-html-aam-10">[HTML-AAM-1.0]
+   <dd>Steve Faulkner; et al. <a href="https://www.w3.org/TR/html-aam-1.0/"><cite>HTML Accessibility API Mappings 1.0</cite></a>. 17 August 2020. WD. URL: <a href="https://www.w3.org/TR/html-aam-1.0/">https://www.w3.org/TR/html-aam-1.0/</a>
    <dt id="biblio-rfc2119">[RFC2119]
    <dd>S. Bradner. <a href="https://datatracker.ietf.org/doc/html/rfc2119"><cite>Key words for use in RFCs to Indicate Requirement Levels</cite></a>. March 1997. Best Current Practice. URL: <a href="https://datatracker.ietf.org/doc/html/rfc2119">https://datatracker.ietf.org/doc/html/rfc2119</a>
    <dt id="biblio-webidl">[WebIDL]

tests/github/WICG/app-history/spec.html

@@ -2396,7 +2396,7 @@ <h3 class="no-num no-ref heading settled" id="index-defined-elsewhere"><span cla
      <li><span class="dfn-paneled" id="term-for-concept-url-serializer">url serializer</span>
     </ul>
    <li>
-    <a data-link-type="biblio">[uuid]</a> defines the following terms:
+    <a data-link-type="biblio">[UUID]</a> defines the following terms:
     <ul>
      <li><span class="dfn-paneled" id="term-for-dfn-generate-a-random-uuid">generate a random uuid</span>
     </ul>
@@ -2455,6 +2455,8 @@ <h3 class="no-num no-ref heading settled" id="informative"><span class="content"
    <dd>Mike West. <a href="https://w3c.github.io/webappsec-csp/"><cite>Content Security Policy Level 3</cite></a>. URL: <a href="https://w3c.github.io/webappsec-csp/">https://w3c.github.io/webappsec-csp/</a>
    <dt id="biblio-webappsec-fetch-metadata-1">[WEBAPPSEC-FETCH-METADATA-1]
    <dd>Fetch Metadata Request Headers URL: <a href="https://w3c.github.io/webappsec-fetch-metadata/">https://w3c.github.io/webappsec-fetch-metadata/</a>
+   <dt id="biblio-webidl①">[WebIDL]
+   <dd>Boris Zbarsky. <a href="https://heycam.github.io/webidl/"><cite>Web IDL</cite></a>. URL: <a href="https://heycam.github.io/webidl/">https://heycam.github.io/webidl/</a>
   </dl>
   <h2 class="no-num no-ref heading settled" id="idl-index"><span class="content">IDL Index</span><a class="self-link" href="#idl-index"></a></h2>
 <pre class="idl highlight def"><c- b>partial</c-> <c- b>interface</c-> <a class="idl-code" data-link-type="interface" href="https://html.spec.whatwg.org/multipage/window-object.html#window"><c- g>Window</c-></a> {

tests/github/WICG/background-fetch/index.html

@@ -758,6 +758,7 @@ <h2 class="no-num no-toc no-ref" id="contents">Table of Contents</h2>
      <a href="#references"><span class="secno"></span> <span class="content">References</span></a>
      <ol class="toc">
       <li><a href="#normative"><span class="secno"></span> <span class="content">Normative References</span></a>
+      <li><a href="#informative"><span class="secno"></span> <span class="content">Informative References</span></a>
      </ol>
     <li><a href="#idl-index"><span class="secno"></span> <span class="content">IDL Index</span></a>
     <li><a href="#issues-index"><span class="secno"></span> <span class="content">Issues Index</span></a>
@@ -3465,6 +3466,11 @@ <h3 class="no-num no-ref heading settled" id="normative"><span class="content">N
    <dt id="biblio-webidl">[WebIDL]
    <dd>Boris Zbarsky. <a href="https://heycam.github.io/webidl/"><cite>Web IDL</cite></a>. 15 December 2016. ED. URL: <a href="https://heycam.github.io/webidl/">https://heycam.github.io/webidl/</a>
   </dl>
+  <h3 class="no-num no-ref heading settled" id="informative"><span class="content">Informative References</span><a class="self-link" href="#informative"></a></h3>
+  <dl>
+   <dt id="biblio-webdriver①">[WebDriver]
+   <dd>Simon Stewart; David Burns. <a href="https://www.w3.org/TR/webdriver1/"><cite>WebDriver</cite></a>. 5 June 2018. REC. URL: <a href="https://www.w3.org/TR/webdriver1/">https://www.w3.org/TR/webdriver1/</a>
+  </dl>
   <h2 class="no-num no-ref heading settled" id="idl-index"><span class="content">IDL Index</span><a class="self-link" href="#idl-index"></a></h2>
 <pre class="idl highlight def"><c- b>partial</c-> <c- b>interface</c-> <a class="idl-code" data-link-type="interface" href="https://w3c.github.io/ServiceWorker/#serviceworkerglobalscope"><c- g>ServiceWorkerGlobalScope</c-></a> {
   <c- b>attribute</c-> <a data-link-type="idl-name" href="https://html.spec.whatwg.org/multipage/webappapis.html#eventhandler"><c- n>EventHandler</c-></a> <a data-type="EventHandler" href="#dom-serviceworkerglobalscope-onbackgroundfetchsuccess"><code><c- g>onbackgroundfetchsuccess</c-></code></a>;

tests/github/WICG/background-sync/spec/PeriodicBackgroundSync-index.html

@@ -1769,6 +1769,8 @@ <h3 class="no-num no-ref heading settled" id="normative"><span class="content">N
    <dd>S. Bradner. <a href="https://datatracker.ietf.org/doc/html/rfc2119"><cite>Key words for use in RFCs to Indicate Requirement Levels</cite></a>. March 1997. Best Current Practice. URL: <a href="https://datatracker.ietf.org/doc/html/rfc2119">https://datatracker.ietf.org/doc/html/rfc2119</a>
    <dt id="biblio-service-workers-1">[SERVICE-WORKERS-1]
    <dd>Alex Russell; et al. <a href="https://www.w3.org/TR/service-workers-1/"><cite>Service Workers 1</cite></a>. 19 November 2019. CR. URL: <a href="https://www.w3.org/TR/service-workers-1/">https://www.w3.org/TR/service-workers-1/</a>
+   <dt id="biblio-service-workers-1①">[SERVICE-WORKERS-1]
+   <dd>Alex Russell; et al. <a href="https://www.w3.org/TR/service-workers-1/"><cite>Service Workers 1</cite></a>. 19 November 2019. CR. URL: <a href="https://www.w3.org/TR/service-workers-1/">https://www.w3.org/TR/service-workers-1/</a>
    <dt id="biblio-web-background-sync">[WEB-BACKGROUND-SYNC]
    <dd><a href="https://wicg.github.io/background-sync/spec/"><cite>Web Background Synchronization</cite></a>. cg-draft. URL: <a href="https://wicg.github.io/background-sync/spec/">https://wicg.github.io/background-sync/spec/</a>
    <dt id="biblio-webidl">[WebIDL]

tests/github/WICG/background-sync/spec/index.html

@@ -698,6 +698,7 @@ <h2 class="no-num no-toc no-ref" id="contents">Table of Contents</h2>
      <a href="#references"><span class="secno"></span> <span class="content">References</span></a>
      <ol class="toc">
       <li><a href="#normative"><span class="secno"></span> <span class="content">Normative References</span></a>
+      <li><a href="#informative"><span class="secno"></span> <span class="content">Informative References</span></a>
      </ol>
     <li><a href="#idl-index"><span class="secno"></span> <span class="content">IDL Index</span></a>
    </ol>
@@ -1359,6 +1360,11 @@ <h3 class="no-num no-ref heading settled" id="normative"><span class="content">N
    <dt id="biblio-webidl">[WebIDL]
    <dd>Boris Zbarsky. <a href="https://heycam.github.io/webidl/"><cite>Web IDL</cite></a>. 15 December 2016. ED. URL: <a href="https://heycam.github.io/webidl/">https://heycam.github.io/webidl/</a>
   </dl>
+  <h3 class="no-num no-ref heading settled" id="informative"><span class="content">Informative References</span><a class="self-link" href="#informative"></a></h3>
+  <dl>
+   <dt id="biblio-service-workers-1①">[SERVICE-WORKERS-1]
+   <dd>Alex Russell; et al. <a href="https://www.w3.org/TR/service-workers-1/"><cite>Service Workers 1</cite></a>. 19 November 2019. CR. URL: <a href="https://www.w3.org/TR/service-workers-1/">https://www.w3.org/TR/service-workers-1/</a>
+  </dl>
   <h2 class="no-num no-ref heading settled" id="idl-index"><span class="content">IDL Index</span><a class="self-link" href="#idl-index"></a></h2>
 <pre class="idl highlight def"><c- b>partial</c-> <c- b>interface</c-> <a class="idl-code" data-link-type="interface" href="https://slightlyoff.github.io/ServiceWorker/spec/service_worker/index.html#service-worker-registration-interface"><c- g>ServiceWorkerRegistration</c-></a> {
   <c- b>readonly</c-> <c- b>attribute</c-> <a data-link-type="idl-name" href="#syncmanager"><c- n>SyncManager</c-></a> <a data-readonly data-type="SyncManager" href="#dom-serviceworkerregistration-sync"><code><c- g>sync</c-></code></a>;

tests/github/WICG/construct-stylesheets/index.html

@@ -2102,6 +2102,7 @@ <h2 class="no-num no-toc no-ref" id="contents">Table of Contents</h2>
      <a href="#references"><span class="secno"></span> <span class="content">References</span></a>
      <ol class="toc">
       <li><a href="#normative"><span class="secno"></span> <span class="content">Normative References</span></a>
+      <li><a href="#informative"><span class="secno"></span> <span class="content">Informative References</span></a>
      </ol>
     <li><a href="#idl-index"><span class="secno"></span> <span class="content">IDL Index</span></a>
    </ol>
@@ -2827,6 +2828,11 @@ <h3 class="no-num no-ref heading settled" id="normative"><span class="content">N
    <dt id="biblio-webidl">[WebIDL]
    <dd>Boris Zbarsky. <a href="https://heycam.github.io/webidl/"><cite>Web IDL</cite></a>. 15 December 2016. ED. URL: <a href="https://heycam.github.io/webidl/">https://heycam.github.io/webidl/</a>
   </dl>
+  <h3 class="no-num no-ref heading settled" id="informative"><span class="content">Informative References</span><a class="self-link" href="#informative"></a></h3>
+  <dl>
+   <dt id="biblio-css-cascade-5①">[CSS-CASCADE-5]
+   <dd>Elika Etemad; Miriam Suzanne; Tab Atkins Jr.. <a href="https://www.w3.org/TR/css-cascade-5/"><cite>CSS Cascading and Inheritance Level 5</cite></a>. 19 March 2021. WD. URL: <a href="https://www.w3.org/TR/css-cascade-5/">https://www.w3.org/TR/css-cascade-5/</a>
+  </dl>
   <h2 class="no-num no-ref heading settled" id="idl-index"><span class="content">IDL Index</span><a class="self-link" href="#idl-index"></a></h2>
 <pre class="idl highlight def"><c- b>partial</c-> <c- b>interface</c-> <a class="idl-code" data-link-type="interface" href="https://drafts.csswg.org/cssom-1/#cssstylesheet"><c- g>CSSStyleSheet</c-></a> {
   <a class="idl-code" data-link-type="constructor" href="#dom-cssstylesheet-cssstylesheet"><c- g>constructor</c-></a>(<c- b>optional</c-> <a data-link-type="idl-name" href="#dictdef-cssstylesheetinit"><c- n>CSSStyleSheetInit</c-></a> <a href="#dom-cssstylesheet-cssstylesheet-options-options"><code><c- g>options</c-></code></a> = {});