hash global imports

src/client/search.js

@@ -8,7 +8,7 @@ const resultsContainer = document.querySelector("#observablehq-search-results");
 const activeClass = "observablehq-link-active";
 let currentValue;
 
-const index = await fetch(import.meta.resolve(global.__minisearch))
+const index = await fetch(import.meta.resolve("observablehq:minisearch.json"))
   .then((response) => {
     if (!response.ok) throw new Error(`unable to load minisearch.json: ${response.status}`);
     return response.json();

src/deploy.ts

@@ -62,7 +62,7 @@ export interface DeployEffects extends ConfigEffects, TtyEffects, AuthEffects {
   output: NodeJS.WritableStream;
   visitFiles: (root: string) => Generator<string>;
   stat: (path: string) => Promise<Stats>;
-  build: ({config, addPublic}: BuildOptions, effects?: BuildEffects) => Promise<void>;
+  build: ({config}: BuildOptions, effects?: BuildEffects) => Promise<void>;
   readCacheFile: (sourceRoot: string, path: string) => Promise<string>;
 }
 

src/javascript/module.ts

@@ -1,12 +1,16 @@
+import type {Hash} from "node:crypto";
 import {createHash} from "node:crypto";
 import {accessSync, constants, existsSync, readFileSync, statSync} from "node:fs";
 import {readFile} from "node:fs/promises";
 import {join} from "node:path/posix";
 import type {Program} from "acorn";
 import {transform, transformSync} from "esbuild";
+import {resolveNodeImport} from "../node.js";
+import {resolveNpmImport} from "../npm.js";
 import {resolvePath} from "../path.js";
+import {builtins} from "../resolvers.js";
 import {findFiles} from "./files.js";
-import {findImports} from "./imports.js";
+import {findImports, parseImports} from "./imports.js";
 import {parseProgram} from "./parse.js";
 
 export type FileInfo = {
@@ -46,6 +50,10 @@ const moduleInfoCache = new Map<string, ModuleInfo>();
  * transitive imports or files that are invalid or do not exist.
  */
 export function getModuleHash(root: string, path: string): string {
+  return getModuleHashInternal(root, path).digest("hex");
+}
+
+function getModuleHashInternal(root: string, path: string): Hash {
   const hash = createHash("sha256");
   const paths = new Set([path]);
   for (const path of paths) {
@@ -64,6 +72,36 @@ export function getModuleHash(root: string, path: string): string {
       hash.update(f.hash);
     }
   }
+  return hash;
+}
+
+/**
+ * Like getModuleHash, but further takes into consideration the resolved exact
+ * versions of any npm imports (and their transitive imports). This is needed
+ * during build because we want the hash of the built module to change if the
+ * version of an imported npm package changes.
+ */
+export async function getLocalModuleHash(root: string, path: string): Promise<string> {
+  const hash = getModuleHashInternal(root, path);
+  const info = getModuleInfo(root, path);
+  if (info) {
+    const globalPaths = new Set<string>();
+    for (const i of [...info.globalStaticImports, ...info.globalDynamicImports]) {
+      if (i.startsWith("npm:") && !builtins.has(i)) {
+        globalPaths.add(await resolveNpmImport(root, i.slice("npm:".length)));
+      } else if (!/^\w+:/.test(i)) {
+        globalPaths.add(await resolveNodeImport(root, i));
+      }
+    }
+    for (const p of globalPaths) {
+      hash.update(p);
+      for (const i of await parseImports(join(root, ".observablehq", "cache"), p)) {
+        if (i.type === "local") {
+          globalPaths.add(resolvePath(p, i.name));
+        }
+      }
+    }
+  }
   return hash.digest("hex");
 }
 

src/npm.ts

@@ -215,25 +215,25 @@ function getNpmVersionCache(root: string): Promise<Map<string, string[]>> {
   return cache;
 }
 
-async function resolveNpmVersion(root: string, specifier: NpmSpecifier): Promise<string> {
-  const {name, range} = specifier;
+async function resolveNpmVersion(root: string, {name, range}: NpmSpecifier): Promise<string> {
   if (range && /^\d+\.\d+\.\d+([-+].*)?$/.test(range)) return range; // exact version specified
   const cache = await getNpmVersionCache(root);
-  const versions = cache.get(specifier.name);
+  const versions = cache.get(name);
   if (versions) for (const version of versions) if (!range || satisfies(version, range)) return version;
   const href = `https://data.jsdelivr.com/v1/packages/npm/${name}/resolved${range ? `?specifier=${range}` : ""}`;
   let promise = npmVersionRequests.get(href);
   if (promise) return promise; // coalesce concurrent requests
   promise = (async function () {
-    process.stdout.write(`npm:${formatNpmSpecifier(specifier)} ${faint("→")} `);
+    const input = formatNpmSpecifier({name, range});
+    process.stdout.write(`npm:${input} ${faint("→")} `);
     const response = await fetch(href);
     if (!response.ok) throw new Error(`unable to fetch: ${href}`);
     const {version} = await response.json();
-    if (!version) throw new Error(`unable to resolve version: ${formatNpmSpecifier({name, range})}`);
-    const spec = formatNpmSpecifier({name, range: version});
-    process.stdout.write(`npm:${spec}\n`);
-    cache.set(specifier.name, versions ? rsort(versions.concat(version)) : [version]);
-    mkdir(join(root, ".observablehq", "cache", "_npm", spec), {recursive: true}); // disk cache
+    if (!version) throw new Error(`unable to resolve version: ${input}`);
+    const output = formatNpmSpecifier({name, range: version});
+    process.stdout.write(`npm:${output}\n`);
+    cache.set(name, versions ? rsort(versions.concat(version)) : [version]);
+    mkdir(join(root, ".observablehq", "cache", "_npm", output), {recursive: true}); // disk cache
     return version;
   })();
   promise.catch(console.error).then(() => npmVersionRequests.delete(href));

src/render.ts

@@ -76,7 +76,7 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from
 ${preview ? `\nopen({hash: ${JSON.stringify(resolvers.hash)}, eval: (body) => eval(body)});\n` : ""}${page.code
     .map(({node, id, mode}) => `\n${transpileJavaScript(node, {id, path, mode, resolveImport})}`)
     .join("")}`)}
-</script>${sidebar ? html`\n${await renderSidebar(options, resolvers.resolveLink)}` : ""}${
+</script>${sidebar ? html`\n${await renderSidebar(options, resolvers)}` : ""}${
     toc.show ? html`\n${renderToc(findHeaders(page), toc.label)}` : ""
   }
 <div id="observablehq-center">${renderHeader(page.header, resolvers)}
@@ -124,7 +124,7 @@ function registerFile(
   })});`;
 }
 
-async function renderSidebar(options: RenderOptions, resolveLink: (href: string) => string): Promise<Html> {
+async function renderSidebar(options: RenderOptions, {resolveImport, resolveLink}: Resolvers): Promise<Html> {
   const {title = "Home", pages, root, path, search} = options;
   return html`<input id="observablehq-sidebar-toggle" type="checkbox" title="Toggle sidebar">
 <label id="observablehq-sidebar-backdrop" for="observablehq-sidebar-toggle"></label>
@@ -139,7 +139,7 @@ async function renderSidebar(options: RenderOptions, resolveLink: (href: string)
       ? html`\n  <div id="observablehq-search"><input type="search" placeholder="Search"></div>
   <div id="observablehq-search-results"></div>
   <script>{${html.unsafe(
-    (await rollupClient(getClientPath("search-init.js"), root, path, {minify: true})).trim()
+    (await rollupClient(getClientPath("search-init.js"), root, path, {resolveImport, minify: true})).trim()
   )}}</script>`
       : ""
   }${pages.map((p, i) =>
@@ -159,7 +159,7 @@ async function renderSidebar(options: RenderOptions, resolveLink: (href: string)
   )}
 </nav>
 <script>{${html.unsafe(
-    (await rollupClient(getClientPath("sidebar-init.js"), root, path, {minify: true})).trim()
+    (await rollupClient(getClientPath("sidebar-init.js"), root, path, {resolveImport, minify: true})).trim()
   )}}</script>`;
 }
 

src/resolvers.ts

@@ -42,6 +42,8 @@ const defaultImports = [
 ];
 
 export const builtins = new Map<string, string>([
+  ["@observablehq/runtime", "/_observablehq/runtime.js"],
+  ["@observablehq/stdlib", "/_observablehq/stdlib.js"],
   ["npm:@observablehq/runtime", "/_observablehq/runtime.js"],
   ["npm:@observablehq/stdlib", "/_observablehq/stdlib.js"],
   ["npm:@observablehq/dot", "/_observablehq/stdlib/dot.js"], // TODO publish to npm
@@ -184,7 +186,9 @@ export async function getResolvers(
     globalImports.add(i);
   }
 
-  // Resolve npm: and bare imports.
+  // Resolve npm: and bare imports. This has the side-effect of populating the
+  // npm import cache with direct dependencies, and the node import cache with
+  // all transitive dependencies.
   for (const i of globalImports) {
     if (i.startsWith("npm:") && !builtins.has(i)) {
       resolutions.set(i, await resolveNpmImport(root, i.slice("npm:".length)));
@@ -197,9 +201,8 @@ export async function getResolvers(
     }
   }
 
-  // Follow transitive imports of npm and bare imports. This has the side-effect
-  // of populating the npm cache; the node import cache is already transitively
-  // populated above.
+  // Follow transitive imports of npm and bare imports. This populates the
+  // remainder of the npm import cache.
   for (const [key, value] of resolutions) {
     if (key.startsWith("npm:")) {
       for (const i of await resolveNpmImports(root, value)) {

src/rollup.ts

@@ -12,6 +12,7 @@ import {getStringLiteralValue, isStringLiteral} from "./javascript/source.js";
 import {resolveNpmImport} from "./npm.js";
 import {getObservableUiOrigin} from "./observableApiClient.js";
 import {isPathImport, relativePath} from "./path.js";
+import {builtins} from "./resolvers.js";
 import {Sourcemap} from "./sourcemap.js";
 import {THEMES, renderTheme} from "./theme.js";
 
@@ -54,18 +55,26 @@ export async function bundleStyles({
   return text;
 }
 
+type ImportResolver = (specifier: string) => Promise<string | undefined> | string | undefined;
+
 export async function rollupClient(
   input: string,
   root: string,
   path: string,
-  {define, keepNames, minify}: {define?: {[key: string]: string}; keepNames?: boolean; minify?: boolean} = {}
+  {
+    define,
+    keepNames,
+    minify,
+    resolveImport = getDefaultResolver(root)
+  }: {define?: {[key: string]: string}; keepNames?: boolean; minify?: boolean; resolveImport?: ImportResolver} = {}
 ): Promise<string> {
+  if (typeof resolveImport !== "function") throw new Error(`invalid resolveImport: ${resolveImport}`);
   const bundle = await rollup({
     input,
     external: [/^https:/],
     plugins: [
       nodeResolve({resolveOnly: BUNDLED_MODULES}),
-      importResolve(input, root, path),
+      importResolve(input, path, resolveImport),
       esbuild({
         format: "esm",
         platform: "browser",
@@ -74,12 +83,11 @@ export async function rollupClient(
         keepNames,
         minify,
         define: {
-          "global.__minisearch": '"./minisearch.json"',
           "process.env.OBSERVABLE_ORIGIN": JSON.stringify(String(getObservableUiOrigin()).replace(/\/$/, "")),
           ...define
         }
       }),
-      importMetaResolve(root, path)
+      importMetaResolve(path, resolveImport)
     ],
     onwarn(message, warn) {
       if (message.code === "CIRCULAR_DEPENDENCY") return;
@@ -106,37 +114,30 @@ function rewriteTypeScriptImports(code: string): string {
   return code.replace(/(?<=\bimport\(([`'"])[\w./]+)\.ts(?=\1\))/g, ".js");
 }
 
-function importResolve(input: string, root: string, path: string): Plugin {
+function getDefaultResolver(root: string): ImportResolver {
+  return (specifier: string) => resolveImport(root, specifier);
+}
+
+export async function resolveImport(root: string, specifier: string): Promise<string | undefined> {
+  return BUNDLED_MODULES.includes(specifier)
+    ? undefined
+    : builtins.has(specifier)
+    ? builtins.get(specifier)
+    : specifier.startsWith("observablehq:")
+    ? `/_observablehq/${specifier.slice("observablehq:".length)}${extname(specifier) ? "" : ".js"}`
+    : specifier.startsWith("npm:")
+    ? await resolveNpmImport(root, specifier.slice("npm:".length))
+    : !/^[a-z]:\\/i.test(specifier) && !isPathImport(specifier)
+    ? await resolveNpmImport(root, specifier)
+    : undefined;
+}
+
+function importResolve(input: string, path: string, resolveImport: ImportResolver): Plugin {
   async function resolve(specifier: string | AstNode): Promise<ResolveIdResult> {
-    return typeof specifier !== "string" || specifier === input
-      ? null
-      : specifier.startsWith("observablehq:")
-      ? {id: relativePath(path, `/_observablehq/${specifier.slice("observablehq:".length)}${extname(specifier) ? "" : ".js"}`), external: true} // prettier-ignore
-      : specifier === "npm:@observablehq/runtime"
-      ? {id: relativePath(path, "/_observablehq/runtime.js"), external: true}
-      : specifier === "npm:@observablehq/stdlib" || specifier === "@observablehq/stdlib"
-      ? {id: relativePath(path, "/_observablehq/stdlib.js"), external: true}
-      : specifier === "npm:@observablehq/dot"
-      ? {id: relativePath(path, "/_observablehq/stdlib/dot.js"), external: true} // TODO publish to npm
-      : specifier === "npm:@observablehq/duckdb"
-      ? {id: relativePath(path, "/_observablehq/stdlib/duckdb.js"), external: true} // TODO publish to npm
-      : specifier === "npm:@observablehq/inputs"
-      ? {id: relativePath(path, "/_observablehq/stdlib/inputs.js"), external: true}
-      : specifier === "npm:@observablehq/mermaid"
-      ? {id: relativePath(path, "/_observablehq/stdlib/mermaid.js"), external: true} // TODO publish to npm
-      : specifier === "npm:@observablehq/tex"
-      ? {id: relativePath(path, "/_observablehq/stdlib/tex.js"), external: true} // TODO publish to npm
-      : specifier === "npm:@observablehq/sqlite"
-      ? {id: relativePath(path, "/_observablehq/stdlib/sqlite.js"), external: true} // TODO publish to npm
-      : specifier === "npm:@observablehq/xlsx"
-      ? {id: relativePath(path, "/_observablehq/stdlib/xlsx.js"), external: true} // TODO publish to npm
-      : specifier === "npm:@observablehq/zip"
-      ? {id: relativePath(path, "/_observablehq/stdlib/zip.js"), external: true} // TODO publish to npm
-      : specifier.startsWith("npm:")
-      ? {id: relativePath(path, await resolveNpmImport(root, specifier.slice("npm:".length))), external: true}
-      : !/^[a-z]:\\/i.test(specifier) && !isPathImport(specifier) && !BUNDLED_MODULES.includes(specifier) // e.g., inputs.js imports "htl"
-      ? {id: relativePath(path, await resolveNpmImport(root, specifier)), external: true}
-      : null;
+    if (typeof specifier !== "string" || specifier === input) return null;
+    const resolution = await resolveImport(specifier);
+    if (resolution) return {id: relativePath(path, resolution), external: true};
+    return null;
   }
   return {
     name: "resolve-import",
@@ -145,7 +146,7 @@ function importResolve(input: string, root: string, path: string): Plugin {
   };
 }
 
-function importMetaResolve(root: string, path: string): Plugin {
+function importMetaResolve(path: string, resolveImport: ImportResolver): Plugin {
   return {
     name: "resolve-import-meta-resolve",
     async transform(code) {
@@ -173,10 +174,8 @@ function importMetaResolve(root: string, path: string): Plugin {
       for (const node of resolves) {
         const source = node.arguments[0];
         const specifier = getStringLiteralValue(source as StringLiteral);
-        if (specifier.startsWith("npm:")) {
-          const resolution = relativePath(path, await resolveNpmImport(root, specifier.slice("npm:".length)));
-          output.replaceLeft(source.start, source.end, JSON.stringify(resolution));
-        }
+        const resolution = await resolveImport(specifier);
+        if (resolution) output.replaceLeft(source.start, source.end, JSON.stringify(relativePath(path, resolution)));
       }
 
       return {code: String(output)};

test/build-test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert";
 import {existsSync, readdirSync, statSync} from "node:fs";
-import {mkdir, mkdtemp, open, readFile, rm, writeFile} from "node:fs/promises";
+import {mkdir, mkdtemp, readFile, rm, writeFile} from "node:fs/promises";
 import os from "node:os";
 import {join, normalize, relative} from "node:path/posix";
 import {PassThrough} from "node:stream";
@@ -40,23 +40,15 @@ describe("build", () => {
       const expectedDir = join(outputRoot, outname);
       const generate = !existsSync(expectedDir) && process.env.CI !== "true";
       const outputDir = generate ? expectedDir : actualDir;
-      const addPublic = name.endsWith("-public");
 
       await rm(actualDir, {recursive: true, force: true});
       if (generate) console.warn(`! generating ${expectedDir}`);
       const config = {...(await readConfig(undefined, path)), output: outputDir};
-      await build({config, addPublic}, new TestEffects(outputDir, join(config.root, ".observablehq", "cache")));
-
-      // In the addPublic case, we don’t want to test the contents of the public
-      // files because they change often; replace them with empty files so we
-      // can at least check that the expected files exist.
-      if (addPublic) {
-        const publicDir = join(outputDir, "_observablehq");
-        for (const file of findFiles(publicDir)) {
-          if (file.endsWith(".json")) continue; // e.g., minisearch.json
-          await (await open(join(publicDir, file), "w")).close();
-        }
-      }
+      await build({config}, new TestEffects(outputDir, join(config.root, ".observablehq", "cache")));
+
+      // For non-public tests (most of them), we don’t want to test the contents
+      // of the _observablehq files because they change often.
+      if (!name.endsWith("-public")) await rm(join(outputDir, "_observablehq"), {recursive: true, force: true});
 
       if (generate) return;