fix: shiki and pjs dep errors (#842)

_tasks/generate_artifacts.ts

@@ -1,37 +1,59 @@
 import { hex } from "../crypto/mod.ts"
 import { testUser } from "../crypto/mod.ts"
 import * as $ from "../deps/scale.ts"
+import { emptyDir } from "../deps/std/fs.ts"
 import * as path from "../deps/std/path.ts"
 import dprintConfig from "../dprint.json" assert { type: "json" }
-import { DEFAULT_TEST_USER_COUNT, publicKeysUrl } from "../providers/frame/common.ts"
+import { compress } from "../util/compression.ts"
 
-const GENERATION_NOTICE = `// This file was generated by \`_tasks/downloads.ts\``
+export const TEST_USER_COUNT = 100_000
 
-const response = await fetch(
-  dprintConfig.plugins.find((v) => v.startsWith("https://plugins.dprint.dev/typescript-"))!,
-)
-const wasm = new TextDecoder().decode(hex.encodeBuf(new Uint8Array(await response.arrayBuffer())))
-const tsFormatterPath = new URL("../util/tsFormatter.ts", import.meta.url)
-await Deno.writeTextFile(
-  tsFormatterPath,
-  `${GENERATION_NOTICE}
-import { createFromBuffer } from "../deps/dprint.ts"
-import dprintConfig from "../dprint.json" assert { type: "json" }
-import { hex } from "../crypto/mod.ts"
+const artifacts: Record<string, () => Promise<Uint8Array>> = {
+  async tsFormatterWasm() {
+    const url = dprintConfig.plugins.find((v) =>
+      v.startsWith("https://plugins.dprint.dev/typescript-")
+    )!
+    return await fetchBinary(url)
+  },
+  async shikiWasm() {
+    return await fetchBinary("https://unpkg.com/[email protected]/dist/onig.wasm")
+  },
+  async testUserPublicKeysData() {
+    const publicKeys = []
+    for (let i = 0; i < TEST_USER_COUNT; i++) {
+      publicKeys.push(testUser(i).publicKey)
+    }
+    const encoded = $.array($.sizedUint8Array(32)).encode(publicKeys)
+    return encoded
+  },
+}
 
-const wasm = "${wasm}"
+const artifactsDir = "util/_artifacts/"
 
-export const tsFormatter = createFromBuffer(hex.decode(wasm))
-const { indentWidth, lineWidth, typescript: config } = dprintConfig
-tsFormatter.setConfig({ indentWidth, lineWidth }, config)
-`,
+await emptyDir(artifactsDir)
+
+await Promise.all(
+  Object.entries(artifacts).map(async ([name, fn]) => {
+    const data = await fn()
+    let code
+    const compressed = await compress(data)
+    if (compressed.length < data.length) {
+      code = `await decompress(hex.decode("${hex.encode(compressed)}"))`
+    } else {
+      code = `hex.decode("${hex.encode(data)}")`
+    }
+
+    const file = `
+// This file was generated by \`_tasks/generate_artifacts.ts\`
+import { hex } from "../../crypto/mod.ts"
+import { decompress } from "../compression.ts"
+export const ${name} = ${code}
+`.trimStart()
+    await Deno.writeTextFile(path.join(artifactsDir, name + ".ts"), file)
+  }),
 )
 
-const publicKeys = []
-for (let i = 0; i < DEFAULT_TEST_USER_COUNT; i++) {
-  publicKeys.push(testUser(i).publicKey)
+async function fetchBinary(url: string | URL) {
+  const data = await fetch(url).then((r) => r.arrayBuffer())
+  return new Uint8Array(data)
 }
-await Deno.writeFile(
-  path.fromFileUrl(publicKeysUrl),
-  $.array($.sizedUint8Array(32)).encode(publicKeys),
-)

_tasks/udd.ts

@@ -5,7 +5,10 @@ const pending = [udd("_tasks/udd.ts")]
 
 for await (
   const entry of fs.walk(".", {
-    match: [path.globToRegExp("deps/**/*.ts")],
+    match: [
+      path.globToRegExp("examples/**/*.ts"),
+      path.globToRegExp("deps/**/*.ts"),
+    ],
   })
 ) pending.push(udd(entry.path))
 

deno.jsonc

@@ -12,7 +12,7 @@
   "lock": false,
   "lint": {
     "files": {
-      "exclude": ["target"],
+      "exclude": ["target", "util/_artifacts"],
       "include": ["."]
     },
     "rules": {

deps/shiki.ts

@@ -0,0 +1,22 @@
+import * as shiki from "https://esm.sh/[email protected]?bundle"
+import typescriptLang from "https://unpkg.com/[email protected]/languages/typescript.tmLanguage.json" assert {
+  type: "json",
+}
+import githubDark from "https://unpkg.com/[email protected]/themes/github-dark.json" assert {
+  type: "json",
+}
+import { shikiWasm } from "../util/_artifacts/shikiWasm.ts"
+
+shiki.setWasm(shikiWasm)
+
+// A gross hack to get the data url to work, since shiki requires the cdn to end with a slash
+shiki.setCDN("data:application/")
+
+export const highlighterPromise = shiki.getHighlighter({
+  theme: githubDark as any,
+  langs: [{
+    id: "ts",
+    scopeName: "source.ts",
+    path: "json;base64," + btoa(JSON.stringify(typescriptLang)),
+  }],
+})

dprint.json

@@ -14,10 +14,7 @@
   "includes": [
     "**.{dockerfile,json,md,toml,ts,tsx}"
   ],
-  "excludes": [
-    "target",
-    "util/tsFormatter.ts"
-  ],
+  "excludes": ["target", "util/_artifacts"],
   "plugins": [
     "https://plugins.dprint.dev/dockerfile-0.3.0.wasm",
     "https://plugins.dprint.dev/json-0.17.0.wasm",

examples/sign/pjs.eg.ts

@@ -9,8 +9,8 @@
 import { ss58 } from "capi"
 import { pjsSender, PjsSigner } from "capi/patterns/compat/pjs_sender.ts"
 import { signature } from "capi/patterns/signature/polkadot.ts"
-import { createPair } from "https://deno.land/x/[email protected].25/keyring/mod.ts"
-import { TypeRegistry } from "https://deno.land/x/[email protected].25/types/mod.ts"
+import { createPair } from "https://deno.land/x/[email protected].34/keyring/mod.ts"
+import { TypeRegistry } from "https://deno.land/x/[email protected].34/types/mod.ts"
 import { Balances, chain, createUsers, System } from "polkadot_dev/mod.js"
 
 const { alexa, billy } = await createUsers()

server/factories.ts

@@ -1,11 +1,8 @@
-import * as shiki from "https://esm.sh/[email protected]?bundle"
 import { escapeHtml } from "../deps/escape.ts"
+import { highlighterPromise } from "../deps/shiki.ts"
 import { Status } from "../deps/std/http.ts"
 import { CacheBase } from "../util/cache/base.ts"
 
-shiki.setCDN("https://unpkg.com/shiki/")
-export const highlighterPromise = shiki.getHighlighter({ theme: "github-dark", langs: ["ts"] })
-
 const codeTtl = 60_000
 
 export async function code(cache: CacheBase, request: Request, genCode: () => Promise<string>) {

util/compression.ts

@@ -0,0 +1,17 @@
+import { Buffer } from "../deps/std/streams.ts"
+
+export async function compress(data: Uint8Array) {
+  const compressed = new Buffer()
+  await new Buffer(data).readable
+    .pipeThrough(new CompressionStream("gzip"))
+    .pipeTo(compressed.writable)
+  return compressed.bytes()
+}
+
+export async function decompress(data: Uint8Array) {
+  const decompressed = new Buffer()
+  await new Buffer(data).readable
+    .pipeThrough(new DecompressionStream("gzip"))
+    .pipeTo(decompressed.writable)
+  return decompressed.bytes()
+}