feat: initial codegen (#229)

codegen/codecVisitor.ts

@@ -0,0 +1,212 @@
+import * as M from "../frame_metadata/mod.ts";
+import { Decl, Files, getCodecPath, getName, getRawCodecPath, importSource, S } from "./utils.ts";
+
+export function createCodecVisitor(
+  tys: M.Ty[],
+  decls: Decl[],
+  typeVisitor: M.TyVisitor<S>,
+  files: Files,
+) {
+  ["import { $, $null, $era } from", S.string(importSource)];
+  const namespaceImports = new Set<string>();
+  const codecs: S[] = [];
+
+  files.set("codecs.ts", {
+    getContent: () => [
+      "\n",
+      ["import { ChainError, BitSequence, Era, $, $era, $null } from", S.string(importSource)],
+      [`import type * as t from "./mod.ts"`],
+      ...codecs,
+      [
+        "export const _all: $.AnyCodec[] =",
+        S.array(tys.map((ty) => getName(getRawCodecPath(ty)))),
+      ],
+    ],
+  });
+
+  const compactCodecVisitor = new M.TyVisitor<string | null>(tys, {
+    unitStruct: () => "$null",
+    wrapperStruct(_, inner) {
+      return this.visit(inner);
+    },
+    tupleStruct: () => null,
+    objectStruct: () => null,
+    option: () => null,
+    result: () => null,
+    never: () => null,
+    stringUnion: () => null,
+    taggedUnion: () => null,
+    array: () => null,
+    sizedArray: () => null,
+    primitive: (ty) => {
+      const lookup: Partial<Record<typeof ty.kind, string>> = {
+        u8: "$.compactU8",
+        u16: "$.compactU16",
+        u32: "$.compactU32",
+        u64: "$.compactU64",
+        u128: "$.compactU128",
+        u256: "$.compactU256",
+      };
+      return lookup[ty.kind] ?? null;
+    },
+    compact: () => null,
+    bitSequence: () => null,
+    circular: () => null,
+  });
+
+  return new M.TyVisitor<S>(tys, {
+    unitStruct(ty) {
+      return addCodecDecl(ty, "$null");
+    },
+    wrapperStruct(ty, inner) {
+      return addCodecDecl(ty, this.visit(inner));
+    },
+    tupleStruct(ty, members) {
+      return addCodecDecl(ty, ["$.tuple(", members.map((x) => [this.visit(x), ","]), ")"]);
+    },
+    objectStruct(ty) {
+      return addCodecDecl(
+        ty,
+        [
+          "$.object(",
+          ty.fields.map(
+            (x) => [S.array([S.string(x.name!), this.visit(x.ty)]), ","],
+          ),
+          ")",
+        ],
+      );
+    },
+    option(ty, some) {
+      return addCodecDecl(ty, ["$.option(", this.visit(some), ")"]);
+    },
+    result(ty, ok, err) {
+      return addCodecDecl(ty, ["$.result(", this.visit(ok), ",", [
+        "$.instance(ChainError<",
+        fixType(typeVisitor.visit(err)),
+        `>, ["value", `,
+        this.visit(err),
+        "])",
+      ], ")"]);
+    },
+    never(ty) {
+      return addCodecDecl(ty, "$.never");
+    },
+    stringUnion(ty) {
+      return addCodecDecl(ty, [
+        "$.stringUnion(",
+        S.object(...ty.members.map((x): [S, S] => [x.index, S.string(x.name)])),
+        ")",
+      ]);
+    },
+    taggedUnion(ty) {
+      return addCodecDecl(
+        ty,
+        [
+          `$.taggedUnion("type",`,
+          S.object(
+            ...ty.members.map(({ fields, name: type, index }): [S, S] => {
+              let props: S[];
+              if (fields.length === 0) {
+                props = [];
+              } else if (fields[0]!.name === undefined) {
+                // Tuple variant
+                const value = fields.length === 1
+                  ? this.visit(fields[0]!.ty)
+                  : ["$.tuple(", fields.map((f) => [this.visit(f.ty), ","]), ")"];
+                props = [S.array([S.string("value"), value])];
+              } else {
+                // Object variant
+                props = fields.map((field) =>
+                  S.array([
+                    S.string(field.name!),
+                    this.visit(field.ty),
+                  ])
+                );
+              }
+              return [index, S.array([S.string(type), ...props])];
+            }),
+          ),
+          ")",
+        ],
+      );
+    },
+    uint8array(ty) {
+      return addCodecDecl(ty, "$.uint8array");
+    },
+    array(ty) {
+      return addCodecDecl(ty, ["$.array(", this.visit(ty.typeParam), ")"]);
+    },
+    sizedUint8Array(ty) {
+      return addCodecDecl(ty, `$.sizedUint8array(${ty.len})`);
+    },
+    sizedArray(ty) {
+      return addCodecDecl(ty, ["$.sizedArray(", this.visit(ty.typeParam), ",", ty.len, ")"]);
+    },
+    primitive(ty) {
+      return addCodecDecl(ty, getCodecPath(tys, ty)!);
+    },
+    compact(ty) {
+      const result = compactCodecVisitor.visit(ty.typeParam);
+      if (result) return addCodecDecl(ty, result);
+      throw new Error(
+        "Cannot create compact codec for " + S.toString(typeVisitor.visit(ty.typeParam)),
+      );
+    },
+    bitSequence(ty) {
+      return addCodecDecl(ty, "$.bitSequence");
+    },
+    map(ty, key, val) {
+      return addCodecDecl(ty, ["$.map(", this.visit(key), ",", this.visit(val), ")"]);
+    },
+    set(ty, val) {
+      return addCodecDecl(ty, ["$.set(", this.visit(val), ")"]);
+    },
+    era(ty) {
+      return addCodecDecl(ty, "$era");
+    },
+    circular(ty) {
+      return ["$.deferred(() =>", getName(getRawCodecPath(ty)), ")"];
+    },
+  });
+
+  function addCodecDecl(ty: M.Ty, value: S) {
+    const rawPath = getRawCodecPath(ty);
+    if (ty.path.length > 1) {
+      namespaceImports.add(ty.path[0]!);
+    }
+    codecs.push([
+      ["export const", getName(rawPath)],
+      ": $.Codec<",
+      fixType(typeVisitor.visit(ty)),
+      "> =",
+      value,
+    ]);
+    const path = getCodecPath(tys, ty);
+    // Deduplicate -- metadata has redundant entries (e.g. pallet_collective::RawOrigin)
+    if (path !== rawPath && path !== value && !decls.some((x) => x.path === path)) {
+      decls.push({
+        path,
+        code: [
+          ["export const", getName(path)],
+          ": $.Codec<",
+          typeVisitor.visit(ty),
+          "> =",
+          rawPath,
+        ],
+      });
+    }
+    return getName(rawPath);
+  }
+
+  /**
+   * Prefix generated types with `t.`
+   * e.g. `[Compact<u8>, foo.Bar, Uint8Array]` -> `[t.Compact<t.u8>, t.foo.Bar, Uint8Array]`
+   */
+  function fixType(type: S) {
+    return S.toString(type).replace(
+      // Matches paths (`a.b.c`) that either contain a `.`, or are a number type (either `u123` or `Compact`)
+      /\b([\w\$]+\.[\w\.$]+|u\d+|Compact)\b/g,
+      (x) => "t." + x,
+    );
+  }
+}

codegen/genMetadata.ts

@@ -0,0 +1,107 @@
+import * as M from "../frame_metadata/mod.ts";
+import { Decl, getPath, getRawCodecPath, makeDocComment, S } from "./utils.ts";
+
+export function genMetadata(metadata: M.Metadata, decls: Decl[]) {
+  const { tys, extrinsic, pallets } = metadata;
+
+  const isUnitVisitor = new M.TyVisitor<boolean>(tys, {
+    unitStruct: () => true,
+    wrapperStruct(_, inner) {
+      return this.visit(inner);
+    },
+    tupleStruct: () => false,
+    objectStruct: () => false,
+    option: () => false,
+    result: () => false,
+    never: () => false,
+    stringUnion: () => false,
+    taggedUnion: () => false,
+    array: () => false,
+    sizedArray: () => false,
+    primitive: () => false,
+    compact: () => false,
+    bitSequence: () => false,
+    circular: () => false,
+  });
+
+  decls.push({
+    path: "_metadata.extrinsic",
+    code: [
+      "export const extrinsic =",
+      S.object(
+        ["version", extrinsic.version],
+        ["extras", getExtrasCodec(extrinsic.signedExtensions.map((x) => [x.ident, x.ty]))],
+        [
+          "additional",
+          getExtrasCodec(extrinsic.signedExtensions.map((x) => [x.ident, x.additionalSigned])),
+        ],
+      ),
+    ],
+  });
+  for (const pallet of pallets) {
+    for (const entry of pallet.storage?.entries ?? []) {
+      decls.push({
+        path: `${pallet.name}.${entry.name}`,
+        code: [
+          makeDocComment(entry.docs),
+          `export const ${entry.name} =`,
+          S.object(
+            ["type", S.string(entry.type)],
+            ["modifier", S.string(entry.modifier)],
+            [
+              "hashers",
+              entry.type === "Map" ? JSON.stringify(entry.hashers) : "[]",
+            ],
+            [
+              "key",
+              entry.type === "Map"
+                ? entry.hashers.length === 1
+                  ? ["$.tuple(", getRawCodecPath(tys[entry.key]!), ")"]
+                  : getRawCodecPath(tys[entry.key]!)
+                : "[]",
+            ],
+            ["value", getRawCodecPath(tys[entry.value]!)],
+          ),
+        ],
+      });
+    }
+    if (pallet.calls) {
+      const ty = tys[pallet.calls.ty]! as M.Ty & M.UnionTyDef;
+      const isStringUnion = ty.members.every((x) => !x.fields.length);
+      for (const call of ty.members) {
+        const typeName = isStringUnion ? S.string(call.name) : getPath(tys, ty)! + "." + call.name;
+        const [params, data]: [S, S] = call.fields.length
+          ? call.fields[0]!.name
+            ? [`value: Omit<${typeName}, "type">`, ["{ ...value, type:", S.string(call.name), "}"]]
+            : [[call.fields.length > 1 ? "..." : "", `value: ${typeName}["value"]`], [
+              "{ ...value, type:",
+              S.string(call.name),
+              "}",
+            ]]
+          : ["", isStringUnion ? S.string(call.name) : S.object(["type", S.string(call.name)])];
+        decls.push({
+          path: `${pallet.name}.${call.name}`,
+          code: [
+            makeDocComment(call.docs),
+            "export function",
+            call.name,
+            ["(", params, ")"],
+            [":", typeName],
+            ["{ return", data, "}"],
+          ],
+        });
+      }
+    }
+  }
+
+  decls.push({
+    path: "_metadata.types",
+    code: "export const types = _codec._all",
+  });
+
+  function getExtrasCodec(xs: [string, number][]) {
+    return S.array(
+      xs.filter((x) => !isUnitVisitor.visit(x[1])).map((x) => getRawCodecPath(tys[x[1]]!)),
+    );
+  }
+}

codegen/mod.ts

@@ -0,0 +1,66 @@
+import { tsFormatter } from "../deps/dprint.ts";
+import * as path from "../deps/std/path.ts";
+import * as M from "../frame_metadata/mod.ts";
+import { createCodecVisitor } from "./codecVisitor.ts";
+import { genMetadata } from "./genMetadata.ts";
+import { createTypeVisitor } from "./typeVisitor.ts";
+import { Decl, Files, importSource, printDecls, S } from "./utils.ts";
+
+export async function run(metadataFile: string, outputDir: string) {
+  const metadata = M.fromPrefixedHex(await Deno.readTextFile(metadataFile));
+  const output = codegen(metadata);
+  const errors = [];
+  try {
+    await Deno.remove(outputDir, { recursive: true });
+  } catch (e) {
+    if (!(e instanceof Deno.errors.NotFound)) {
+      throw e;
+    }
+  }
+  await Deno.mkdir(outputDir, { recursive: true });
+  for (const [relativePath, file] of output.entries()) {
+    const outputPath = path.join(outputDir, relativePath);
+    const content = S.toString(file.getContent());
+    try {
+      const formatted = tsFormatter.formatText("gen.ts", content);
+      await Deno.writeTextFile(outputPath, formatted);
+    } catch (e) {
+      await Deno.writeTextFile(outputPath, content);
+      errors.push(e);
+    }
+  }
+  if (errors.length) {
+    throw errors;
+  }
+}
+
+export function codegen(metadata: M.Metadata): Files {
+  const decls: Decl[] = [];
+
+  const { tys } = metadata;
+
+  const files: Files = new Map();
+
+  decls.push({
+    path: "_",
+    code: [
+      "\n",
+      ["import { ChainError, BitSequence, Era, $ } from", S.string(importSource)],
+      [`import * as _codec from "./codecs.ts"`],
+    ],
+  });
+
+  const typeVisitor = createTypeVisitor(tys, decls);
+  const codecVisitor = createCodecVisitor(tys, decls, typeVisitor, files);
+
+  for (const ty of metadata.tys) {
+    typeVisitor.visit(ty);
+    codecVisitor.visit(ty);
+  }
+
+  genMetadata(metadata, decls);
+
+  files.set("mod.ts", { getContent: () => printDecls(decls) });
+
+  return files;
+}