feat: implement sync car decode (#253)

Gozala · web-flow · commit 40acaac52870 · 2023-03-07T14:52:05.000-08:00
* feat: implement sync car decode

* fix: reverse root encoding

* revert to block order but retain root order
diff --git a/package.json b/package.json
@@ -22,8 +22,8 @@
   },
   "devDependencies": {
     "mocha": "^10.1.0",
-    "prettier": "2.7.1",
-    "typescript": "^4.9.4"
+    "prettier": "2.8.4",
+    "typescript": "^4.9.5"
   },
   "prettier": {
     "trailingComma": "es5",
diff --git a/packages/client/package.json b/packages/client/package.json
@@ -34,18 +34,18 @@
   },
   "devDependencies": {
     "@types/chai": "^4.3.3",
-    "@types/mocha": "^9.1.0",
+    "@types/mocha": "^10.0.1",
     "@ucanto/core": "workspace:^",
     "@ucanto/principal": "workspace:^",
     "@ucanto/transport": "workspace:^",
     "@web-std/fetch": "^4.1.0",
     "@web-std/file": "^3.0.2",
-    "c8": "^7.11.0",
+    "c8": "^7.13.0",
     "chai": "^4.3.6",
     "mocha": "^10.1.0",
     "nyc": "^15.1.0",
-    "playwright-test": "^8.1.1",
-    "typescript": "^4.9.4"
+    "playwright-test": "^8.2.0",
+    "typescript": "^4.9.5"
   },
   "type": "module",
   "main": "src/lib.js",
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -29,22 +29,22 @@
     "build": "tsc --build"
   },
   "dependencies": {
-    "@ipld/car": "^5.0.3",
+    "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "@ipld/dag-ucan": "^3.2.0",
     "@ucanto/interface": "workspace:^",
     "multiformats": "^11.0.0"
   },
   "devDependencies": {
     "@types/chai": "^4.3.3",
-    "@types/mocha": "^9.1.0",
+    "@types/mocha": "^10.0.1",
     "@ucanto/principal": "workspace:^",
-    "c8": "^7.11.0",
+    "c8": "^7.13.0",
     "chai": "^4.3.6",
     "mocha": "^10.1.0",
     "nyc": "^15.1.0",
-    "playwright-test": "^8.1.1",
-    "typescript": "^4.9.4"
+    "playwright-test": "^8.2.0",
+    "typescript": "^4.9.5"
   },
   "type": "module",
   "main": "src/lib.js",
diff --git a/packages/interface/package.json b/packages/interface/package.json
@@ -27,7 +27,7 @@
     "multiformats": "^11.0.0"
   },
   "devDependencies": {
-    "typescript": "^4.9.4"
+    "typescript": "^4.9.5"
   },
   "exports": {
     ".": {
diff --git a/packages/principal/package.json b/packages/principal/package.json
@@ -28,20 +28,20 @@
   },
   "dependencies": {
     "@ipld/dag-ucan": "^3.2.0",
-    "@noble/ed25519": "^1.7.1",
+    "@noble/ed25519": "^1.7.3",
     "@ucanto/interface": "workspace:^",
     "multiformats": "^11.0.0",
     "one-webcrypto": "^1.0.3"
   },
   "devDependencies": {
     "@types/chai": "^4.3.3",
-    "@types/mocha": "^9.1.0",
-    "c8": "^7.11.0",
+    "@types/mocha": "^10.0.1",
+    "c8": "^7.13.0",
     "chai": "^4.3.6",
     "mocha": "^10.1.0",
     "nyc": "^15.1.0",
-    "playwright-test": "^8.1.1",
-    "typescript": "^4.9.4"
+    "playwright-test": "^8.2.0",
+    "typescript": "^4.9.5"
   },
   "type": "module",
   "main": "src/lib.js",
diff --git a/packages/server/package.json b/packages/server/package.json
@@ -35,20 +35,20 @@
   "devDependencies": {
     "@types/chai": "^4.3.3",
     "@types/chai-subset": "^1.3.3",
-    "@types/mocha": "^9.1.0",
+    "@types/mocha": "^10.0.1",
     "@ucanto/client": "workspace:^",
     "@ucanto/principal": "workspace:^",
     "@ucanto/transport": "workspace:^",
     "@web-std/fetch": "^4.1.0",
     "@web-std/file": "^3.0.2",
-    "c8": "^7.11.0",
+    "c8": "^7.13.0",
     "chai": "^4.3.6",
     "chai-subset": "^1.6.0",
     "mocha": "^10.1.0",
     "multiformats": "^11.0.0",
     "nyc": "^15.1.0",
-    "playwright-test": "^8.1.1",
-    "typescript": "^4.9.4"
+    "playwright-test": "^8.2.0",
+    "typescript": "^4.9.5"
   },
   "exports": {
     ".": {
diff --git a/packages/transport/package.json b/packages/transport/package.json
@@ -29,23 +29,23 @@
     "build": "tsc --build"
   },
   "dependencies": {
-    "@ipld/car": "^5.0.3",
+    "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "@ucanto/core": "workspace:^",
     "@ucanto/interface": "workspace:^",
     "multiformats": "^11.0.0"
   },
   "devDependencies": {
     "@types/chai": "^4.3.3",
-    "@types/mocha": "^9.1.0",
+    "@types/mocha": "^10.0.1",
     "@ucanto/principal": "workspace:^",
     "@web-std/fetch": "^4.1.0",
-    "c8": "^7.11.0",
+    "c8": "^7.13.0",
     "chai": "^4.3.6",
     "mocha": "^10.1.0",
     "nyc": "^15.1.0",
-    "playwright-test": "^8.1.1",
-    "typescript": "^4.9.4"
+    "playwright-test": "^8.2.0",
+    "typescript": "^4.9.5"
   },
   "type": "module",
   "typesVersions": {
diff --git a/packages/transport/src/car.js b/packages/transport/src/car.js
@@ -50,7 +50,7 @@ export const decode = async ({ headers, body }) => {
     )
   }
 
-  const { roots, blocks } = await CAR.decode(body)
+  const { roots, blocks } = CAR.decode(body)
 
   const invocations = []
 
diff --git a/packages/transport/src/car/codec.js b/packages/transport/src/car/codec.js
@@ -1,6 +1,5 @@
 import * as API from '@ucanto/interface'
-import * as CARWriter from '@ipld/car/buffer-writer'
-import { CarReader } from '@ipld/car/reader'
+import { CarBufferReader, CarBufferWriter } from '@ipld/car'
 import { base32 } from 'multiformats/bases/base32'
 import { UCAN, createLink } from '@ucanto/core'
 import { sha256 } from 'multiformats/hashes/sha2'
@@ -33,8 +32,8 @@ class Writer {
       const id = block.cid.toString(base32)
       if (!this.written.has(id)) {
         this.blocks.push(block)
-        this.byteLength += CARWriter.blockLength(
-          /** @type {CARWriter.Block} */ (block)
+        this.byteLength += CarBufferWriter.blockLength(
+          /** @type {CarBufferWriter.Block} */ (block)
         )
         this.written.add(id)
       }
@@ -46,25 +45,28 @@ class Writer {
    */
   flush(...rootBlocks) {
     const roots = []
+    // We reverse the roots so that the first root is the last block in the CAR
     for (const block of rootBlocks.reverse()) {
       const id = block.cid.toString(base32)
       if (!this.written.has(id)) {
         this.blocks.unshift(block)
-        this.byteLength += CARWriter.blockLength({
-          cid: /** @type {CARWriter.CID} */ (block.cid),
+        this.byteLength += CarBufferWriter.blockLength({
+          cid: /** @type {CarBufferWriter.CID} */ (block.cid),
           bytes: block.bytes,
         })
         this.written.add(id)
       }
-      roots.push(/** @type {CARWriter.CID} */ (block.cid))
+
+      // We unshift here because we want to preserve the order of the roots
+      roots.unshift(/** @type {CarBufferWriter.CID} */ (block.cid))
     }
 
-    this.byteLength += CARWriter.headerLength({ roots })
+    this.byteLength += CarBufferWriter.headerLength({ roots })
 
     const buffer = new ArrayBuffer(this.byteLength)
-    const writer = CARWriter.createWriter(buffer, { roots })
+    const writer = CarBufferWriter.createWriter(buffer, { roots })
 
-    for (const block of /** @type {CARWriter.Block[]} */ (this.blocks)) {
+    for (const block of /** @type {CarBufferWriter.Block[]} */ (this.blocks)) {
       writer.write(block)
     }
 
@@ -89,20 +91,22 @@ export const encode = ({ roots = [], blocks }) => {
 
 /**
  * @param {API.ByteView<Partial<Model>>} bytes
- * @returns {Promise<Model>}
+ * @returns {Model}
  */
-export const decode = async bytes => {
-  const reader = await /** @type {any} */ (CarReader.fromBytes(bytes))
-  /** @type {{_header: { roots: CARWriter.CID[] }, _keys: string[], _blocks: UCAN.Block[] }} */
-  const { _header, _blocks, _keys } = reader
+export const decode = bytes => {
+  const reader = CarBufferReader.fromBytes(bytes)
   const roots = []
   const blocks = new Map()
-  const index = _header.roots.map(cid => _keys.indexOf(String(cid)))
 
-  for (const [n, block] of _blocks.entries()) {
-    if (index.includes(n)) {
-      roots.push(/** @type {Block} */ (block))
-    } else {
+  for (const root of reader.getRoots()) {
+    const block = reader.get(root)
+    if (block) {
+      roots.push(block)
+    }
+  }
+
+  for (const block of reader.blocks()) {
+    if (!roots.includes(block)) {
       blocks.set(block.cid.toString(), block)
     }
   }
diff --git a/packages/validator/package.json b/packages/validator/package.json
@@ -27,25 +27,25 @@
     "build": "tsc --build"
   },
   "dependencies": {
-    "@ipld/car": "^5.0.3",
-    "@ipld/dag-cbor": "^8.0.0",
+    "@ipld/car": "^5.1.0",
+    "@ipld/dag-cbor": "^9.0.0",
     "@ucanto/core": "workspace:^",
     "@ucanto/interface": "workspace:^",
     "multiformats": "^11.0.0"
   },
   "devDependencies": {
     "@types/chai": "^4.3.3",
     "@types/chai-subset": "^1.3.3",
-    "@types/mocha": "^9.1.0",
+    "@types/mocha": "^10.0.1",
     "@ucanto/client": "workspace:^",
     "@ucanto/principal": "workspace:^",
-    "c8": "^7.11.0",
+    "c8": "^7.13.0",
     "chai": "^4.3.6",
     "chai-subset": "^1.6.0",
     "mocha": "^10.1.0",
     "nyc": "^15.1.0",
-    "playwright-test": "^8.1.1",
-    "typescript": "^4.9.4"
+    "playwright-test": "^8.2.0",
+    "typescript": "^4.9.5"
   },
   "type": "module",
   "main": "src/lib.js",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ export const decode = async ({ headers, body }) => {`
`50`	`50`	`)`
`51`	`51`	`}`
`52`	`52`
`53`		`- const { roots, blocks } = await CAR.decode(body)`
	`53`	`+ const { roots, blocks } = CAR.decode(body)`
`54`	`54`
`55`	`55`	`const invocations = []`
`56`	`56`