feat(blob-lib): make blob lib and fix encoding test flake

yarn-project/archiver/package.json

@@ -64,6 +64,7 @@
     ]
   },
   "dependencies": {
+    "@aztec/blob-lib": "workspace:^",
     "@aztec/blob-sink": "workspace:^",
     "@aztec/circuit-types": "workspace:^",
     "@aztec/circuits.js": "workspace:^",

yarn-project/archiver/src/archiver/archiver.test.ts

@@ -1,8 +1,8 @@
+import { Blob } from '@aztec/blob-lib';
 import { type BlobSinkClientInterface } from '@aztec/blob-sink/client';
 import { InboxLeaf, type L1RollupConstants, L2Block } from '@aztec/circuit-types';
 import { GENESIS_ARCHIVE_ROOT, PrivateLog } from '@aztec/circuits.js';
 import { DefaultL1ContractsConfig } from '@aztec/ethereum';
-import { Blob } from '@aztec/foundation/blob';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { Fr } from '@aztec/foundation/fields';
 import { type Logger, createLogger } from '@aztec/foundation/log';

yarn-project/archiver/src/archiver/data_retrieval.ts

@@ -1,8 +1,8 @@
+import { Blob, BlobDeserializationError } from '@aztec/blob-lib';
 import { type BlobSinkClientInterface } from '@aztec/blob-sink/client';
 import { Body, InboxLeaf, L2Block } from '@aztec/circuit-types';
 import { AppendOnlyTreeSnapshot, BlockHeader, Fr, Proof } from '@aztec/circuits.js';
 import { asyncPool } from '@aztec/foundation/async-pool';
-import { Blob, BlobDeserializationError } from '@aztec/foundation/blob';
 import { type EthAddress } from '@aztec/foundation/eth-address';
 import { type ViemSignature } from '@aztec/foundation/eth-signature';
 import { type Logger, createLogger } from '@aztec/foundation/log';

yarn-project/archiver/tsconfig.json

@@ -6,6 +6,9 @@
     "tsBuildInfoFile": ".tsbuildinfo"
   },
   "references": [
+    {
+      "path": "../blob-lib"
+    },
     {
       "path": "../blob-sink"
     },

yarn-project/blob-lib/package.json

@@ -0,0 +1,83 @@
+{
+  "name": "@aztec/blob-lib",
+  "version": "0.1.0",
+  "type": "module",
+  "exports": {
+    ".": "./dest/index.js"
+  },
+  "typedocOptions": {
+    "entryPoints": [
+      "./src/index.ts"
+    ],
+    "name": "Blob Lib",
+    "tsconfig": "./tsconfig.json"
+  },
+  "scripts": {
+    "build": "yarn clean && tsc -b",
+    "build:dev": "tsc -b --watch",
+    "clean": "rm -rf ./dest .tsbuildinfo",
+    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+    "start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'",
+    "start": "node ./dest/index.js",
+    "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
+  },
+  "inherits": [
+    "../package.common.json"
+  ],
+  "dependencies": {
+    "@aztec/foundation": "workspace:^",
+    "tslib": "^2.4.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.5.0",
+    "@types/jest": "^29.5.0",
+    "@types/node": "^18.14.6",
+    "get-port": "^7.1.0",
+    "jest": "^29.5.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "files": [
+    "dest",
+    "src",
+    "!*.test.*"
+  ],
+  "types": "./dest/index.d.ts",
+  "jest": {
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    },
+    "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+    "rootDir": "./src",
+    "transform": {
+      "^.+\\.tsx?$": [
+        "@swc/jest",
+        {
+          "jsc": {
+            "parser": {
+              "syntax": "typescript",
+              "decorators": true
+            },
+            "transform": {
+              "decoratorVersion": "2022-03"
+            }
+          }
+        }
+      ]
+    },
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "reporters": [
+      "default"
+    ],
+    "testTimeout": 30000,
+    "setupFiles": [
+      "../../foundation/src/jest/setup.mjs"
+    ]
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

yarn-project/foundation/src/blob/blob.test.ts → yarn-project/blob-lib/src/blob.test.ts

@@ -1,8 +1,9 @@
+import { poseidon2Hash } from '@aztec/foundation/crypto';
+import { Fr } from '@aztec/foundation/fields';
+
 import cKzg from 'c-kzg';
 import type { Blob as BlobBuffer, Bytes48, KZGProof } from 'c-kzg';
 
-import { poseidon2Hash } from '../crypto/index.js';
-import { Fr } from '../fields/index.js';
 import { Blob, makeEncodedBlob } from './index.js';
 
 // Importing directly from 'c-kzg' does not work, ignoring import/no-named-as-default-member err:

yarn-project/foundation/src/blob/blob.ts → yarn-project/blob-lib/src/blob.ts

@@ -1,10 +1,11 @@
 // Importing directly from 'c-kzg' does not work, ignoring import/no-named-as-default-member err:
+import { poseidon2Hash, sha256 } from '@aztec/foundation/crypto';
+import { Fr } from '@aztec/foundation/fields';
+import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
+
 import cKzg from 'c-kzg';
 import type { Blob as BlobBuffer } from 'c-kzg';
 
-import { poseidon2Hash, sha256 } from '../crypto/index.js';
-import { Fr } from '../fields/index.js';
-import { BufferReader, serializeToBuffer } from '../serialize/index.js';
 import { deserializeEncodedBlobToFields, extractBlobFieldsFromBuffer } from './encoding.js';
 import { BlobDeserializationError } from './errors.js';
 import { type BlobJson } from './interface.js';
@@ -50,6 +51,10 @@ export class Blob {
   static fromEncodedBlobBuffer(blob: BlobBuffer, multiBlobFieldsHash?: Fr): Promise<Blob> {
     try {
       const fields: Fr[] = deserializeEncodedBlobToFields(blob);
+      console.log(
+        `from encoded blob buffer Fields`,
+        fields.slice(0, 10).map(f => f.toString()),
+      );
       return Blob.fromFields(fields, multiBlobFieldsHash);
     } catch (err) {
       throw new BlobDeserializationError(

yarn-project/foundation/src/blob/encoding.ts → yarn-project/blob-lib/src/encoding.ts

@@ -3,12 +3,14 @@ import { BufferReader, FieldReader } from '@aztec/foundation/serialize';
 
 import type { Blob as BlobBuffer } from 'c-kzg';
 
+// Note duplicated from circuit-types !
 // This will appear as 0x74785f7374617274 in logs
 export const TX_START_PREFIX = 8392562855083340404n;
 // These are helper constants to decode tx effects from blob encoded fields
 export const TX_START_PREFIX_BYTES_LENGTH = TX_START_PREFIX.toString(16).length / 2;
 // 7 bytes for: | 0 | txlen[0] | txlen[1] | 0 | REVERT_CODE_PREFIX | 0 | revertCode |
 export const TX_EFFECT_PREFIX_BYTE_LENGTH = TX_START_PREFIX_BYTES_LENGTH + 7;
+export const REVERT_CODE_PREFIX = 1;
 
 /**
  * Deserializes a blob buffer into an array of field elements.
@@ -66,11 +68,50 @@ export function deserializeEncodedBlobToFields(blob: BlobBuffer): Fr[] {
   return array.slice(0, fieldReader.cursor);
 }
 
+/**
+ * Get the length of the transaction from the first field.
+ *
+ * @param firstField - The first field of the transaction.
+ * @returns The length of the transaction.
+ *
+ * @throws If the first field does not include the correct prefix - encoding invalid.
+ */
 export function getLengthFromFirstField(firstField: Fr): number {
+  // Check that the first field includes the correct prefix
+  if (!isValidFirstField(firstField)) {
+    throw new Error('Invalid prefix');
+  }
   const buf = firstField.toBuffer().subarray(-TX_EFFECT_PREFIX_BYTE_LENGTH);
   return new Fr(buf.subarray(TX_START_PREFIX_BYTES_LENGTH + 1, TX_START_PREFIX_BYTES_LENGTH + 3)).toNumber();
 }
 
+// NOTE: duplicated from circuit-types tx effect!
+/**
+ * Determines whether a field is the first field of a tx effect
+ */
+export function isValidFirstField(field: Fr): boolean {
+  const buf = field.toBuffer();
+  if (
+    !buf
+      .subarray(0, field.size - TX_EFFECT_PREFIX_BYTE_LENGTH)
+      .equals(Buffer.alloc(field.size - TX_EFFECT_PREFIX_BYTE_LENGTH))
+  ) {
+    return false;
+  }
+  const sliced = buf.subarray(-TX_EFFECT_PREFIX_BYTE_LENGTH);
+  if (
+    // Checking we start with the correct prefix...
+    !new Fr(sliced.subarray(0, TX_START_PREFIX_BYTES_LENGTH)).equals(new Fr(TX_START_PREFIX)) ||
+    // ...and include the revert code prefix..
+    sliced[sliced.length - 3] !== REVERT_CODE_PREFIX ||
+    // ...and the following revert code is valid.
+    sliced[sliced.length - 1] > 4
+  ) {
+    return false;
+  }
+  return true;
+}
+
 /**
  * Extract the fields from a blob buffer, but do not take into account encoding
  * that will include trailing zeros.

yarn-project/blob-lib/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "..",
+  "compilerOptions": {
+    "outDir": "dest",
+    "rootDir": "src",
+    "tsBuildInfoFile": ".tsbuildinfo"
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../foundation"
+    }
+  ]
+}

yarn-project/blob-sink/package.json

@@ -52,6 +52,7 @@
     ]
   },
   "dependencies": {
+    "@aztec/blob-lib": "workspace:^",
     "@aztec/circuit-types": "workspace:^",
     "@aztec/foundation": "workspace:^",
     "@aztec/kv-store": "workspace:*",

yarn-project/blob-sink/src/blobstore/blob_store_test_suite.ts

@@ -1,4 +1,4 @@
-import { Blob } from '@aztec/foundation/blob';
+import { Blob } from '@aztec/blob-lib';
 import { Fr } from '@aztec/foundation/fields';
 
 import { BlobWithIndex } from '../types/index.js';

yarn-project/blob-sink/src/client/blob-sink-client-tests.ts

@@ -1,4 +1,4 @@
-import { makeEncodedBlob } from '@aztec/foundation/blob';
+import { makeEncodedBlob } from '@aztec/blob-lib';
 
 import { type BlobSinkClientInterface } from './interface.js';
 

yarn-project/blob-sink/src/client/http.test.ts

@@ -1,4 +1,4 @@
-import { Blob, makeEncodedBlob, makeUnencodedBlob } from '@aztec/foundation/blob';
+import { Blob, makeEncodedBlob, makeUnencodedBlob } from '@aztec/blob-lib';
 import { Fr } from '@aztec/foundation/fields';
 
 import { jest } from '@jest/globals';

yarn-project/blob-sink/src/client/http.ts

@@ -1,4 +1,4 @@
-import { Blob, BlobDeserializationError, type BlobJson } from '@aztec/foundation/blob';
+import { Blob, BlobDeserializationError, type BlobJson } from '@aztec/blob-lib';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { makeBackoff, retry } from '@aztec/foundation/retry';
 

yarn-project/blob-sink/src/client/interface.ts

@@ -1,4 +1,4 @@
-import { type Blob } from '@aztec/foundation/blob';
+import { type Blob } from '@aztec/blob-lib';
 
 export interface BlobSinkClientInterface {
   sendBlobsToBlobSink(blockId: string, blobs: Blob[]): Promise<boolean>;

yarn-project/blob-sink/src/client/local.ts

@@ -1,4 +1,4 @@
-import { type Blob } from '@aztec/foundation/blob';
+import { type Blob } from '@aztec/blob-lib';
 
 import { type BlobStore } from '../blobstore/index.js';
 import { BlobWithIndex } from '../types/blob_with_index.js';

yarn-project/blob-sink/src/server/server.test.ts

@@ -1,4 +1,4 @@
-import { Blob, makeEncodedBlob } from '@aztec/foundation/blob';
+import { Blob, makeEncodedBlob } from '@aztec/blob-lib';
 
 import request from 'supertest';
 

yarn-project/blob-sink/src/server/server.ts

@@ -1,4 +1,4 @@
-import { Blob } from '@aztec/foundation/blob';
+import { Blob } from '@aztec/blob-lib';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { type AztecAsyncKVStore } from '@aztec/kv-store';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';

yarn-project/blob-sink/src/types/blob_with_index.test.ts

@@ -1,4 +1,4 @@
-import { Blob } from '@aztec/foundation/blob';
+import { Blob } from '@aztec/blob-lib';
 import { Fr } from '@aztec/foundation/fields';
 
 import { BlobWithIndex, BlobsWithIndexes } from './blob_with_index.js';

yarn-project/blob-sink/src/types/blob_with_index.ts

@@ -1,4 +1,4 @@
-import { Blob, type BlobJson } from '@aztec/foundation/blob';
+import { Blob, type BlobJson } from '@aztec/blob-lib';
 import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
 
 /** Serialized an array of blobs with their indexes to be stored at a given block id */

yarn-project/blob-sink/tsconfig.json

@@ -6,6 +6,9 @@
     "tsBuildInfoFile": ".tsbuildinfo"
   },
   "references": [
+    {
+      "path": "../blob-lib"
+    },
     {
       "path": "../circuit-types"
     },

yarn-project/circuit-types/package.json

@@ -4,6 +4,7 @@
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
+    "./blob": "./dest/blob/index.js",
     "./stats": "./dest/stats/index.js",
     "./jest": "./dest/jest/index.js",
     "./interfaces": "./dest/interfaces/index.js",

yarn-project/circuits.js/src/structs/blobs/blob_public_inputs.test.ts

@@ -1,4 +1,4 @@
-import { Blob } from '@aztec/foundation/blob';
+import { Blob } from '@aztec/blob-lib';
 import { timesParallel } from '@aztec/foundation/collection';
 import { randomInt } from '@aztec/foundation/crypto';
 

yarn-project/circuits.js/src/structs/blobs/blob_public_inputs.ts

@@ -1,6 +1,6 @@
+import { type Blob } from '@aztec/blob-lib';
 import { makeTuple } from '@aztec/foundation/array';
 import { toBigIntBE, toBufferBE, toHex } from '@aztec/foundation/bigint-buffer';
-import { type Blob } from '@aztec/foundation/blob';
 import { sha256, sha256Trunc } from '@aztec/foundation/crypto';
 import { Fr } from '@aztec/foundation/fields';
 import { BufferReader, FieldReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize';

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -1,6 +1,7 @@
 import { type ArchiveSource } from '@aztec/archiver';
 import { getConfigEnvVars } from '@aztec/aztec-node';
 import { AztecAddress, Fr, GlobalVariables, type L2Block, createLogger } from '@aztec/aztec.js';
+import { Blob } from '@aztec/blob-lib';
 // eslint-disable-next-line no-restricted-imports
 import { type L2Tips, type ProcessedTx } from '@aztec/circuit-types';
 import { makeBloatedProcessedTx } from '@aztec/circuit-types/test';
@@ -27,7 +28,6 @@ import {
 } from '@aztec/ethereum';
 import { EthCheatCodesWithState } from '@aztec/ethereum/test';
 import { range } from '@aztec/foundation/array';
-import { Blob } from '@aztec/foundation/blob';
 import { timesParallel } from '@aztec/foundation/collection';
 import { sha256, sha256ToField } from '@aztec/foundation/crypto';
 import { TestDateProvider } from '@aztec/foundation/timer';

yarn-project/end-to-end/src/web/main.js.LICENSE.txt

@@ -0,0 +1,25 @@
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <feross@feross.org> <http://feross.org>
+ * @license  MIT
+ */
+
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */
+
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! noble-curves - MIT License (c) 2022 Paul Miller (paulmillr.com) */
+
+/*! noble-hashes - MIT License (c) 2022 Paul Miller (paulmillr.com) */
+
+/**
+ * @license
+ * Copyright 2019 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */

yarn-project/ethereum/package.json

@@ -32,6 +32,7 @@
     "../package.common.json"
   ],
   "dependencies": {
+    "@aztec/blob-lib": "workspace:^",
     "@aztec/foundation": "workspace:^",
     "@aztec/l1-artifacts": "workspace:^",
     "@viem/anvil": "^0.0.10",

yarn-project/ethereum/src/l1_tx_utils.test.ts

@@ -1,4 +1,4 @@
-import { Blob } from '@aztec/foundation/blob';
+import { Blob } from '@aztec/blob-lib';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import { sleep } from '@aztec/foundation/sleep';