dnm: experiment: include all transactions in block proposal

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -878,7 +878,7 @@ export class AztecNodeService implements AztecNode, Traceable {
       const processor = publicProcessorFactory.create(fork, newGlobalVariables, skipFeeEnforcement);
 
       // REFACTOR: Consider merging ProcessReturnValues into ProcessedTx
-      const [processedTxs, failedTxs, returns] = await processor.process([tx]);
+      const [processedTxs, failedTxs, _txs, returns] = await processor.process([tx]);
       // REFACTOR: Consider returning the error rather than throwing
       if (failedTxs.length) {
         this.log.warn(`Simulated tx ${txHash} fails: ${failedTxs[0].error}`, { txHash });

yarn-project/circuit-types/src/p2p/block_proposal.test.ts

@@ -5,17 +5,21 @@ import { BlockProposal } from './block_proposal.js';
 import { makeBlockProposal } from './mocks.js';
 
 describe('Block Proposal serialization / deserialization', () => {
-  const checkEquivalence = (serialized: BlockProposal, deserialized: BlockProposal) => {
-    expect(deserialized.getSize()).toEqual(serialized.getSize());
-    expect(deserialized).toEqual(serialized);
+  const checkEquivalence = async (original: BlockProposal, deserialized: BlockProposal) => {
+    // tmp
+    // force compute tx hashes - this is not done in the deserialization
+    await Promise.all(deserialized.payload.txs.map(tx => tx.getTxHash()));
+    expect(deserialized.getSize()).toEqual(original.getSize());
+    expect(deserialized).toEqual(original);
   };
 
   it('Should serialize / deserialize', async () => {
     const proposal = await makeBlockProposal();
 
     const serialized = proposal.toBuffer();
     const deserialized = BlockProposal.fromBuffer(serialized);
-    checkEquivalence(proposal, deserialized);
+
+    await checkEquivalence(proposal, deserialized);
   });
 
   it('Should serialize / deserialize + recover sender', async () => {
@@ -25,7 +29,7 @@ describe('Block Proposal serialization / deserialization', () => {
     const serialized = proposal.toBuffer();
     const deserialized = BlockProposal.fromBuffer(serialized);
 
-    checkEquivalence(proposal, deserialized);
+    await checkEquivalence(proposal, deserialized);
 
     // Recover signature
     const sender = await deserialized.getSender();

yarn-project/circuit-types/src/p2p/block_proposal.ts

@@ -5,7 +5,7 @@ import { Signature } from '@aztec/foundation/eth-signature';
 import { type Fr } from '@aztec/foundation/fields';
 import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
 
-import { ConsensusPayload } from './consensus_payload.js';
+import { BlockProposalPayload } from './consensus_payload.js';
 import { Gossipable } from './gossipable.js';
 import {
   SignatureDomainSeparator,
@@ -33,7 +33,7 @@ export class BlockProposal extends Gossipable {
 
   constructor(
     /** The payload of the message, and what the signature is over */
-    public readonly payload: ConsensusPayload,
+    public readonly payload: BlockProposalPayload,
 
     /** The signer of the BlockProposal over the header of the new block*/
     public readonly signature: Signature,
@@ -58,7 +58,7 @@ export class BlockProposal extends Gossipable {
   }
 
   static async createProposalFromSigner(
-    payload: ConsensusPayload,
+    payload: BlockProposalPayload,
     payloadSigner: (payload: Buffer32) => Promise<Signature>,
   ) {
     const hashed = await getHashedSignaturePayload(payload, SignatureDomainSeparator.blockProposal);
@@ -93,7 +93,7 @@ export class BlockProposal extends Gossipable {
 
   static fromBuffer(buf: Buffer | BufferReader): BlockProposal {
     const reader = BufferReader.asReader(buf);
-    return new BlockProposal(reader.readObject(ConsensusPayload), reader.readObject(Signature));
+    return new BlockProposal(reader.readObject(BlockProposalPayload), reader.readObject(Signature));
   }
 
   getSize(): number {

yarn-project/circuit-types/src/p2p/consensus_payload.ts

@@ -7,9 +7,99 @@ import { type FieldsOf } from '@aztec/foundation/types';
 import { encodeAbiParameters, parseAbiParameters } from 'viem';
 import { z } from 'zod';
 
+import { Tx } from '../tx/tx.js';
 import { TxHash } from '../tx/tx_hash.js';
 import { type Signable, type SignatureDomainSeparator } from './signature_utils.js';
 
+export class BlockProposalPayload implements Signable {
+  private size: number | undefined;
+
+  constructor(
+    /** The block header the attestation is made over */
+    public readonly header: BlockHeader,
+    // TODO(https://github.com/AztecProtocol/aztec-packages/pull/7727#discussion_r1713670830): temporary
+    public readonly archive: Fr,
+    /** The sequence of transactions in the block */
+    public readonly txs: Tx[],
+  ) {}
+
+  static get schema() {
+    return z
+      .object({
+        header: BlockHeader.schema,
+        archive: Fr.schema,
+        txs: z.array(Tx.schema),
+      })
+      .transform(obj => new BlockProposalPayload(obj.header, obj.archive, obj.txs));
+  }
+
+  static getFields(fields: FieldsOf<BlockProposalPayload>) {
+    return [fields.header, fields.archive, fields.txs] as const;
+  }
+
+  async getPayloadToSign(domainSeparator: SignatureDomainSeparator): Promise<Buffer> {
+    const abi = parseAbiParameters('uint8, (bytes32, bytes32, (uint256), bytes, bytes32[])');
+
+    // Sign over the tx hashes
+    const txArray = (await Promise.all(this.txs.map(tx => tx.getTxHash()))).map(tx => tx.toString());
+    const encodedData = encodeAbiParameters(abi, [
+      domainSeparator,
+      [
+        this.archive.toString(),
+        (await this.header.hash()).toString(),
+        [0n] /* @todo See #9963 */,
+        this.header.toString(),
+        txArray,
+      ],
+    ] as const);
+
+    return hexToBuffer(encodedData);
+  }
+
+  toBuffer(): Buffer {
+    const buffer = serializeToBuffer([this.header, this.archive, this.txs.length, this.txs]);
+    this.size = buffer.length;
+    return buffer;
+  }
+
+  static fromBuffer(buf: Buffer | BufferReader): BlockProposalPayload {
+    const reader = BufferReader.asReader(buf);
+    return new BlockProposalPayload(
+      reader.readObject(BlockHeader),
+      reader.readObject(Fr),
+      reader.readArray(reader.readNumber(), Tx),
+    );
+  }
+
+  static fromFields(fields: FieldsOf<BlockProposalPayload>): BlockProposalPayload {
+    return new BlockProposalPayload(fields.header, fields.archive, fields.txs);
+  }
+
+  static empty(): BlockProposalPayload {
+    return new BlockProposalPayload(BlockHeader.empty(), Fr.ZERO, []);
+  }
+
+  /**
+   * Get the size of the block proposal payload in bytes.
+   * @returns The size of the block proposal payload.
+   */
+  getSize(): number {
+    // We cache size to avoid recalculating it
+    if (this.size) {
+      return this.size;
+    }
+    this.size = this.toBuffer().length;
+    return this.size;
+  }
+
+  // Convert from the block proposal payload to the consensus payload
+  // This version does not include the entire transaction bodies
+  async toConsensusPayload(): Promise<ConsensusPayload> {
+    const txHashes = await Promise.all(this.txs.map(tx => tx.getTxHash()));
+    return new ConsensusPayload(this.header, this.archive, txHashes);
+  }
+}
+
 export class ConsensusPayload implements Signable {
   private size: number | undefined;
 

yarn-project/circuit-types/src/p2p/mocks.ts

@@ -3,10 +3,11 @@ import { makeHeader } from '@aztec/circuits.js/testing';
 import { Secp256k1Signer } from '@aztec/foundation/crypto';
 import { Fr } from '@aztec/foundation/fields';
 
+import { Tx } from '../tx/tx.js';
 import { TxHash } from '../tx/tx_hash.js';
 import { BlockAttestation } from './block_attestation.js';
 import { BlockProposal } from './block_proposal.js';
-import { ConsensusPayload } from './consensus_payload.js';
+import { BlockProposalPayload, ConsensusPayload } from './consensus_payload.js';
 import { SignatureDomainSeparator, getHashedSignaturePayloadEthSignedMessage } from './signature_utils.js';
 
 export interface MakeConsensusPayloadOptions {
@@ -39,8 +40,27 @@ const makeAndSignConsensusPayload = async (
   return { payload, signature };
 };
 
+const makeAndSignBlockProposalPayload = async (
+  domainSeparator: SignatureDomainSeparator,
+  options?: MakeConsensusPayloadOptions,
+) => {
+  const txs = await Promise.all(Array.from({ length: 6 }, () => Tx.random()));
+  const { signer = Secp256k1Signer.random(), header = makeHeader(1), archive = Fr.random() } = options ?? {};
+
+  const payload = BlockProposalPayload.fromFields({
+    header,
+    archive,
+    txs,
+  });
+
+  const hash = await getHashedSignaturePayloadEthSignedMessage(payload, domainSeparator);
+  const signature = signer.sign(hash);
+
+  return { payload, signature };
+};
+
 export const makeBlockProposal = async (options?: MakeConsensusPayloadOptions): Promise<BlockProposal> => {
-  const { payload, signature } = await makeAndSignConsensusPayload(SignatureDomainSeparator.blockProposal, options);
+  const { payload, signature } = await makeAndSignBlockProposalPayload(SignatureDomainSeparator.blockProposal, options);
   return new BlockProposal(payload, signature);
 };
 

yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts

@@ -41,6 +41,12 @@ describe('e2e_p2p_network', () => {
       numberOfNodes: NUM_NODES,
       basePort: BOOT_NODE_UDP_PORT,
       metricsPort: shouldCollectMetrics(),
+      // TODO(md): put in central location - part of different pr
+      // initialConfig: {
+      //   aztecEpochDuration: 8,
+      //   aztecSlotDuration: 18,
+      //   ethereumSlotDuration: 6,
+      // },
     });
 
     await t.setupAccount();

yarn-project/end-to-end/src/e2e_p2p/reex.test.ts

@@ -121,10 +121,11 @@ describe('e2e_p2p_reex', () => {
       jest.spyOn((node as any).p2pClient, 'broadcastProposal').mockImplementation(async (...args: unknown[]) => {
         // We remove one of the transactions, therefore the block root will be different!
         const proposal = args[0] as BlockProposal;
-        const { txHashes } = proposal.payload;
+        // TODO: reex
+        const { txs } = proposal.payload;
 
         // We need to mutate the proposal, so we cast to any
-        (proposal.payload as any).txHashes = txHashes.slice(0, txHashes.length - 1);
+        (proposal.payload as any).txs = txs.slice(0, txs.length - 1);
 
         // We sign over the proposal using the node's signing key
         // Abusing javascript to access the nodes signing key

yarn-project/foundation/package.json

@@ -132,6 +132,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
+    "@libp2p/interface": "1.3.1",
     "@types/bn.js": "^5.1.3",
     "@types/debug": "^4.1.7",
     "@types/detect-node": "^2.0.0",

yarn-project/foundation/src/log/index.ts

@@ -3,3 +3,4 @@ export * from './debug.js';
 export * from './pino-logger.js';
 export * from './log_history.js';
 export * from './log_fn.js';
+export * from './libp2p_logger.js';

yarn-project/foundation/src/log/libp2p_logger.ts

@@ -0,0 +1,48 @@
+import { type ComponentLogger, type Logger } from '@libp2p/interface';
+
+import { getLogLevelFromFilters } from './log-filters.js';
+import { logFilters, logger } from './pino-logger.js';
+
+/**
+ * Creates a libp2p compatible logger that wraps our pino logger.
+ * This adapter implements the ComponentLogger interface required by libp2p.
+ */
+export function createLibp2pComponentLogger(namespace: string): ComponentLogger {
+  return {
+    forComponent: (component: string) => createLibp2pLogger(`${namespace}:${component}`),
+  };
+}
+
+function createLibp2pLogger(component: string): Logger {
+  // Create a direct pino logger instance for libp2p that supports string interpolation
+  const log = logger.child({ module: component }, { level: getLogLevelFromFilters(logFilters, component) });
+
+  // Default log level is trace as this is super super noisy
+  const logFn = (message: string, ...args: unknown[]) => {
+    log.trace(message, ...args);
+  };
+
+  return Object.assign(logFn, {
+    enabled: log.isLevelEnabled('debug'),
+    error(message: string, ...args: unknown[]) {
+      // We write error outputs as debug as they are often expected, e.g. connection errors can happen in happy paths
+      log.debug(message, ...args);
+    },
+
+    debug(message: string, ...args: unknown[]) {
+      log.debug(message, ...args);
+    },
+
+    info(message: string, ...args: unknown[]) {
+      log.info(message, ...args);
+    },
+
+    warn(message: string, ...args: unknown[]) {
+      log.warn(message, ...args);
+    },
+
+    trace(message: string, ...args: unknown[]) {
+      log.trace(message, ...args);
+    },
+  });
+}

yarn-project/foundation/src/log/pino-logger.ts

@@ -11,23 +11,14 @@ import { getLogLevelFromFilters, parseEnv } from './log-filters.js';
 import { type LogLevel } from './log-levels.js';
 import { type LogData, type LogFn } from './log_fn.js';
 
-export function createLogger(module: string, fixedTerms = {}): Logger {
+export function createLogger(module: string): Logger {
   module = logNameHandlers.reduce((moduleName, handler) => handler(moduleName), module.replace(/^aztec:/, ''));
   const pinoLogger = logger.child({ module }, { level: getLogLevelFromFilters(logFilters, module) });
 
-  // Only perform copy of data if fixed terms are provided
-  const hasFixedTerms = Object.keys(fixedTerms).length > 0;
-
   // We check manually for isLevelEnabled to avoid calling processLogData unnecessarily.
   // Note that isLevelEnabled is missing from the browser version of pino.
   const logFn = (level: LogLevel, msg: string, data?: unknown) =>
-    isLevelEnabled(pinoLogger, level) &&
-    pinoLogger[level](
-      hasFixedTerms
-        ? processLogData({ ...fixedTerms, ...(data ?? {}) } as LogData)
-        : processLogData((data as LogData) ?? {}),
-      msg,
-    );
+    isLevelEnabled(pinoLogger, level) && pinoLogger[level](processLogData((data as LogData) ?? {}), msg);
 
   return {
     silent: () => {},
@@ -92,7 +83,7 @@ function isLevelEnabled(logger: pino.Logger<'verbose', boolean>, level: LogLevel
 
 // Load log levels from environment variables.
 const defaultLogLevel = process.env.NODE_ENV === 'test' ? 'silent' : 'info';
-const [logLevel, logFilters] = parseEnv(process.env.LOG_LEVEL, defaultLogLevel);
+export const [logLevel, logFilters] = parseEnv(process.env.LOG_LEVEL, defaultLogLevel);
 
 // Define custom logging levels for pino.
 const customLevels = { verbose: 25 };
@@ -178,7 +169,7 @@ function makeLogger() {
   }
 }
 
-const logger = makeLogger();
+export const logger = makeLogger();
 
 // Log the logger configuration.
 logger.verbose(