test: prove one epoch in kind

.github/workflows/nightly-kind-test.yml

@@ -53,7 +53,9 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        test: ["transfer", "reorg"]
+        test:
+          - transfer
+          - reorg
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -62,7 +64,7 @@ jobs:
         uses: ./.github/ensure-tester-with-images
         timeout-minutes: 90
         with:
-          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
+          runner_type: 16core-tester-x86
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
@@ -73,13 +75,42 @@ jobs:
           run: |
             set -eux
             ./spartan/scripts/setup_local_k8s.sh
+            cd yarn-project/end-to-end
             export FORCE_COLOR=1
-            NAMESPACE=${{ matrix.test }} FRESH_INSTALL=true VALUES_FILE="16-validators.yaml" ./scripts/network_test.sh ./src/spartan/${{ matrix.test }}.test.ts || true
+            NAMESPACE=${{ matrix.test }} FRESH_INSTALL=true CLEANUP_CLUSTER=true VALUES_FILE="16-validators.yaml" ./scripts/network_test.sh ./src/spartan/${{ matrix.test }}.test.ts || true
+
+  proving-test:
+    needs: build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Setup and Test
+        uses: ./.github/ensure-tester-with-images
+        timeout-minutes: 90
+        with:
+          runner_type: 128core-tester-x86
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: 90
+          run: |
+            set -eux
+            ./spartan/scripts/setup_local_k8s.sh
+            cd yarn-project/end-to-end
+            export FORCE_COLOR=1
+            NAMESPACE=proving FRESH_INSTALL=true CLEANUP_CLUSTER=true INSTALL_TIMEOUT=45m VALUES_FILE="1-validator-with-proving.yaml" ./scripts/network_test.sh ./src/spartan/proving.test.ts || true
 
   success-check:
     runs-on: ubuntu-20.04
     needs:
       - test
+      - proving-test
     if: always()
     steps:
       - name: Report overall success

spartan/aztec-network/templates/deploy-l1-verifier.yaml

@@ -33,7 +33,7 @@ spec:
               export ROLLUP_CONTRACT_ADDRESS=$(curl -X POST -H 'Content-Type: application/json' \
                   -d '{"jsonrpc":"2.0","method":"node_getL1ContractAddresses","params":[],"id":1}' \
                   "$AZTEC_NODE_URL" \
-                  | jq -r '.result.rollupAddress.data')
+                  | jq -r '.result.rollupAddress.value')
 
               echo "Rollup contract address: $ROLLUP_CONTRACT_ADDRESS"
               node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real

spartan/aztec-network/values/1-validator-with-proving-and-metrics.yaml

@@ -0,0 +1,48 @@
+validator:
+  replicas: 1
+  validatorKeys:
+    - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+  validatorAddresses:
+    - 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
+  validator:
+    disabled: false
+
+bootNode:
+  validator:
+    disabled: true
+
+proverNode:
+  realProofs: true
+
+proverAgent:
+  replicas: 6
+  realProofs: true
+  bb:
+    hardwareConcurrency: 16
+  resources:
+    requests:
+      memory: "64Gi"
+      cpu: "16"
+    limits:
+      memory: "96Gi"
+      cpu: "16"
+
+pxe:
+  proverEnabled: true
+
+bot:
+  enabled: true
+  pxeProverEnabled: true
+  txIntervalSeconds: 200
+
+jobs:
+  deployL1Verifier:
+    enable: true
+
+aztec:
+  slotDuration: 36
+  epochDuration: 32
+
+telemetry:
+  enabled: true
+  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318

spartan/aztec-network/values/1-validator-with-proving.yaml

@@ -19,6 +19,13 @@ proverAgent:
   realProofs: true
   bb:
     hardwareConcurrency: 16
+  resources:
+    requests:
+      memory: "64Gi"
+      cpu: "16"
+    limits:
+      memory: "96Gi"
+      cpu: "16"
 
 pxe:
   proverEnabled: true
@@ -35,7 +42,3 @@ jobs:
 aztec:
   slotDuration: 36
   epochDuration: 32
-
-telemetry:
-  enabled: true
-  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318

yarn-project/cli/src/cmds/infrastructure/setup_protocol_contract.ts

@@ -20,7 +20,7 @@ export async function setupProtocolContracts(
   };
   log('setupProtocolContracts: Wait options' + JSON.stringify(waitOpts));
   log('setupProtocolContracts: Creating PXE client...');
-  const pxe = createPXEClient(rpcUrl, makeFetch([], true));
+  const pxe = createPXEClient(rpcUrl, makeFetch([1, 1, 1, 1, 1], false));
   const wallet = new SignerlessWallet(pxe, new DefaultMultiCallEntrypoint(l1ChainId, 1));
 
   log('setupProtocolContracts: Getting fee juice portal address...');

yarn-project/cli/src/cmds/misc/setup_contracts.ts

@@ -17,9 +17,22 @@ export async function setupCanonicalL2FeeJuice(
   const { FeeJuiceContract } = await import('@aztec/noir-contracts.js');
 
   const feeJuiceContract = await FeeJuiceContract.at(ProtocolContractAddress.FeeJuice, deployer);
-  log('setupCanonicalL2FeeJuice: Calling initialize on fee juice contract...');
-  await feeJuiceContract.methods
-    .initialize(feeJuicePortalAddress)
-    .send({ fee: { paymentMethod: new NoFeePaymentMethod(), gasSettings: GasSettings.teardownless() } })
-    .wait(waitOpts);
+
+  const portalAddress = await deployer.getPublicStorageAt(
+    feeJuiceContract.address,
+    feeJuiceContract.artifact.storageLayout.portal_address.slot,
+  );
+
+  if (portalAddress.isZero()) {
+    log('setupCanonicalL2FeeJuice: Calling initialize on fee juice contract...');
+    await feeJuiceContract.methods
+      .initialize(feeJuicePortalAddress)
+      .send({ fee: { paymentMethod: new NoFeePaymentMethod(), gasSettings: GasSettings.teardownless() } })
+      .wait(waitOpts);
+  } else {
+    log(
+      'setupCanonicalL2FeeJuice: Fee juice contract already initialized. Fee Juice Portal address: ' +
+        portalAddress.toString(),
+    );
+  }
 }

yarn-project/end-to-end/scripts/network_test.sh

@@ -30,6 +30,8 @@ INSTALL_CHAOS_MESH="${INSTALL_CHAOS_MESH:-}"
 CHAOS_VALUES="${CHAOS_VALUES:-}"
 FRESH_INSTALL="${FRESH_INSTALL:-false}"
 AZTEC_DOCKER_TAG=${AZTEC_DOCKER_TAG:-$(git rev-parse HEAD)}
+INSTALL_TIMEOUT=${INSTALL_TIMEOUT:-30m}
+CLEANUP_CLUSTER=${CLEANUP_CLUSTER:-false}
 
 # Check required environment variable
 if [ -z "${NAMESPACE:-}" ]; then
@@ -51,8 +53,11 @@ if [ "$FRESH_INSTALL" = "true" ]; then
   kubectl delete namespace "$NAMESPACE" --ignore-not-found=true --wait=true --now --timeout=10m
 fi
 
+STERN_PID=""
 function copy_stern_to_log() {
-  stern spartan -n $NAMESPACE > $SCRIPT_DIR/network-test.log
+  ulimit -n 4096
+  stern spartan -n $NAMESPACE > $SCRIPT_DIR/network-test.log &
+  STERN_PID=$!
 }
 
 function show_status_until_pxe_ready() {
@@ -103,12 +108,16 @@ handle_network_shaping() {
     return 0
 }
 
-copy_stern_to_log &
+copy_stern_to_log
 show_status_until_pxe_ready &
 
 function cleanup() {
   # kill everything in our process group except our process
-  trap - SIGTERM && kill $(pgrep -g $$ | grep -v $$) $(jobs -p) &>/dev/null || true
+  trap - SIGTERM && kill -9 $(pgrep -g $$ | grep -v $$) $(jobs -p) $STERN_PID &>/dev/null || true
+
+  if [ "$CLEANUP_CLUSTER" = "true" ]; then
+    kind delete cluster || true
+  fi
 }
 trap cleanup SIGINT SIGTERM EXIT
 
@@ -126,7 +135,7 @@ helm upgrade --install spartan "$REPO/spartan/aztec-network/" \
       --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \
       --wait \
       --wait-for-jobs=true \
-      --timeout=30m
+      --timeout="$INSTALL_TIMEOUT"
 
 kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=10m
 

yarn-project/end-to-end/src/spartan/proving.test.ts

@@ -0,0 +1,73 @@
+import { type PXE, createCompatibleClient, sleep } from '@aztec/aztec.js';
+import { createDebugLogger } from '@aztec/foundation/log';
+
+import { jest } from '@jest/globals';
+import { type ChildProcess } from 'child_process';
+
+import { getConfig, isK8sConfig, startPortForward } from './k8_utils.js';
+
+jest.setTimeout(2_400_000); // 40 minutes
+
+const config = getConfig(process.env);
+const debugLogger = createDebugLogger('aztec:spartan-test:proving');
+const SLEEP_MS = 1000;
+
+describe('proving test', () => {
+  let pxe: PXE;
+  let proc: ChildProcess | undefined;
+  beforeAll(async () => {
+    let PXE_URL;
+    if (isK8sConfig(config)) {
+      proc = await startPortForward({
+        resource: 'svc/spartan-aztec-network-pxe',
+        namespace: config.NAMESPACE,
+        containerPort: config.CONTAINER_PXE_PORT,
+        hostPort: config.HOST_PXE_PORT,
+      });
+      PXE_URL = `http://127.0.0.1:${config.HOST_PXE_PORT}`;
+    } else {
+      PXE_URL = config.PXE_URL;
+    }
+    pxe = await createCompatibleClient(PXE_URL, debugLogger);
+  });
+
+  afterAll(() => {
+    proc?.kill('SIGKILL');
+  });
+
+  it('advances the proven chain', async () => {
+    let [provenBlockNumber, blockNumber] = await Promise.all([pxe.getProvenBlockNumber(), pxe.getBlockNumber()]);
+    let ok: boolean;
+
+    debugLogger.info(`Initial pending chain tip: ${blockNumber}`);
+    debugLogger.info(`Initial proven chain tip: ${provenBlockNumber}`);
+
+    while (true) {
+      const [newProvenBlockNumber, newBlockNumber] = await Promise.all([
+        pxe.getProvenBlockNumber(),
+        pxe.getBlockNumber(),
+      ]);
+
+      if (newBlockNumber > blockNumber) {
+        debugLogger.info(`Pending chain has advanced: ${blockNumber} -> ${newBlockNumber}`);
+      } else if (newBlockNumber < blockNumber) {
+        debugLogger.error(`Pending chain has been pruned: ${blockNumber} -> ${newBlockNumber}`);
+        ok = false;
+        break;
+      }
+
+      if (newProvenBlockNumber > provenBlockNumber) {
+        debugLogger.info(`Proven chain has advanced: ${provenBlockNumber} -> ${newProvenBlockNumber}`);
+        ok = true;
+        break;
+      }
+
+      provenBlockNumber = newProvenBlockNumber;
+      blockNumber = newBlockNumber;
+
+      await sleep(SLEEP_MS);
+    }
+
+    expect(ok).toBeTrue();
+  });
+});