diff --git a/.github/workflows/deploy-fisherman-network.yaml b/.github/workflows/deploy-fisherman-network.yaml index 6b4b8ecabf9b..1ef3df5ec584 100644 --- a/.github/workflows/deploy-fisherman-network.yaml +++ b/.github/workflows/deploy-fisherman-network.yaml @@ -185,6 +185,7 @@ jobs: network: staging-ignition namespace: ignition-fisherman-sepolia l1_network: sepolia + cluster: aztec-gke-private setup-irm-mainnet: needs: deploy-fisherman @@ -195,3 +196,4 @@ jobs: network: mainnet namespace: ignition-fisherman-mainnet l1_network: mainnet + cluster: aztec-gke-private diff --git a/.github/workflows/deploy-irm.yml b/.github/workflows/deploy-irm.yml index beafdb40f09f..48efd988d4c3 100644 --- a/.github/workflows/deploy-irm.yml +++ b/.github/workflows/deploy-irm.yml @@ -53,7 +53,6 @@ on: required: true type: string default: "aztec-gke-private" - jobs: deploy-irm: env: @@ -86,6 +85,13 @@ jobs: run: | gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --region ${{ env.REGION }} + - name: Get IRM version from VERSION file + id: get_version + run: | + VERSION=$(cat spartan/metrics/irm-monitor/VERSION | tr -d '[:space:]') + echo "IRM version: ${VERSION}" + echo "IMAGE_TAG=${VERSION}" >> "$GITHUB_ENV" + - name: Validate inputs run: | # Validate l1_network @@ -123,10 +129,15 @@ jobs: echo "MONITORING_NAMESPACE=${EFFECTIVE_MONITORING_NAMESPACE}" >> "$GITHUB_ENV" - name: Deploy IRM + env: + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + IMAGE_TAG: ${{ env.IMAGE_TAG }} run: | echo "Deploying IRM to network: ${{ inputs.network }}" echo "Namespace to monitor: ${NAMESPACE}" echo "Monitoring namespace: ${MONITORING_NAMESPACE}" echo "L1 network: ${{ inputs.l1_network }}" + echo "Image tag: ${IMAGE_TAG}" ./spartan/metrics/irm-monitor/scripts/update-monitoring.sh $NAMESPACE $MONITORING_NAMESPACE ${{ inputs.network }} $INFURA_SECRET_NAME diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index e4790c0db5c0..8f8e7d0df1e3 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -143,7 +143,7 @@ jobs: if [ -n "$CLUSTER" ]; then echo "cluster=$CLUSTER" >> $GITHUB_OUTPUT - else + else echo "cluster=" >> $GITHUB_OUTPUT fi diff --git a/spartan/metrics/grafana/dashboards/l1_fees.json b/spartan/metrics/grafana/dashboards/l1_fees.json index 36d13035cfea..33747c63c2ed 100644 --- a/spartan/metrics/grafana/dashboards/l1_fees.json +++ b/spartan/metrics/grafana/dashboards/l1_fees.json @@ -985,13 +985,342 @@ "title": "Blob Transaction Counts", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Total number of blobs in pending and included blocks", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "blobs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pending" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Included" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum(increase(aztec_fisherman_fee_analysis_pending_blob_count_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) / sum(increase(aztec_fisherman_fee_analysis_pending_blob_count_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", + "legendFormat": "Pending", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum(increase(aztec_fisherman_fee_analysis_included_blob_count_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) / sum(increase(aztec_fisherman_fee_analysis_included_blob_count_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", + "legendFormat": "Included", + "range": true, + "refId": "B" + } + ], + "title": "Total Blob Counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Percentage of blocks that reached 100% blob capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 34, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum(increase(aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\",aztec_ok=\"true\"}[$__range])) / sum(increase(aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\"}[$__range]))", + "legendFormat": "Blocks Full Rate", + "range": true, + "refId": "A" + } + ], + "title": "Block Blob Capacity Full Rate", + "type": "gauge" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 45 + }, + "id": 60, + "panels": [], + "title": "Full Block Analysis", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Inclusion rate by strategy for blocks that reached 100% blob capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum by(aztec_fisherman_strategy_id) (\n increase(aztec_fisherman_fee_analysis_would_be_included{k8s_namespace_name=\"$namespace\",aztec_ok=\"true\"}[$__rate_interval])\n * on() group_left()\n (aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\",aztec_ok=\"true\"} > 0)\n) / sum by(aztec_fisherman_strategy_id) (\n increase(aztec_fisherman_fee_analysis_would_be_included{k8s_namespace_name=\"$namespace\"}[$__rate_interval])\n * on() group_left()\n (aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\"} > 0)\n)", + "legendFormat": "{{aztec_fisherman_strategy_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Inclusion Rate by Strategy (Full Blocks Only)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 }, "id": 50, "panels": [], @@ -1044,7 +1373,7 @@ "h": 10, "w": 12, "x": 0, - "y": 38 + "y": 56 }, "id": 51, "options": { @@ -1149,7 +1478,7 @@ "h": 10, "w": 12, "x": 12, - "y": 38 + "y": 56 }, "id": 52, "options": { @@ -1214,7 +1543,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 66 }, "id": 40, "panels": [], @@ -1342,7 +1671,7 @@ "h": 9, "w": 24, "x": 0, - "y": 49 + "y": 67 }, "id": 41, "options": { diff --git a/spartan/metrics/irm-monitor/VERSION b/spartan/metrics/irm-monitor/VERSION new file mode 100644 index 000000000000..3eefcb9dd5b3 --- /dev/null +++ b/spartan/metrics/irm-monitor/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/spartan/metrics/irm-monitor/index.ts b/spartan/metrics/irm-monitor/index.ts index a8735853a09f..75a8be9d6f95 100644 --- a/spartan/metrics/irm-monitor/index.ts +++ b/spartan/metrics/irm-monitor/index.ts @@ -4,6 +4,10 @@ import client from "prom-client"; const { ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOST, NETWORK } = process.env; +////////////////////////////// +// IMPORTANT: Bump VERSION file when making changes +////////////////////////////// + if (!ROLLUP_CONTRACT_ADDRESS || !ETHEREUM_HOST || !NETWORK) { console.error( "ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOST and NETWORK are required. Provided: ", diff --git a/spartan/metrics/irm-monitor/scripts/build-and-publish.sh b/spartan/metrics/irm-monitor/scripts/build-and-publish.sh index db9027cedcbb..93d1df568ade 100755 --- a/spartan/metrics/irm-monitor/scripts/build-and-publish.sh +++ b/spartan/metrics/irm-monitor/scripts/build-and-publish.sh @@ -2,16 +2,22 @@ set -euo pipefail -# Build and publish aztecprotocol/aztec-block-height-monitor if the tag doesn't exist -# Usage: ./build-and-publish.sh - -TAG=${1:-latest} -IMAGE="spypsy/block-height-monitor:${TAG}" +# Build and publish the IRM monitor image if it doesn't exist +# Usage: ./build-and-publish.sh (e.g., aztecprotocol/block-height-monitor:2.3.4) +IMAGE=${1:-aztecprotocol/block-height-monitor:latest} echo "Checking if ${IMAGE} exists on Docker Hub..." -if curl -fsSL "https://hub.docker.com/v2/repositories/spypsy/block-height-monitor/tags/${TAG}" >/dev/null 2>&1; then - echo "Image tag already exists: ${IMAGE}" + +# Extract repository and tag from the full image name +REPO="${IMAGE%%:*}" +TAG="${IMAGE##*:}" + +echo "REPO: ${REPO}" +echo "TAG: ${TAG}" + +if curl -fsSL "https://hub.docker.com/v2/repositories/${REPO}/tags/${TAG}" >/dev/null 2>&1; then + echo "Image already exists: ${IMAGE}" exit 0 fi diff --git a/spartan/metrics/irm-monitor/scripts/update-monitoring.sh b/spartan/metrics/irm-monitor/scripts/update-monitoring.sh index 1658b3600bb4..69e860041bf4 100755 --- a/spartan/metrics/irm-monitor/scripts/update-monitoring.sh +++ b/spartan/metrics/irm-monitor/scripts/update-monitoring.sh @@ -13,9 +13,9 @@ INFURA_URL_SECRET=${4:-"infura-sepolia-url"} # Deployment name includes the monitoring namespace prefix export DEPLOYMENT_NAME="${MONITORING_NAMESPACE}-monitor" -# Docker image (can be overridden via IMAGE_TAG or IMAGE environment variable) +# Docker image tag (defaults to 'latest' if not provided via IMAGE_TAG environment variable) IMAGE_TAG=${IMAGE_TAG:-latest} -IMAGE=${IMAGE:-"spypsy/block-height-monitor:${IMAGE_TAG}"} +IMAGE="aztecprotocol/block-height-monitor:${IMAGE_TAG}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BASE_DIR="$SCRIPT_DIR/.." @@ -156,8 +156,8 @@ yq eval ".metadata.name = \"${DEPLOYMENT_NAME}\" | # Build image if missing (initial install path only) SCRIPT_BUILD="$SCRIPT_DIR/build-and-publish.sh" if [ -x "$SCRIPT_BUILD" ]; then - echo "Ensuring image spypsy/block-height-monitor:${IMAGE_TAG} exists..." - "$SCRIPT_BUILD" "$IMAGE_TAG" + echo "Ensuring image ${IMAGE} exists..." + "$SCRIPT_BUILD" "$IMAGE" fi echo "Applying Deployment..." diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index a675b161f00b..3ecc55309b70 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -49,6 +49,9 @@ export class SequencerMetrics { private fishermanTimeBeforeBlock: Histogram; private fishermanPendingBlobTxCount: Histogram; private fishermanIncludedBlobTxCount: Histogram; + private fishermanPendingBlobCount: Histogram; + private fishermanIncludedBlobCount: Histogram; + private fishermanBlockBlobsFull: UpDownCounter; private fishermanCalculatedPriorityFee: Histogram; private fishermanPriorityFeeDelta: Histogram; private fishermanEstimatedCost: Histogram; @@ -238,6 +241,29 @@ export class SequencerMetrics { valueType: ValueType.DOUBLE, }, ); + + this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT, { + description: 'Number of blobs seen in the pending block', + valueType: ValueType.INT, + }); + + this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT, { + description: 'Number of blobs that got included in the mined block', + valueType: ValueType.INT, + }); + + this.fishermanBlockBlobsFull = this.meter.createUpDownCounter(Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL, { + valueType: ValueType.INT, + description: 'Whether the mined block reached 100% blob capacity', + }); + + // Initialize the counter + this.fishermanBlockBlobsFull.add(0, { + [Attributes.OK]: true, + }); + this.fishermanBlockBlobsFull.add(0, { + [Attributes.OK]: false, + }); } public recordRequiredAttestations(requiredAttestationsCount: number, allowanceMs: number) { @@ -348,10 +374,12 @@ export class SequencerMetrics { // Record pending block snapshot data (once per strategy for comparison) this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes); + this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes); // Record mined block data if available if (analysis.minedBlock) { this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes); + this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes); // Record actual fees from blob transactions in the mined block for (const blobTx of analysis.minedBlock.includedBlobTxs) { @@ -385,6 +413,13 @@ export class SequencerMetrics { if (analysis.analysis) { this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes); + // Record whether the block reached 100% blob capacity + if (analysis.analysis.blockBlobsFull) { + this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: true }); + } else { + this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false }); + } + // Record strategy-specific inclusion result if (strategyResult.wouldBeIncluded !== undefined) { if (strategyResult.wouldBeIncluded) { diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 3ed87b855310..d27107e491f0 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -98,6 +98,9 @@ export const FISHERMAN_FEE_ANALYSIS_ESTIMATED_OVERPAYMENT = 'aztec.fisherman.fee export const FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_PRIORITY_FEE = 'aztec.fisherman.fee_analysis.mined_blob_tx_priority_fee'; export const FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST = 'aztec.fisherman.fee_analysis.mined_blob_tx_total_cost'; +export const FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT = 'aztec.fisherman.fee_analysis.pending_blob_count'; +export const FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT = 'aztec.fisherman.fee_analysis.included_blob_count'; +export const FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL = 'aztec.fisherman.fee_analysis.block_blobs_full'; export const VALIDATOR_INVALID_ATTESTATION_RECEIVED_COUNT = 'aztec.validator.invalid_attestation_received_count';