Merge branch 'aptos-labs:main' into fix_large_packages

.github/actions/fullnode-sync/action.yaml

@@ -27,6 +27,11 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Install Dependencies
+      shell: bash
+      run: |
+        echo "${HOME}/bin/" >> $GITHUB_PATH # default INSTALL_DIR to path
+        scripts/dev_setup.sh -b # Install dependencies
     - name: Run fullnode sync
       shell: bash
       run: |

.github/runs-on.yml

@@ -1,11 +1,3 @@
-images:
-  aptos-ubuntu-x64:
-    platform: "linux"
-    arch: "x64"
-    ami: "ami-09f7d4bfb08b771be" # image for testing, with gcloud sdk and kubectl
-    owner: "058264343338" # aptos-ci AWS account id
-  aptos-ubuntu-x64-latest:
-    platform: "linux"
-    arch: "x64"
-    owner: "058264343338" # aptos-ci AWS account id
-    name: "aptos-runs-on-*"
+# Inherit from the internal-ops repo
+# https://runs-on.com/configuration/repo-config/
+_extends: internal-ops

.github/workflows/adhoc-forge.yaml

@@ -8,33 +8,33 @@ on:
       GIT_SHA:
         required: true
         type: string
-        description: The git SHA1 to checkout and test
+        description: The git SHA1 to checkout and test. This can also be a branch or tag
       IMAGE_TAG:
         required: false
         type: string
-        description: The docker image tag to test. If not specified, falls back on GIT_SHA
+        description: The docker image tag to test. If not specified, it is derived from the current check-out
       FORGE_IMAGE_TAG:
         required: false
         type: string
-        description: The docker image tag to use for forge runner. If not specified, falls back on GIT_SHA
+        description: The docker image tag to use for forge runner. If not specified, it is derived from the current check-out
       FORGE_RUNNER_DURATION_SECS:
         required: false
         type: string
         default: "480"
-        description: Duration of the forge test run
+        description: Duration of the forge test run. For multi-phasic tests this is the length of a single phase
       FORGE_TEST_SUITE:
         required: false
         type: string
         default: land_blocking
-        description: Test suite to run
+        description: Test suite to run. See https://github.com/aptos-labs/aptos-core/blob/main/testsuite/forge-cli/src/main.rs
       FORGE_CLUSTER_NAME:
         required: false
         type: string
         description: The Forge k8s cluster to be used for test
-      FORGE_ENABLE_INDEXER:
+      FORGE_INDEXER_DEPLOYER_PROFILE:
         required: false
-        type: boolean
-        description: Whether to use indexer
+        type: string
+        description: If set enables an indexer stack with your test, spinning up with the provided profile from https://github.com/aptos-labs/internal-ops/blob/main/infra/cli/commands/forge/profiles.ts
       FORGE_NUM_VALIDATORS:
         required: false
         type: string
@@ -66,21 +66,21 @@ jobs:
           echo "FORGE_RUNNER_DURATION_SECS: ${{ inputs.FORGE_RUNNER_DURATION_SECS }}"
           echo "FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}"
           echo "FORGE_CLUSTER_NAME: ${{ inputs.FORGE_CLUSTER_NAME }}"
-          echo "FORGE_ENABLE_INDEXER: ${{ inputs.FORGE_ENABLE_INDEXER }}"
           echo "FORGE_NUM_VALIDATORS: ${{ inputs.FORGE_NUM_VALIDATORS }}"
           echo "FORGE_NUM_VALIDATOR_FULLNODES: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}"
           echo "FORGE_RETAIN_DEBUG_LOGS: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}"
+          echo "FORGE_INDEXER_DEPLOYER_PROFILE: ${{ inputs.FORGE_INDEXER_DEPLOYER_PROFILE }}"
     outputs:
       gitSha: ${{ inputs.GIT_SHA }}
       imageTag: ${{ inputs.IMAGE_TAG }}
       forgeImageTag: ${{ inputs.FORGE_IMAGE_TAG }}
       forgeRunnerDurationSecs: ${{ inputs.FORGE_RUNNER_DURATION_SECS || 600 }}
       forgeTestSuite: ${{ inputs.FORGE_TEST_SUITE }}
       forgeClusterName: ${{ inputs.FORGE_CLUSTER_NAME }}
-      forgeEnableIndexer: ${{ inputs.FORGE_ENABLE_INDEXER }}
       forgeNumValidators: ${{ inputs.FORGE_NUM_VALIDATORS }}
       forgeNumValidatorFullnodes: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}
       forgeRetainDebugLogs: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}
+      forgeIndexerDeployerProfile: ${{ inputs.FORGE_INDEXER_DEPLOYER_PROFILE }}
   adhoc-forge-test:
     uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
     needs: [determine-forge-run-metadata]
@@ -92,7 +92,9 @@ jobs:
       FORGE_TEST_SUITE: ${{ needs.determine-forge-run-metadata.outputs.forgeTestSuite }}
       FORGE_RUNNER_DURATION_SECS: ${{ fromJSON(needs.determine-forge-run-metadata.outputs.forgeRunnerDurationSecs) }} # fromJSON converts to integer
       FORGE_CLUSTER_NAME: ${{ needs.determine-forge-run-metadata.outputs.forgeClusterName }}
-      FORGE_ENABLE_INDEXER: ${{ needs.determine-forge-run-metadata.outputs.forgeEnableIndexer == 'true' }}
       FORGE_NUM_VALIDATORS: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidators }}
       FORGE_NUM_VALIDATOR_FULLNODES: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidatorFullnodes }}
       FORGE_RETAIN_DEBUG_LOGS: ${{ needs.determine-forge-run-metadata.outputs.forgeRetainDebugLogs == 'true' }}
+      # Indexer outputs are derived from a single one, to avoid using too many inputs for this workflow_dispatch trigger
+      FORGE_ENABLE_INDEXER: ${{ needs.determine-forge-run-metadata.outputs.forgeIndexerDeployerProfile != '' }}
+      FORGE_DEPLOYER_PROFILE: ${{ needs.determine-forge-run-metadata.outputs.forgeIndexerDeployerProfile }}

.github/workflows/copy-images-to-dockerhub.yaml

@@ -10,6 +10,11 @@ on:
         required: false
         type: string
         description: the git sha to use for the image tag. If not provided, the git sha of the triggering branch will be used
+      dry_run:
+        required: false
+        type: boolean
+        default: false
+        description: If true, run the workflow without actually pushing images
   workflow_dispatch:
     inputs:
       image_tag_prefix:
@@ -21,6 +26,11 @@ on:
         required: false
         type: string
         description: the git sha to use for the image tag. If not provided, the git sha of the triggering branch will be used
+      dry_run:
+        required: false
+        type: boolean
+        default: false
+        description: If true, run the workflow without actually pushing images
 
 permissions:
   contents: read
@@ -29,7 +39,7 @@ permissions:
 jobs:
   copy-images:
     # Run on a machine with more local storage for large docker images
-    runs-on: medium-perf-docker-with-local-ssd
+    runs-on: runs-on,cpu=16,family=m6id,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }}
     steps:
       - uses: actions/checkout@v4
 
@@ -61,4 +71,4 @@ jobs:
           AWS_ACCOUNT_ID: ${{ secrets.AWS_ECR_ACCOUNT_NUM }}
           GCP_DOCKER_ARTIFACT_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}
           IMAGE_TAG_PREFIX: ${{ inputs.image_tag_prefix }}
-        run: ./docker/release-images.mjs --wait-for-image-seconds=3600
+        run: ./docker/release-images.mjs --wait-for-image-seconds=3600 ${{ inputs.dry_run && '--dry-run' || '' }}

.github/workflows/coverage-move-only.yaml

@@ -10,12 +10,12 @@ on:
       - 'aptos-move/e2e-move-tests/**'
       - 'aptos-move/framework/**'
       - '.github/workflows/coverage-move-only.yaml'
-  pull_request:
-    paths:
-      - 'third_party/move/**'
-      - 'aptos-move/e2e-move-tests/**'
-      - 'aptos-move/framework/**'
-      - '.github/workflows/coverage-move-only.yaml'
+  # pull_request:
+  # paths:
+  #   - 'third_party/move/**'
+  #   - 'aptos-move/e2e-move-tests/**'
+  #   - 'aptos-move/framework/**'
+  #   - '.github/workflows/coverage-move-only.yaml'
 
 env:
   CARGO_INCREMENTAL: "0"

.github/workflows/forge-stable.yaml

@@ -120,6 +120,7 @@ jobs:
             echo "BRANCH: [${BRANCH}](https://github.com/${{ github.repository }}/tree/${BRANCH})" >> $GITHUB_STEP_SUMMARY
           fi
           echo "IMAGE_TAG: [${IMAGE_TAG}](https://github.com/${{ github.repository }}/commit/${IMAGE_TAG})" >> $GITHUB_STEP_SUMMARY
+          echo "To cancel this job, do `pnpm infra ci cancel-workflow ${{ github.run_id }}` from internal-ops" >> $GITHUB_STEP_SUMMARY
 
   ### Real-world-network tests.
   # Run forge framework upgradability test. This is a PR required job.

.github/workflows/forge-unstable.yaml

@@ -114,6 +114,7 @@ jobs:
             echo "BRANCH: [${BRANCH}](https://github.com/${{ github.repository }}/tree/${BRANCH})" >> $GITHUB_STEP_SUMMARY
           fi
           echo "IMAGE_TAG: [${IMAGE_TAG}](https://github.com/${{ github.repository }}/commit/${IMAGE_TAG})" >> $GITHUB_STEP_SUMMARY
+          echo "To cancel this job, do `pnpm infra ci cancel-workflow ${{ github.run_id }}` from internal-ops" >> $GITHUB_STEP_SUMMARY
 
 # Test definitions start below
 # To add a new Forge test, add a new job definition below. Copy all fields and change only the:
@@ -128,6 +129,7 @@ jobs:
     secrets: inherit
     with:
       IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
+      FORGE_NAMESPACE: forge-indexer-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
       COMMENT_HEADER: forge-indexer
       FORGE_TEST_SUITE: indexer_test
       FORGE_ENABLE_INDEXER: true

.github/workflows/indexer-processor-testing.yaml

@@ -6,6 +6,8 @@ on:
   pull_request:  # Trigger on PR-level events
     branches:
       - main
+    paths:
+      - 'ecosystem/indexer-grpc/indexer-test-transactions/**'  # Only trigger if files under this path change
 
 # the required permissions to request the ID token
 permissions:
@@ -14,7 +16,7 @@ permissions:
 
 jobs:
   dispatch_event:
-    runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }}
+    runs-on: runs-on,cpu=16,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }}
 
     steps:
       - name: Checkout the repository

.github/workflows/run-fullnode-sync.yaml

@@ -57,7 +57,7 @@ on:
 
 jobs:
   fullnode-sync:
-    runs-on: runs-on,cpu=16,ram=64,family=m5ad,image=aptos-ubuntu-x64,run-id=${{ github.run_id }}
+    runs-on: medium-perf-docker-with-local-ssd
     timeout-minutes: ${{ inputs.TIMEOUT_MINUTES || 300 }} # the default run is 300 minutes (5 hours). Specified here because workflow_dispatch uses string rather than number
     steps:
       - uses: actions/checkout@v4

.github/workflows/workflow-run-execution-performance.yaml

@@ -22,10 +22,19 @@ on:
         default: false
         type: boolean
         description: Ignore target determination and run the tests
+      SKIP_MOVE_E2E:
+        required: false
+        default: false
+        type: boolean
+        description: Whether to run or skip move-only e2e tests at the beginning.
       SOURCE:
         required: false
         default: CI
         type: string
+      NUMBER_OF_EXECUTION_THREADS:
+        required: false
+        default: "32"
+        type: string
   # This allows the workflow to be triggered manually from the Github UI or CLI
   # NOTE: because the "number" type is not supported, we default to 720 minute timeout
   workflow_dispatch:
@@ -40,12 +49,23 @@ on:
         type: choice
         options:
         - executor-benchmark-runner
+        - executor-benchmark-ext4
+        - executor-benchmark-xfs
         - benchmark-t2d-32
         - benchmark-t2d-60
         - benchmark-c3d-30
+        - benchmark-c3d-60
+        - benchmark-c3d-180
         - benchmark-n4-32
         - benchmark-c4-32
+        - benchmark-c4-48
+        - benchmark-c4-96
         description: The name of the runner to use for the test. (which decides machine specs)
+      NUMBER_OF_EXECUTION_THREADS:
+        required: false
+        default: "32"
+        type: string
+        description: Number of execution threads to use for the tests.
       FLOW:
         required: false
         default: LAND_BLOCKING
@@ -56,6 +76,11 @@ on:
         - MAINNET_LARGE_DB
         type: choice
         description: Which set of tests to run. MAINNET/MAINNET_LARGE_DB are for performance validation of mainnet nodes.
+      SKIP_MOVE_E2E:
+        required: false
+        default: false
+        type: boolean
+        description: Whether to skip move-only e2e tests at the beginning.
       IGNORE_TARGET_DETERMINATION:
         required: false
         default: true
@@ -98,7 +123,7 @@ jobs:
 
       - name: Run single node execution benchmark in performance build mode
         shell: bash
-        run: TABULATE_INSTALL=lib-only pip install tabulate && FLOW=${{ inputs.FLOW }} SOURCE=${{ inputs.SOURCE }} testsuite/single_node_performance.py
+        run: TABULATE_INSTALL=lib-only pip install tabulate && FLOW="${{ inputs.FLOW }}" SOURCE="${{ inputs.SOURCE }}" RUNNER_NAME="${{ inputs.RUNNER_NAME }}" SKIP_MOVE_E2E="${{ inputs.SKIP_MOVE_E2E && '1' || '' }}" NUMBER_OF_EXECUTION_THREADS="${{ inputs.NUMBER_OF_EXECUTION_THREADS }}" testsuite/single_node_performance.py
         if: ${{ (inputs.IGNORE_TARGET_DETERMINATION || needs.test-target-determinator.outputs.run_execution_performance_test == 'true') }}
 
       - run: echo "Skipping single node execution performance! Unrelated changes detected."

.github/workflows/workflow-run-forge.yaml

@@ -52,10 +52,6 @@ on:
         required: false
         type: string
         description: Whether to use HAPRoxy
-      FORGE_ENABLE_INDEXER:
-        required: false
-        type: boolean
-        description: Whether to use indexer
       FORGE_ENABLE_PERFORMANCE:
         required: false
         type: string
@@ -83,6 +79,14 @@ on:
         required: false
         type: string
         description: Number of validator fullnodes to use for the forge test
+      FORGE_ENABLE_INDEXER:
+        required: false
+        type: boolean
+        description: Whether to use indexer
+      FORGE_DEPLOYER_PROFILE:
+        required: false
+        type: string
+        description: The deployer profile used to spin up and configure forge infrastructure
 
 env:
   AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
@@ -104,6 +108,7 @@ env:
   FORGE_NAMESPACE: ${{ inputs.FORGE_NAMESPACE }}
   FORGE_ENABLE_HAPROXY: ${{ inputs.FORGE_ENABLE_HAPROXY }}
   FORGE_ENABLE_INDEXER: ${{ inputs.FORGE_ENABLE_INDEXER }}
+  FORGE_DEPLOYER_PROFILE: ${{ inputs.FORGE_DEPLOYER_PROFILE }}
   FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}
   POST_TO_SLACK: ${{ inputs.POST_TO_SLACK }}
   FORGE_ENABLE_FAILPOINTS: ${{ inputs.FORGE_ENABLE_FAILPOINTS }}