feat: phase 1 opts + recursion circuit (#1638)

Co-authored-by: rkm0959 <[email protected]> Co-authored-by: Tamir Hemo <[email protected]> Co-authored-by: Ratan Kaliani <[email protected]> Co-authored-by: Chris T. <[email protected]> Co-authored-by: umadayal <[email protected]> Co-authored-by: Conner Swann <[email protected]> Co-authored-by: tamirhemo <[email protected]>
succinctlabs · Oct 16, 2024 · 79884ff · 79884ff
1 parent 6512b56
commit 79884ff
Show file tree

Hide file tree

Showing 512 changed files with 21,201 additions and 42,233 deletions.
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -3,6 +3,14 @@ inputs:
   pull_token:
     description: "Token to use for private repo access"
     required: true
+  setup_gcc:
+    description: "Whether to setup GCC or not"
+    required: false
+    default: 'true'
+  setup_aws_cli:
+    description: "Whether to install AWS CLI or not"
+    required: false
+    default: 'true'
 runs:
   using: "composite"
   steps:
@@ -22,11 +30,41 @@ runs:
       shell: bash
       run: go version
 
+    - name: Check GCC version
+      id: check-gcc
+      shell: bash
+      run: |
+        if command -v gcc &> /dev/null; then
+          echo "gcc_exists=true" >> $GITHUB_OUTPUT
+          echo "gcc_version=$(gcc --version | head -n1 | awk '{print $NF}')" >> $GITHUB_OUTPUT
+        else
+          echo "gcc_exists=false" >> $GITHUB_OUTPUT
+        fi
+
     - name: Setup GCC
       uses: Dup4/actions-setup-gcc@v1
+      if: inputs.setup_gcc == 'true' && steps.check-gcc.outputs.gcc_exists != 'true'
       with:
         version: latest
 
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10' 
+
+    - name: Install AWS CLI v2
+      if: inputs.setup_aws_cli == 'true'
+      shell: bash
+      run: |
+        if ! command -v aws &> /dev/null; then
+          echo "AWS CLI not found. Installing..."
+          python3 -m pip install --user awscli
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+        else
+          echo "AWS CLI is already installed."
+        fi
+        export PATH="$HOME/.local/bin:$PATH"
+        aws --version
+
     - name: rust-cache
       uses: actions/cache@v3
       with:
@@ -45,3 +83,20 @@ runs:
       shell: bash
       run: |
         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.81.0 -y
+        . "$HOME/.cargo/env" 
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+    # install pkg-config and openssl
+    - name: Install pkg-config and openssl
+      shell: bash
+      run: |
+        if ! dpkg -s pkg-config libssl-dev &> /dev/null; then
+          echo "pkg-config and/or libssl-dev not found. Installing..."
+          sudo apt-get update
+          sudo apt-get install -y pkg-config libssl-dev
+        else
+          echo "pkg-config and libssl-dev are already installed."
+        fi
+      
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v3
diff --git a/.github/runs-on.yml b/.github/runs-on.yml
@@ -0,0 +1,17 @@
+images:
+  nvidia-linux:
+    platform: "linux"
+    arch: "x64"
+    ami: "ami-0a63dc9cb9e934ba3"
+    owner: "421253708207"
+
+  dlami-x64:
+    platform: "linux"
+    arch: "x64"
+    owner: "898082745236" # AWS
+    name: "Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04)*"
+
+runners:
+  gpu-nvidia:
+    family: ["g6.4xlarge"]
+    image: dlami-x64
diff --git a/.github/workflows/docker-publish-gnark.yml b/.github/workflows/docker-publish-gnark.yml
@@ -1,4 +1,3 @@
-# Source: https://raw.githubusercontent.com/foundry-rs/foundry/master/.github/workflows/docker-publish.yml
 name: docker-gnark
 
 on:
@@ -9,7 +8,6 @@ on:
       - "v*.*.*"
   schedule:
     - cron: "0 0 * * *"
-  # Trigger without any parameters a proactive rebuild
   workflow_dispatch:
     inputs:
       tags:
@@ -22,83 +20,85 @@ env:
   IMAGE_NAME: succinctlabs/sp1-gnark
 
 jobs:
-  container:
-    runs-on: ubuntu-latest
-    # https://docs.github.com/en/actions/reference/authentication-in-a-workflow
+  build-amd64:
+    runs-on: [runs-on, runner=64cpu-linux-x64, spot=false, "run-id=${{ github.run_id }}"]
     permissions:
       id-token: write
       packages: write
       contents: read
-    timeout-minutes: 120
     steps:
       - name: Checkout repository
-        id: checkout
         uses: actions/checkout@v4
-
-      - name: Install Docker BuildX
+      - name: Set up Docker BuildX
         uses: docker/setup-buildx-action@v3
-        id: buildx
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3
         with:
-          install: true
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push AMD64 image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.gnark-ffi
+          platforms: linux/amd64
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-      # Login against a Docker registry except on PR
-      # https://github.com/docker/login-action
+  build-arm64:
+    runs-on: [runs-on, runner=64cpu-linux-arm64, spot=false, "run-id=${{ github.run_id }}"]
+    permissions:
+      id-token: write
+      packages: write
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Docker BuildX
+        uses: docker/setup-buildx-action@v3
       - name: Log into registry ${{ env.REGISTRY }}
-        # Ensure this doesn't trigger on PR's
-        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
-      - name: Extract Docker metadata
-        id: meta
-        uses: docker/metadata-action@v5
+      - name: Build and push ARM64 image
+        uses: docker/build-push-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          context: .
+          file: ./Dockerfile.gnark-ffi
+          platforms: linux/arm64
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-      # Creates an additional 'latest' or 'nightly' tag
-      # If the job is triggered via cron schedule, tag nightly and nightly-{SHA}
-      # If the job is triggered via workflow dispatch and on a master branch, tag branch and latest
-      # Otherwise, just tag as the branch name
-      - name: Finalize Docker Metadata
-        id: docker_tagging
+  create-manifest:
+    needs: [build-amd64, build-arm64]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create and push manifest
         run: |
+          TAGS=""
           if [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then
-            echo "manual trigger from workflow_dispatch, assigning tag ${{ github.event.inputs.tags }}"
-            echo "docker_tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.tags }}" >> $GITHUB_OUTPUT
+            TAGS="${{ github.event.inputs.tags }}"
           elif [[ "${{ github.event_name }}" == 'schedule' ]]; then
-            echo "cron trigger, assigning nightly tag"
-            echo "docker_tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly-${GITHUB_SHA}" >> $GITHUB_OUTPUT
+            TAGS="nightly nightly-${{ github.sha }}"
           else
-            echo "Neither scheduled nor manual release from main branch. Just tagging as branch name"
-            echo "docker_tags=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
+            TAGS="${GITHUB_REF##*/}"
           fi
-
-      # Log docker metadata to explicitly know what is being pushed
-      - name: Inspect Docker Metadata
-        run: |
-          echo "TAGS -> ${{ steps.docker_tagging.outputs.docker_tags }}"
-          echo "LABELS ->  ${{ steps.meta.outputs.labels }}"
-
-      # Build and push Docker image
-      # https://github.com/docker/build-push-action
-      # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ./Dockerfile.gnark-ffi
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.docker_tagging.outputs.docker_tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          build-args: |
-            BUILDTIME=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
-            VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
-            REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+          
+          for TAG in $TAGS; do
+            docker buildx imagetools create -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:$TAG \
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-amd64 \
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-arm64
+          done