WIP: kernel

Signed-off-by: Misha Sakhnov <[email protected]>
neondatabase · Oct 28, 2024 · 84ee7ca · 84ee7ca
1 parent 4ca3d4f
commit 84ee7ca
Show file tree

Hide file tree

Showing 2 changed files with 103 additions and 56 deletions.
diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml
@@ -91,19 +91,19 @@ jobs:
           echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
           echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
 
-  # vm-kernel:
-  #   # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'.
-  #   if: ${{ format('{0}', inputs.skip) != 'true' }}
-  #   uses: ./.github/workflows/vm-kernel.yaml
-  #   with:
-  #     tag: ${{ inputs.kernel-image || inputs.tag }}
-  #     return-image-for-tag: ${{ inputs.kernel-image }}
-  #   secrets: inherit
+  vm-kernel:
+    # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'.
+    if: ${{ format('{0}', inputs.skip) != 'true' }}
+    uses: ./.github/workflows/vm-kernel.yaml
+    with:
+      tag: ${{ inputs.kernel-image || inputs.tag }}
+      return-image-for-tag: ${{ inputs.kernel-image }}
+    secrets: inherit
 
   build:
     # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'.
     if: ${{ format('{0}', inputs.skip) != 'true' }}
-    needs: [ tags ] # vm-kernel ]
+    needs: [ tags , vm-kernel ]
     permissions:
       contents: read  # This is required for actions/checkout
       id-token: write # This is required for aws-actions/configure-aws-credentials
@@ -202,14 +202,15 @@ jobs:
           docker version
           docker buildx version
 
-      # - name: Load VM kernel
-      #   env:
-      #     IMAGE: ${{ needs.vm-kernel.outputs.image }}
-      #   run: |
-      #     docker pull --quiet $IMAGE
-      #     ID=$(docker create $IMAGE true)
-      #     docker cp ${ID}:/vmlinuz neonvm-kernel/vmlinuz
-      #     docker rm -f ${ID}
+      - name: Load VM kernel
+        env:
+          IMAGE: ${{ matrix.arch == 'amd64' && needs.vm-kernel.outputs.image_amd64 || needs.vm-kernel.outputs.image_arm64 }}
+        run: |
+          docker pull --quiet $IMAGE
+          ID=$(docker create $IMAGE true)
+          docker cp ${ID}:/vmlinuz neonvm-kernel/vmlinuz
+          docker rm -f ${ID}
+  
       - run: uname -a 
       - run: echo "${{ matrix.arch }}"
       # - name: Build go dependencies image
@@ -325,7 +326,7 @@ jobs:
       #       neonvm-controller \
       #       neonvm-vxlan-controller \
       #       neonvm-runner \
-      #       # vm-kernel \
+      #       vm-kernel \
       #       autoscale-scheduler \
       #       autoscaler-agent \
       #       cluster-autoscaler-neonvm \

diff --git a/.github/workflows/vm-kernel.yaml b/.github/workflows/vm-kernel.yaml
@@ -1,37 +1,43 @@
 name: vm-kernel
 
-on: {}
-  # workflow_dispatch: # adds ability to run this manually
-  #   inputs:
-  #     tag:
-  #       description: 'Tag to use for Docker image'
-  #       type: string
-  #       required: false
-  #     force-rebuild:
-  #       description: 'Rebuild the kernel image even if it already exists'
-  #       type: boolean
-  #       required: false
-  #       default: false
-  # workflow_call:
-  #   inputs:
-  #     tag:
-  #       description: 'Tag to use for Docker image'
-  #       type: string
-  #       required: false
-  #     return-image-for-tag:
-  #       description: 'Make workflow to return image for the passed tag without building or tagging anything'
-  #       type: string
-  #       required: false
-  #       default: ''
-  #     force-rebuild:
-  #       description: 'Rebuild the kernel image even if it already exists. No-op if `return-image-for-tag` is set'
-  #       type: boolean
-  #       required: false
-  #       default: false
-  #   outputs:
-  #     image:
-  #       description: 'vm-kernel Docker image'
-  #       value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image }}
+on:
+  workflow_dispatch: # adds ability to run this manually
+    inputs:
+      tag:
+        description: 'Tag to use for Docker image'
+        type: string
+        required: false
+      force-rebuild:
+        description: 'Rebuild the kernel image even if it already exists'
+        type: boolean
+        required: false
+        default: false
+  workflow_call:
+    inputs:
+      tag:
+        description: 'Tag to use for Docker image'
+        type: string
+        required: false
+      return-image-for-tag:
+        description: 'Make workflow to return image for the passed tag without building or tagging anything'
+        type: string
+        required: false
+        default: ''
+      force-rebuild:
+        description: 'Rebuild the kernel image even if it already exists. No-op if `return-image-for-tag` is set'
+        type: boolean
+        required: false
+        default: false
+    outputs:
+      # image:
+      #   description: 'vm-kernel Docker image'
+      #   value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image }}
+      image_amd64:
+        description: 'vm-kernel Docker image for amd64'
+        value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image_amd64 }}
+      image_arm64:
+        description: 'vm-kernel Docker image for arm64'
+        value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image_arm64 }}
 
 env:
   VM_KERNEL_IMAGE: "neondatabase/vm-kernel"
@@ -127,10 +133,14 @@ jobs:
           docker push ${VM_KERNEL_IMAGE}:${NEW_TAG}
 
   build-vm-kernel-image:
+    # TODO: fix setup-build-vm-kernel-image to support multiarch
     needs: setup-build-vm-kernel-image
     if: needs.setup-build-vm-kernel-image.outputs.image == ''
     outputs:
-      image: ${{ steps.get-tags.outputs.canonical }}@${{ steps.build-linux-kernel.outputs.digest }}
+      # image for backward compatibility
+      # image: ${{ steps.get-tags-amd64.outputs.canonical }}@${{ steps.build-linux-kernel-amd64.outputs.digest }}
+      image_amd64: ${{ steps.get-tags-amd64.outputs.canonical }}@${{ steps.build-linux-kernel-amd64.outputs.digest }}
+      image_arm64: ${{ steps.get-tags-arm64.outputs.canonical }}@${{ steps.build-linux-kernel-arm64.outputs.digest }}
 
     runs-on: [ self-hosted, gen3, large ]
     steps:
@@ -162,8 +172,8 @@ jobs:
 
           echo VM_KERNEL_VERSION=$kernel_version >> $GITHUB_OUTPUT
 
-      - name: get docker tags
-        id: get-tags
+      - name: get docker tags amd64
+        id: get-tags-amd64
         env:
           KERNEL_VERSION_TAG: ${{ inputs.tag || steps.get-kernel-version.outputs.VM_KERNEL_VERSION }}
           CACHE_TAG: ${{ needs.setup-build-vm-kernel-image.outputs.kernel-cache-tag }}
@@ -179,8 +189,25 @@ jobs:
           # Choose one of them as a "canonical" tag and use it to construct the job output (along with a digest provided by `docker/build-push-action@v6`).
           echo "canonical=${VM_KERNEL_IMAGE}:${GITHUB_RUN_ID}" >> $GITHUB_OUTPUT
 
-      - name: build linux kernel
-        id: build-linux-kernel
+      - name: get docker tags arm64
+        id: get-tags-arm64
+        env:
+          KERNEL_VERSION_TAG: ${{ inputs.tag || steps.get-kernel-version.outputs.VM_KERNEL_VERSION }}
+          CACHE_TAG: ${{ needs.setup-build-vm-kernel-image.outputs.kernel-cache-tag }}
+        run: |
+            # A comma-separated list of tags
+            TAGS="${VM_KERNEL_IMAGE}-arm64:${KERNEL_VERSION_TAG}"
+            TAGS="${VM_KERNEL_IMAGE}-arm64:${CACHE_TAG},${TAGS}"
+            TAGS="${VM_KERNEL_IMAGE}-arm64:${GITHUB_RUN_ID},${TAGS}"
+
+            echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+
+            # `docker/build-push-action@v6` returns all ${TAGS} in metadata ("image.name" field), so it can't be used a image name right away.
+            # Choose one of them as a "canonical" tag and use it to construct the job output (along with a digest provided by `docker/build-push-action@v6`).
+            echo "canonical=${VM_KERNEL_IMAGE}-arm64:${GITHUB_RUN_ID}" >> $GITHUB_OUTPUT 
+
+      - name: build linux kernel amd64
+        id: build-linux-kernel-amd64
         uses: docker/build-push-action@v6
         with:
           build-args: KERNEL_VERSION=${{ steps.get-kernel-version.outputs.VM_KERNEL_VERSION }}
@@ -195,4 +222,23 @@ jobs:
           file: neonvm-kernel/Dockerfile.kernel-builder
           cache-from: type=registry,ref=cache.neon.build/vm-kernel:cache
           cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/vm-kernel:cache,mode=max' || '' }}
-          tags: ${{ steps.get-tags.outputs.tags }}
+          tags: ${{ steps.get-tags-amd64.outputs.tags }}
+
+      - name: build linux kernel arm64
+        id: build-linux-kernel-arm64
+        uses: docker/build-push-action@v6
+        with:
+            build-args: KERNEL_VERSION=${{ steps.get-kernel-version.outputs.VM_KERNEL_VERSION }}
+            context: neonvm-kernel
+            # kernel is built using crosscompilation, so it doesn't matter what runner arch we have
+            platforms: linux/amd64
+            # neonvm-kernel/Dockerfile.kernel-builder has different targets for different architectures
+            # so we need to specify the target explicitly
+            target: kernel_arm64
+            # Push kernel image only for scheduled builds or if workflow_dispatch/workflow_call input is true
+            push: true
+            pull: true
+            file: neonvm-kernel/Dockerfile.kernel-builder
+            cache-from: type=registry,ref=cache.neon.build/vm-kernel:cache
+            cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/vm-kernel:cache,mode=max' || '' }}
+            tags: ${{ steps.get-tags-arm64.outputs.tags }}