feat(ci): init

.github/ci-test.yml

@@ -0,0 +1,70 @@
+name: "[CI]"
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**.sh"
+      - "**.xml"
+      - "helm-chart/**"
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install xmllint
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libxml2-utils
+
+      - name: Install shfmt
+        run: |
+          curl -sSLo shfmt https://github.com/mvdan/sh/releases/download/v3.8.0/shfmt_v3.8.0_linux_amd64 &&
+          chmod +x shfmt &&
+          sudo mv shfmt /usr/local/bin/shfmt
+
+      - name: Lint Shell Scripts
+        run: |
+          find . -name '*.sh' -print0 | xargs -0 shfmt -d -i 2
+
+      - name: Lint XML Files
+        run: |
+          find . -name '*.xml' -print0 | xargs -0 xmllint --noout
+
+      - name: Checkout gravitino-playground
+        uses: actions/checkout@v4
+
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        with:
+          version: "2.14.2"
+
+      - name: Deploy
+        id: deploy
+        timeout-minutes: 40
+        run: |
+          sudo curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /usr/local/bin/yq
+          sudo chmod +x /usr/local/bin/yq
+          cd ${{ github.workspace }}/gravitino-playground
+          nohup ./playground.sh docker start -y > /tmp/playground.log 2>&1 &
+          # wait for gravitino trino ready to use
+          i=0
+          while [[ ! $(curl -k http://127.0.0.1:8090) || ! $(curl -k http://127.0.0.1:18080/v1/info) && $i -le 300 ]]; do
+          sleep 5
+          i=$(expr $i + 1)
+          done
+          docker ps
+          if [[ $(curl -k http://127.0.0.1:8090) && $(curl -k http://127.0.0.1:18080/v1/info) ]]; then
+            echo "gravitino and trino are ready to use"
+          else
+            echo "gravitino or trino not ready"
+            exit 1
+          fi
+
+      - name: Test sql
+        id: test-sql
+        timeout-minutes: 40
+        run: |
+          cd ${{ github.workspace }}/gravitino-playground/test
+          bash -x ./runSQLOnPlayground.sh

.github/workflows/add-to-project.yml

@@ -0,0 +1,16 @@
+name: Add issue to project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/datastrato/projects/1
+          github-token: ${{ secrets.ADD_ISSUE_TO_PROJECT }}

.github/workflows/auto-cherry-pick.yml

@@ -0,0 +1,68 @@
+name: Automatically cherry-pick merged PR to different branches
+
+on:
+  pull_request_target:
+    branches:
+      - main
+    types: ["closed"]
+
+jobs:
+  cherry_pick_branch_0_3:
+    runs-on: ubuntu-latest
+    name: Cherry pick into branch_0.3
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.3') && github.event.pull_request.merged == true }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Cherry pick into branch-0.3
+        uses: carloscastrojumo/[email protected]
+        with:
+          token: ${{ secrets.BOT_TOKEN }}
+          branch: branch-0.3
+          labels: |
+            cherry-pick
+          reviewers: |
+            jerryshao
+
+  cherry_pick_branch_0_4:
+    runs-on: ubuntu-latest
+    name: Cherry pick into branch_0.4
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.4') && github.event.pull_request.merged == true }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Cherry pick into branch-0.4
+        uses: carloscastrojumo/[email protected]
+        with:
+          token: ${{ secrets.BOT_TOKEN }}
+          branch: branch-0.4
+          labels: |
+            cherry-pick
+          reviewers: |
+            jerryshao
+
+  cherry_pick_branch_0_5:
+    runs-on: ubuntu-latest
+    name: Cherry pick into branch_0.5
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.5') && github.event.pull_request.merged == true }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Cherry pick into branch-0.5
+        uses: carloscastrojumo/[email protected]
+        with:
+          token: ${{ secrets.BOT_TOKEN }}
+          branch: branch-0.5
+          labels: |
+            cherry-pick
+          reviewers: |
+            jerryshao
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/backend-integration-test.yml

@@ -0,0 +1,115 @@
+name: Backend Integration Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - .github/**
+              - api/**
+              - bin/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - clients/filesystem-hadoop3/**
+              - clients/filesystem-hadoop3-runtime/**
+              - common/**
+              - conf/**
+              - core/**
+              - dev/**
+              - gradle/**
+              - integration-test/**
+              - integration-test-common/**
+              - meta/**
+              - server/**
+              - server-common/**
+              - trino-connector/**
+              - web/**
+              - docs/open-api/**
+              - build.gradle.kts
+              - gradle.properties
+              - gradlew
+              - setting.gradle.kts
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  # Integration test for AMD64 architecture
+  test-amd64-arch:
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        architecture: [linux/amd64]
+        java-version: [ 8, 11, 17 ]
+        test-mode: [ embedded, deploy ]
+        backend: [ mysql, h2]
+    env:
+      PLATFORM: ${{ matrix.architecture }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Check required command
+        run: |
+          dev/ci/check_commands.sh
+
+      - name: Package Gravitino
+        if : ${{ matrix.test-mode == 'deploy' }}
+        run: |
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Backend Integration Test
+        id: integrationTest
+        run: >
+          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
+          -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
+          -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
+          -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
+
+      - name: Upload integrate tests reports
+        uses: actions/upload-artifact@v3
+        if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
+        with:
+          name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
+          path: |
+            build/reports
+            integration-test/build/*.log
+            integration-test/build/*.tar
+            integration-test/build/trino-ci-container-log/hive/*.*
+            integration-test/build/trino-ci-container-log/hdfs/*.*
+            distribution/package/logs/gravitino-server.out
+            distribution/package/logs/gravitino-server.log
+            catalogs/**/*.log
+            catalogs/**/*.tar
+            distribution/**/*.log

.github/workflows/build.yml

@@ -0,0 +1,109 @@
+name: build
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "main", "branch-*" ]
+  pull_request:
+    branches: [ "main", "branch-*" ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            source_changes:
+              - .github/**
+              - api/**
+              - bin/**
+              - catalogs/**
+              - clients/client-java/**
+              - clients/client-java-runtime/**
+              - clients/filesystem-hadoop3/**
+              - clients/filesystem-hadoop3-runtime/**
+              - common/**
+              - conf/**
+              - core/**
+              - dev/**
+              - gradle/**
+              - integration-test/**
+              - meta/**
+              - server/**
+              - server-common/**
+              - spark-connector/**
+              - flink-connector/**
+              - trino-connector/**
+              - web/**
+              - docs/open-api/**
+              - build.gradle.kts
+              - gradle.properties
+              - gradlew
+              - setting.gradle.kts
+    outputs:
+      source_changes: ${{ steps.filter.outputs.source_changes }}
+
+  compile-check:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.source_changes != 'true'
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 8
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Build with Gradle
+        run: ./gradlew build -x test -PjdkVersion=8
+
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ 8, 11, 17 ]
+    timeout-minutes: 30
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Test publish to local
+        run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }}
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Build with Gradle
+        run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build
+
+      - name: Upload unit tests report
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: unit test report
+          path: |
+            build/reports
+            catalogs/**/*.log
+            catalogs/**/*.tar