Skip to content

Commit

Permalink
chore(ci): add license for cluster test (#15995)
Browse files Browse the repository at this point in the history
  • Loading branch information
everpcpc authored Jul 9, 2024
1 parent b5feb12 commit 2005750
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 67 deletions.
112 changes: 47 additions & 65 deletions .github/workflows/reuse.linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ env:

jobs:
check:
runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -32,7 +32,7 @@ jobs:
github_token: ${{ github.token }}

license:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
outputs:
license: ${{ steps.license.outputs.license }}
steps:
Expand Down Expand Up @@ -128,7 +128,7 @@ jobs:
artifacts: query

build_musl:
runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
strategy:
fail-fast: false
matrix:
Expand All @@ -148,7 +148,7 @@ jobs:
artifacts: query

test_unit:
runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -158,72 +158,78 @@ jobs:
timeout-minutes: 60

test_metactl:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_metactl
timeout-minutes: 10

test_compat_meta_query:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_compat_meta_query
timeout-minutes: 10

test_compat_fuse:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_compat_fuse
timeout-minutes: 20

test_compat_meta_meta:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_compat_meta_meta
timeout-minutes: 20

test_logs:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_logs
timeout-minutes: 20

test_meta_cluster:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_meta_cluster
timeout-minutes: 10

test_stateless_standalone:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_stateless_standalone_linux
timeout-minutes: 15

test_stateless_cluster:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check, license]
steps:
- uses: actions/checkout@v4
- name: Decode license
id: license
run: |
echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
- uses: ./.github/actions/test_stateless_cluster_linux
env:
QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
timeout-minutes: 15

test_stateful_standalone:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_stateful_standalone_linux
Expand All @@ -235,11 +241,17 @@ jobs:
name: test-stateful-standalone-linux

test_stateful_cluster:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check, license]
steps:
- uses: actions/checkout@v4
- name: Decode license
id: license
run: |
echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
- uses: ./.github/actions/test_stateful_cluster_linux
env:
QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
timeout-minutes: 15
- name: Upload failure
if: failure()
Expand All @@ -249,16 +261,16 @@ jobs:

test_stateful_large_data:
if: contains(github.event.pull_request.labels.*.name, 'ci-largedata')
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_stateful_large_data
timeout-minutes: 60

test_stateful_iceberg_rest:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [ build, check ]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: [build, check]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/test_stateful_iceberg_rest_standalone
Expand All @@ -279,8 +291,8 @@ jobs:
# continue-on-error: true

test_ee_standalone:
needs: [ build, check, license ]
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [build, check, license]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
if: needs.license.outputs.license
steps:
- uses: actions/checkout@v4
Expand All @@ -299,8 +311,8 @@ jobs:
name: test-stateful-standalone-linux

test_ee_standalone_background:
needs: [ build, check, license ]
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [build, check, license]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
if: needs.license.outputs.license
steps:
- uses: actions/checkout@v4
Expand All @@ -320,8 +332,8 @@ jobs:
name: test-stateful-standalone-linux

test_ee_management_mode:
needs: [ build, check, license ]
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
needs: [build, check, license]
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
if: needs.license.outputs.license
steps:
- uses: actions/checkout@v4
Expand All @@ -340,38 +352,8 @@ jobs:
with:
name: test-ee-management-mode-linux

sqllogic_ee:
runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
if: needs.license.outputs.license
needs: [ build, check, license ]
strategy:
fail-fast: false
matrix:
format:
- "parquet"
- "native"
steps:
- uses: actions/checkout@v4
- name: Decode license
id: license
run: |
echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
- uses: ./.github/actions/test_ee_sqllogic_standalone_linux
timeout-minutes: 15
env:
QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
with:
dirs: ee
handlers: mysql,http
storage-format: ${{ matrix.format }}
- name: Upload failure
if: failure()
uses: ./.github/actions/artifact_failure
with:
name: test-ee-sqllogic-standalone-ee-${{ matrix.format }}

sqllogic:
needs: [ build, check ]
needs: [build, check]
uses: ./.github/workflows/reuse.sqllogic.yml
secrets: inherit
with:
Expand Down
58 changes: 58 additions & 0 deletions .github/workflows/reuse.sqllogic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,28 @@ env:
RUNNER_PROVIDER: ${{ inputs.runner_provider }}

jobs:
license:
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
outputs:
license: ${{ steps.license.outputs.license }}
steps:
- uses: actions/checkout@v4
- name: Get License from S3
if: inputs.runner_provider == 'aws'
run: |
aws s3 cp s3://databend-ci/misc/license.key license.key
aws s3 cp s3://databend-ci/misc/license.json license.json
- name: Get License from GCS
if: inputs.runner_provider == 'gcp'
run: |
gcloud storage cp gs://databend-ci/misc/license.key license.key
gcloud storage cp gs://databend-ci/misc/license.json license.json
- name: Output License
id: license
run: |
echo "license=$(cat license.key | base64 -w 0)" >> $GITHUB_OUTPUT
cat license.json
management_mode:
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
steps:
Expand Down Expand Up @@ -128,6 +150,7 @@ jobs:

cluster:
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: license
strategy:
fail-fast: false
matrix:
Expand All @@ -145,7 +168,13 @@ jobs:
- "http"
steps:
- uses: actions/checkout@v4
- name: Decode license
id: license
run: |
echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
- uses: ./.github/actions/test_sqllogic_cluster_linux
env:
QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
timeout-minutes: 15
with:
dirs: ${{ matrix.dirs }}
Expand Down Expand Up @@ -201,3 +230,32 @@ jobs:
uses: ./.github/actions/artifact_failure
with:
name: test-sqllogic-standalone-no-table-meta-cache-${{ matrix.dirs }}-${{ matrix.handler }}

ee:
runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
needs: license
strategy:
fail-fast: false
matrix:
format:
- "parquet"
- "native"
steps:
- uses: actions/checkout@v4
- name: Decode license
id: license
run: |
echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
- uses: ./.github/actions/test_ee_sqllogic_standalone_linux
timeout-minutes: 15
env:
QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
with:
dirs: ee
handlers: mysql,http
storage-format: ${{ matrix.format }}
- name: Upload failure
if: failure()
uses: ./.github/actions/artifact_failure
with:
name: test-ee-sqllogic-standalone-ee-${{ matrix.format }}
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ members = [
"src/tests/sqlsmith",
]

[workspace.dependencies]
# Workspace dependencies
[workspace.dependencies]
databend-common-arrow = { path = "src/common/arrow" }
databend-common-ast = { path = "src/query/ast" }
databend-common-async-functions = { path = "src/query/async_functions" }
Expand Down
2 changes: 1 addition & 1 deletion clippy.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,4 @@ disallowed-macros = [
avoid-breaking-exported-api = true
too-many-arguments-threshold = 10
upper-case-acronyms-aggressive = false
enum-variant-size-threshold = 200
enum-variant-size-threshold = 200

0 comments on commit 2005750

Please sign in to comment.