Merge branch 'main' into orc2

databendlabs · Jan 15, 2025 · 1760f7f · 1760f7f
2 parents 871ee87 + 13d1dcd
commit 1760f7f
Show file tree

Hide file tree

Showing 95 changed files with 1,192 additions and 454 deletions.
diff --git a/.github/actions/artifact_upload/action.yml b/.github/actions/artifact_upload/action.yml
@@ -41,9 +41,17 @@ runs:
       if: env.RUNNER_PROVIDER == 'aws'
       shell: bash
       run: |
+        echo "### artifacts 🚀" >> $GITHUB_STEP_SUMMARY
         aws s3 sync ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/ \
           s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ \
           --exclude "*" --include "databend-*" --no-progress
+        artifacts="meta,query,query.debug"
+        for artifact in ${artifacts//,/ }; do
+          if [[ -f ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-${artifact} ]]; then
+            url=$(aws s3 presign s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/databend-${artifact} --expires-in 21600)
+            echo "- [${artifact}](${url})" >> $GITHUB_STEP_SUMMARY
+          fi
+        done
 
     - name: Upload artifact to gcs
       if: env.RUNNER_PROVIDER == 'gcp'

diff --git a/.github/scripts/bump_version.js b/.github/scripts/bump_version.js
@@ -7,27 +7,122 @@ module.exports = async ({ github, context, core }) => {
 
   const { TYPE, TAG } = process.env;
 
-  const RE_TAG_STABLE = /^v(\d+)\.(\d+)\.(\d+)$/g;
-  const RE_TAG_NIGHTLY = /^v(\d+)\.(\d+)\.(\d+)-nightly$/g;
-  const RE_TAG_PATCH = /^v(\d+)\.(\d+)\.(\d+)-p(\d+)$/g;
+  const RE_TAG_STABLE = /^v(\d+)\.(\d+)\.(\d+)$/;
+  const RE_TAG_NIGHTLY = /^v(\d+)\.(\d+)\.(\d+)-nightly$/;
+  const RE_TAG_PATCH = /^v(\d+)\.(\d+)\.(\d+)-p(\d+)$/;
 
-  switch (TYPE) {
-    case "":
-    case "nightly": {
-      core.setOutput("sha", context.sha);
-      core.info(`Nightly release triggered by ${TAG} (${context.sha})`);
+  async function getPreviousNightlyRelease(github, context) {
+    const releases = await github.rest.repos.listReleases({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+    });
+    for (const release of releases.data) {
+      const ret = RE_TAG_NIGHTLY.exec(release.tag_name);
+      if (ret) {
+        return release.tag_name;
+      }
+    }
+  }
 
-      let previous = null;
+  function getNextNightlyRelease(previous) {
+    const nightly = RE_TAG_NIGHTLY.exec(previous);
+    if (nightly) {
+      const major = nightly[1];
+      const minor = nightly[2];
+      const patch = parseInt(nightly[3]) + 1;
+      return `v${major}.${minor}.${patch}-nightly`;
+    }
+  }
+
+  async function getPreviousStableRelease(github, context) {
+    let page = 1;
+    while (true) {
       const releases = await github.rest.repos.listReleases({
         owner: context.repo.owner,
         repo: context.repo.repo,
+        page,
       });
+      if (releases.data.length === 0) {
+        break;
+      }
+      page++;
       for (const release of releases.data) {
-        const result = RE_TAG_NIGHTLY.exec(release.tag_name);
-        if (result) {
-          previous = release.tag_name;
-          break;
+        const ret = RE_TAG_STABLE.exec(release.tag_name);
+        if (ret) {
+          return release.tag_name;
+        }
+      }
+    }
+  }
+
+  function getNextStableRelease() {
+    const nightly = RE_TAG_NIGHTLY.exec(TAG);
+    if (nightly) {
+      const major = nightly[1];
+      const minor = nightly[2];
+      const patch = nightly[3];
+      return `v${major}.${minor}.${patch}`;
+    }
+  }
+
+  async function getPreviousPatchRelease(github, context) {
+    let page = 1;
+    while (true) {
+      const releases = await github.rest.repos.listReleases({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        page,
+      });
+      if (releases.data.length === 0) {
+        break;
+      }
+      page++;
+      for (const release of releases.data) {
+        if (!release.tag_name.startsWith(TAG)) {
+          continue;
+        }
+        if (release.tag_name === TAG) {
+          // no previous patch release, use the previous stable release
+          return release.tag_name;
+        }
+        const ret = RE_TAG_PATCH.exec(release.tag_name);
+        if (!ret) {
+          core.warning(`Ignore invalid patch release ${release.tag_name}`);
+          continue;
         }
+        return release.tag_name;
+      }
+    }
+  }
+
+  function getNextPatchRelease(previous) {
+    const stable = RE_TAG_STABLE.exec(previous);
+    if (stable) {
+      const major = stable[1];
+      const minor = stable[2];
+      const patch = stable[3];
+      return `v${major}.${minor}.${patch}-p1`;
+    }
+    const version = RE_TAG_PATCH.exec(previous);
+    if (version) {
+      const major = version[1];
+      const minor = version[2];
+      const patch = version[3];
+      const pv = parseInt(version[4]) + 1;
+      return `v${major}.${minor}.${patch}-p${pv}`;
+    }
+  }
+
+  switch (TYPE) {
+    case "":
+    case "nightly": {
+      core.setOutput("sha", context.sha);
+      core.info(`Nightly release triggered by (${context.sha})`);
+
+      const previous = await getPreviousNightlyRelease(github, context);
+      if (!previous) {
+        core.setFailed(`No previous nightly release found, ignoring`);
+        return;
       }
       core.setOutput("previous", previous);
       core.info(`Nightly release with previous release: ${previous}`);
@@ -37,15 +132,11 @@ module.exports = async ({ github, context, core }) => {
         core.info(`Release create manually with tag ${TAG}`);
         return;
       }
-      const result = RE_TAG_NIGHTLY.exec(previous);
-      if (!result) {
-        core.setFailed(`The previous tag ${previous} is invalid.`);
+      const nextTag = getNextNightlyRelease(previous);
+      if (!nextTag) {
+        core.setFailed(`No next nightly release from ${previous}`);
         return;
       }
-      const major = result[1];
-      const minor = result[2];
-      const patch = (parseInt(result[3]) + 1).toString();
-      const nextTag = `v${major}.${minor}.${patch}-nightly`;
       core.setOutput("tag", nextTag);
       core.info(`Release create new nightly ${nextTag}`);
       return;
@@ -58,38 +149,15 @@ module.exports = async ({ github, context, core }) => {
         return;
       }
       core.info(`Stable release triggered by ${TAG} (${context.sha})`);
-      const result = RE_TAG_NIGHTLY.exec(TAG);
-      if (!result) {
-        core.setFailed(`The tag ${TAG} is invalid, ignoring`);
+      const nextTag = getNextStableRelease();
+      if (!nextTag) {
+        core.setFailed(`No stable release from ${TAG}`);
         return;
       }
-      const major = result[1];
-      const minor = result[2];
-      const patch = result[3];
-      const nextTag = `v${major}.${minor}.${patch}`;
       core.setOutput("tag", nextTag);
       core.info(`Stable release ${nextTag} from ${TAG}`);
 
-      let previous = null;
-      let page = 1;
-      while (true) {
-        const releases = await github.rest.repos.listReleases({
-          owner: context.repo.owner,
-          repo: context.repo.repo,
-          page,
-        });
-        if (releases.data.length === 0) {
-          break;
-        }
-        page++;
-        for (const release of releases.data) {
-          const ret = RE_TAG_STABLE.exec(release.tag_name);
-          if (ret) {
-            previous = release.tag_name;
-            break;
-          }
-        }
-      }
+      const previous = await getPreviousStableRelease(github, context);
       if (!previous) {
         core.setFailed(`No previous stable release found, ignoring`);
         return;
@@ -120,49 +188,21 @@ module.exports = async ({ github, context, core }) => {
         `Patch release triggered by ${TAG} (${branch.data.commit.sha})`
       );
 
-      let pv = 1;
-      let previous = null;
-      let page = 1;
-      while (true) {
-        const releases = await github.rest.repos.listReleases({
-          owner: context.repo.owner,
-          repo: context.repo.repo,
-          page,
-        });
-        if (releases.data.length === 0) {
-          break;
-        }
-        page++;
-        for (const release of releases.data) {
-          if (!release.tag_name.startsWith(TAG)) {
-            continue;
-          }
-          if (release.tag_name === TAG) {
-            previous = release.tag_name;
-            break;
-          }
-          const ret = RE_TAG_PATCH.exec(release.tag_name);
-          if (!ret) {
-            core.warning(`Ignore previous release ${release.tag_name}`);
-            continue;
-          }
-          pv = parseInt(result[4]) + 1;
-          previous = release.tag_name;
-        }
-      }
+      const previous = await getPreviousPatchRelease(github, context);
       if (!previous) {
-        core.setFailed(`No previous stable release found, ignoring`);
+        core.setFailed(`No previous patch release found, ignoring`);
         return;
       }
       core.setOutput("previous", previous);
       core.info(`Patch release with previous release: ${previous}`);
 
-      const major = result[1];
-      const minor = result[2];
-      const patch = result[3];
-      const nextTag = `v${major}.${minor}.${patch}-p${pv}`;
+      const nextTag = getNextPatchRelease(previous);
+      if (!nextTag) {
+        core.setFailed(`No next patch release from ${previous}`);
+        return;
+      }
       core.setOutput("tag", nextTag);
-      core.info(`Patch release ${nextTag} from ${TAG}`);
+      core.info(`Patch release ${nextTag} from ${previous}`);
       return;
     }
 

diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -95,7 +95,7 @@ jobs:
   #         artifacts: query
 
   test_unit:
-    runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"]
+    runs-on: [self-hosted, X64, Linux, 16c64g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
         with: