diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml new file mode 100644 index 000000000..7920a2fc5 --- /dev/null +++ b/.github/workflows/analysis.yml @@ -0,0 +1,67 @@ +name: Analysis + +on: + push: + branches: + - master + - develop + - 0.2.x + pull_request: + types: + - opened + - synchronize + - reopened + workflow_dispatch: + inputs: + pr: + description: "Pull request#" + required: false + +jobs: + static: + runs-on: ubuntu-latest + name: Static code analysis + steps: + - name: Check out Git repository + uses: actions/checkout@v2 + - name: Check out head + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + if: github.event_name == 'pull_request' + - name: Check out PR + run: | + git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ + origin pull/${{ github.event.inputs.pr }}/head:the-pr && git checkout the-pr + if: github.event.inputs.pr != '' + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache SonarCloud packages + uses: actions/cache@v1 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + - name: Cache Maven packages + uses: actions/cache@v1 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + - name: Check style and spell + uses: zhicwu/checkstyle-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: 'github-pr-check' + # added,diff_context,file,nofilter + filter_mode: 'added' + if: github.event_name == 'pull_request' || github.event.inputs.pr != '' + - name: Build and analyze + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: | + find . -type f -name "log4j.*" -exec rm -fv '{}' \; + mvn -q --batch-mode -Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index aa3455c0a..d45e36794 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,6 +1,11 @@ name: Benchmark on: + pull_request: + types: + - opened + - synchronize + - reopened workflow_dispatch: inputs: clickhouse: @@ -28,7 +33,52 @@ on: required: false jobs: - benchmark: + benchmark-pull-request: + runs-on: ubuntu-latest + name: Benchmark pull request + if: github.event_name == 'pull_request' + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Extract version + id: version + run: | + echo ::set-output name=value::$(grep '' pom.xml | sed -e 's|.*>\(.*\)<.*|\1|') + - name: Update version and group id + run: | + find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ steps.version.outputs.value }}|g' \ + -e 's|^\( \).*\(\)$|\1${{ steps.version.outputs.value }}\2|' \ + -e 's|${parent.groupId}|tech.clickhouse|g' -e 's|.*argLine.*timezone=.*||g' '{}' \; + find . -type f -name "log4j.*" -exec rm -fv '{}' \; + - name: Build project + run: | + mvn --batch-mode --update-snapshots -q -DskipTests install + cd clickhouse-benchmark + mvn --batch-mode --update-snapshots install + java -jar target/benchmarks.jar -rf text -p client=clickhouse-jdbc Basic + echo "BENCHMARK_REPORT<> $GITHUB_ENV + cat jmh-result.text >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + cd - + - name: Comment PR + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + if (process.env.BENCHMARK_REPORT) { + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '```\n' + process.env.BENCHMARK_REPORT + '\n```' + }); + } + + benchmark-on-demand: runs-on: ubuntu-latest name: Benchmark on demand if: github.event_name == 'workflow_dispatch' @@ -47,9 +97,10 @@ jobs: continue-on-error: true - name: Update version and group id run: | - find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.driver }}|g' \ - -e 's|${parent.groupId}|tech.clickhouse|g' '{}' \; - sed -i -e 's|^\( \).*\(\)$|\1${{ github.event.inputs.driver }}\2|' pom.xml + find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \ + -e 's|^\( \).*\(\)$|\1${{ github.event.inputs.driver }}\2|' \ + -e 's|${parent.groupId}|tech.clickhouse|g' -e 's|.*argLine.*timezone=.*||g' '{}' \; + find . -type f -name "log4j.*" -exec rm -fv '{}' \; continue-on-error: true - name: Install driver as needed run: mvn --batch-mode --update-snapshots -q -DskipTests install @@ -58,8 +109,7 @@ jobs: - name: Build project run: | cd clickhouse-benchmark - mvn --batch-mode --update-snapshots -Drevision=${{ github.event.inputs.driver }} \ - -DclickhouseVersion=${{ github.event.inputs.clickhouse }} install + mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ github.event.inputs.clickhouse }} install java -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt echo "BENCHMARK_REPORT<> $GITHUB_ENV tail -n +$(grep -n '^REMEMBER:' output.txt | tail -1 | awk -F: '{print $1+6}') output.txt | head -n -2 | grep -v ':ยท' >> $GITHUB_ENV @@ -94,7 +144,7 @@ jobs: echo ::set-output name=hash::$(git log --pretty=format:'%H' -n 1) id: commit continue-on-error: true - - name: 'Comment PR' + - name: Comment PR uses: actions/github-script@v3 if: github.event.inputs.pr != '' env: @@ -117,8 +167,9 @@ jobs: const flag = result === 'success' ? ':green_circle:' : (result === 'failure' ? ':red_circle:' : ':yellow_circle:'); - const msg = `${flag} [benchmark](${compareUrl}) using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VRESION}]: [${result}](${buildUrl})` - + '
\nExpand to see details...\n\n```\n' - + process.env.BENCHMARK_REPORT || '' - + '\n```\n
'; + let msg = `${flag} [benchmark](${compareUrl}) using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VRESION}]: [${result}](${buildUrl})`; + if (result === 'success') { + msg = msg + '
\nExpand to see details...\n\n```\n' + + process.env.BENCHMARK_REPORT || '' + '\n```\n
'; + } github.issues.createComment({ issue_number, owner, repo, body: msg }); diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bee23239a..f362b2ec8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,6 +21,12 @@ on: - "docs/**" - "**/CHANGELOG" + workflow_dispatch: + inputs: + pr: + description: "Pull request#" + required: false + jobs: build: runs-on: ubuntu-latest @@ -33,6 +39,11 @@ jobs: steps: - name: Check out Git repository uses: actions/checkout@v2 + - name: Check out PR + run: | + git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ + origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr + if: github.event.inputs.pr != '' - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v1 with: @@ -45,5 +56,25 @@ jobs: key: ${{ runner.os }}-build-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-build- + - name: Generate build properties + uses: actions/github-script@v3 + id: props + env: + CURRENT_VERSION: ${{ steps.version.outputs.value }} + with: + script: | + const timezones = [ + 'Asia/Chongqing', 'America/Los_Angeles', 'Etc/UTC', 'Europe/Berlin', 'Europe/Moscow' + ]; + // surprise me + return { + clickhouse: timezones[Math.floor(Math.random() * Math.floor(timezones.length))] || '', + java: timezones[Math.floor(Math.random() * Math.floor(timezones.length))] || '' + }; - name: Build with Maven - run: mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ matrix.clickhouse }} verify + run: | + find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; + mvn --batch-mode --update-snapshots \ + -DclickhouseVersion=${{ matrix.clickhouse }} \ + -DclickhouseTimezone=${{ fromJSON(steps.props.outputs.result).clickhouse }} \ + -Duser.timezone=${{ fromJSON(steps.props.outputs.result).java }} verify diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f158a1ad..e8b53f233 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,8 +20,12 @@ jobs: uses: actions/setup-java@v1 with: java-version: 8 - - run: find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' -e 's|${parent.groupId}|tech.clickhouse|g' '{}' \; - - run: find . -type f -name "log4j.properties" -exec sed -i -e 's|DEBUG|WARN|g' '{}' \; + - name: Update pom files and reduce logs + run: | + find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \ + -e 's|^\( \).*\(\)$|\1${{ github.event.inputs.version }}\2|' \ + -e 's|${parent.groupId}|tech.clickhouse|g' -e 's|.*argLine.*timezone=.*||g' '{}' \; + find . -type f -name "log4j.*" -exec rm -fv '{}' \; - name: Release Maven package uses: samuelmeuli/action-maven-publish@v1 with: @@ -40,4 +44,4 @@ jobs: title: "Release v${{ github.event.inputs.version }}" files: | LICENSE - target/clickhouse*.jar + **/target/clickhouse*.jar diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index f509ff779..86fef8a0d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -11,6 +11,14 @@ on: description: "Java version" required: true default: "8" + chTz: + description: "ClickHouse timezone" + required: true + default: "Asia/Chongqing" + javaTz: + description: "Java timezone" + required: true + default: "America/Los_Angeles" pr: description: "Pull request#" required: false @@ -66,7 +74,8 @@ jobs: - name: Check out commented PR if: steps.check.outputs.triggered == 'true' run: | - git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin pull/${{ fromJSON(steps.commented.outputs.result).pr }}/merge:merged-pr && git checkout merged-pr + git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ + origin pull/${{ fromJSON(steps.commented.outputs.result).pr }}/merge:merged-pr && git checkout merged-pr - name: Set up JDK if: steps.check.outputs.triggered == 'true' uses: actions/setup-java@v1 @@ -75,14 +84,19 @@ jobs: continue-on-error: true - name: Verify with Maven if: steps.check.outputs.triggered == 'true' - run: mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ fromJSON(steps.commented.outputs.result).clickhouse }} verify + run: | + find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; + mvn --batch-mode --update-snapshots \ + -DclickhouseVersion=${{ fromJSON(steps.commented.outputs.result).clickhouse }} \ + -DclickhouseTimezone=Asia/Chongqing \ + -Duser.timezone=America/Los_Angeles verify id: maven continue-on-error: true - name: Comment PR uses: actions/github-script@v3 if: steps.check.outputs.triggered == 'true' env: - CLICKHOUSE_VRESION: ${{ fromJSON(steps.commented.outputs.result).clickhouse }} + CLICKHOUSE_VERSION: ${{ fromJSON(steps.commented.outputs.result).clickhouse }} JAVA_VERSION: ${{ fromJSON(steps.commented.outputs.result).java }} PREV_STEP_RESULT: '${{ steps.maven.outcome }}' with: @@ -94,7 +108,7 @@ jobs: const flag = result === 'success' ? ':green_circle:' : (result === 'failure' ? ':red_circle:' : ':yellow_circle:'); - const msg = `${flag} verify using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VRESION}]: [${result}](${buildUrl})`; + const msg = `${flag} verify using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VERSION}]: [${result}](${buildUrl})`; github.issues.createComment({ issue_number, owner, repo, body: msg }); verify-on-demand: @@ -114,15 +128,22 @@ jobs: java-version: ${{ github.event.inputs.java }} continue-on-error: true - name: Verify with Maven - run: mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ github.event.inputs.clickhouse }} verify + run: | + find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; + mvn --batch-mode --update-snapshots \ + -DclickhouseVersion=${{ github.event.inputs.clickhouse }} \ + -DclickhouseTimezone=${{ github.event.inputs.chTz }} \ + -Duser.timezone=${{ github.event.inputs.javaTz }} verify id: maven continue-on-error: true - - name: 'Comment PR' + - name: Comment PR uses: actions/github-script@v3 if: github.event.inputs.pr != '' env: PR_NO: ${{ github.event.inputs.pr }} - CLICKHOUSE_VRESION: ${{ github.event.inputs.clickhouse }} + CLICKHOUSE_TIMEZONE: ${{ github.event.inputs.chTz }} + CLICKHOUSE_VERSION: ${{ github.event.inputs.clickhouse }} + JAVA_TIMEZONE: ${{ github.event.inputs.javaTz }} JAVA_VERSION: ${{ github.event.inputs.java }} PREV_STEP_RESULT: '${{ steps.maven.outcome }}' with: @@ -135,5 +156,5 @@ jobs: const flag = result === 'success' ? ':green_circle:' : (result === 'failure' ? ':red_circle:' : ':yellow_circle:'); - const msg = `${flag} verify using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VRESION}]: [${result}](${buildUrl})`; + const msg = `${flag} verify using JDK [${process.env.JAVA_VERSION}, timezone=${process.env.JAVA_TIMEZONE}] and ClickHouse [${process.env.CLICKHOUSE_VERSION}, timezone=${process.env.CLICKHOUSE_TIMEZONE}]: [${result}](${buildUrl})`; github.issues.createComment({ issue_number, owner, repo, body: msg });