ClickHouse · zhicwu · Feb 23, 2021 · Feb 23, 2021
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,124 @@
+name: Benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      clickhouse:
+        description: "ClickHouse version"
+        required: true
+        default: "latest"
+      java:
+        description: "Java version"
+        required: true
+        default: "8"
+      baseline:
+        description: "Baseline to compare"
+        required: true
+        default: "0.2.6"
+      driver:
+        description: "Driver version"
+        required: true
+        default: "0.3.0-SNAPSHOT"
+      options:
+        description: "Benchmark options"
+        required: true
+        default: "-prof gc 'Query|Insertion'"
+      pr:
+        description: "Pull request#"
+        required: false
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    name: Benchmark on demand
+    if: github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Check out PR
+        run: |
+          git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \
+            origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr
+        if: github.event.inputs.pr != ''
+      - name: Set up JDK ${{ github.event.inputs.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ github.event.inputs.java }}
+        continue-on-error: true
+      - name: Update version and group id
+        run: |
+          find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.driver }}|g' \
+            -e 's|${parent.groupId}|tech.clickhouse|g' '{}' \;
+          sed -i -e 's|^\(    <version>\).*\(</version>\)$|\1${{ github.event.inputs.driver }}\2|' pom.xml
+        continue-on-error: true
+      - name: Install driver as needed
+        run: mvn --batch-mode --update-snapshots -q -DskipTests install
+        if: endsWith(github.event.inputs.driver, '-SNAPSHOT')
+        continue-on-error: true
+      - name: Build project
+        run: |
+          cd clickhouse-benchmark
+          mvn --batch-mode --update-snapshots -Drevision=${{ github.event.inputs.driver }} \
+            -DclickhouseVersion=${{ github.event.inputs.clickhouse }} install
+          java -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt
+          echo "BENCHMARK_REPORT<<EOF" >> $GITHUB_ENV
+          tail -n +$(grep -n '^REMEMBER:' output.txt | tail -1 | awk -F: '{print $1+6}') output.txt | head -n -2 | grep -v ':·' >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+          cd -
+        id: benchmark
+        continue-on-error: true
+      - name: Record benchmark results
+        run: |
+          mv -fv clickhouse-benchmark/jmh-result.json \
+            clickhouse-benchmark/docs/results/v${{ github.event.inputs.driver }}.json
+        if: github.event.inputs.pr == ''
+        continue-on-error: true
+      - name: Record pull request benchmark results
+        run: |
+          mv -fv clickhouse-benchmark/jmh-result.json \
+            clickhouse-benchmark/docs/results/pull-request_${{github.event.inputs.pr}}.json
+        if: github.event.inputs.pr != ''
+        continue-on-error: true
+      - name: Commit benchmark result
+        uses: zhicwu/add-and-commit@v7
+        with:
+          add: 'clickhouse-benchmark/docs/results/*.json'
+          author_name: zhicwu
+          author_email: [email protected]
+          branch: develop
+          message: 'Record benchmark results'
+          push: true
+        continue-on-error: true
+      - name: Get commit hash
+        run: |
+          echo ::set-output name=hash::$(git log --pretty=format:'%H' -n 1)
+        id: commit
+        continue-on-error: true
+      - name: 'Comment PR'
+        uses: actions/github-script@v3
+        if: github.event.inputs.pr != ''
+        env:
+          PR_NO: ${{ github.event.inputs.pr }}
+          COMMIT_HASH: ${{ steps.commit.outputs.hash }}
+          COMPARE_TO: ${{ github.event.inputs.baseline }}
+          CLICKHOUSE_VRESION: ${{ github.event.inputs.clickhouse }}
+          DRIVER_VRESION: ${{ github.event.inputs.driver }}
+          JAVA_VERSION: ${{ github.event.inputs.java }}
+          PREV_STEP_RESULT: '${{ steps.benchmark.outcome }}'
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const issue_number = process.env.PR_NO;
+            const { repo: { owner, repo }  } = context;
+            const result = process.env.PREV_STEP_RESULT;
+            const compareUrl = `http://jmh.morethan.io/?sources=https://raw.githubusercontent.com/${owner}/${repo}/develop/clickhouse-benchmark/docs/results/v${process.env.COMPARE_TO}.json,https://raw.githubusercontent.com/${owner}/${repo}/${process.env.COMMIT_HASH || 'develop'}/clickhouse-benchmark/docs/results/${issue_number ? 'pull-request_' + issue_number : 'v' + process.env.DRIVER_VERSION}.json`;
+            // const benchmarkUrl = `http://jmh.morethan.io/?source=https://raw.githubusercontent.com/${owner}/${repo}/${process.env.COMMIT_HASH || 'develop'}/clickhouse-benchmark/docs/results/${issue_number ? 'pull-request_' + issue_number : 'v' + process.env.DRIVER_VERSION}.json`;
+            const buildUrl = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}`;
+            const flag = result === 'success'
+              ? ':green_circle:'
+              : (result === 'failure' ? ':red_circle:' : ':yellow_circle:');
+            const msg = `${flag} [benchmark](${compareUrl}) using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VRESION}]: [${result}](${buildUrl})`
+              + '<details>\n<summary>Expand to see details...</summary>\n\n```\n'
+              + process.env.BENCHMARK_REPORT || ''
+              + '\n```\n</details>';
+            github.issues.createComment({ issue_number, owner, repo, body: msg });
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,7 @@
 
 # VSCode
 .vscode
+.factorypath
 
 # Eclipse
 .classpath

diff --git a/clickhouse-benchmark/docs/results/v0.2.4.json b/clickhouse-benchmark/docs/results/v0.2.4.json
diff --git a/clickhouse-benchmark/docs/results/v0.2.5.json b/clickhouse-benchmark/docs/results/v0.2.5.json
diff --git a/clickhouse-benchmark/docs/results/v0.2.6.json b/clickhouse-benchmark/docs/results/v0.2.6.json
diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml
@@ -0,0 +1,189 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>tech.clickhouse</groupId>
+    <artifactId>clickhouse-benchmark</artifactId>
+    <version>${revision}</version>
+    <packaging>jar</packaging>
+
+    <name>clickhouse-benchmark</name>
+
+    <properties>
+        <revision>0.3.0-SNAPSHOT</revision>
+        <clickhouse4j-driver.version>1.4.4</clickhouse4j-driver.version>
+        <native-driver.version>2.5.3</native-driver.version>
+        <mariadb-driver.version>2.7.2</mariadb-driver.version>
+        <mysql-driver.version>8.0.23</mysql-driver.version>
+        <native-driver.version>2.5.3</native-driver.version>
+        <testcontainers.version>1.15.2</testcontainers.version>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <jmh.version>1.27</jmh.version>
+        <javac.target>1.8</javac.target>
+        <shade.name>benchmarks</shade.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- JDBC drivers -->
+        <dependency>
+            <groupId>ru.yandex.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>${revision}</version>
+            <classifier>shaded</classifier>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>cc.blynk.clickhouse</groupId>
+            <artifactId>clickhouse4j</artifactId>
+            <version>${clickhouse4j-driver.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.mariadb.jdbc</groupId>
+            <artifactId>mariadb-java-client</artifactId>
+            <version>${mariadb-driver.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>${mysql-driver.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.github.housepower</groupId>
+            <artifactId>clickhouse-native-jdbc-shaded</artifactId>
+            <version>${native-driver.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>${testcontainers.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.0</version>
+                <configuration>
+                    <compilerVersion>${javac.target}</compilerVersion>
+                    <source>${javac.target}</source>
+                    <target>${javac.target}</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>${shade.name}</finalName>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.openjdk.jmh.Main</mainClass>
+                                </transformer>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-clean-plugin</artifactId>
+                    <version>2.5</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-deploy-plugin</artifactId>
+                    <version>2.8.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-install-plugin</artifactId>
+                    <version>2.5.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-jar-plugin</artifactId>
+                    <version>2.4</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-javadoc-plugin</artifactId>
+                    <version>2.9.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-resources-plugin</artifactId>
+                    <version>2.6</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-site-plugin</artifactId>
+                    <version>3.3</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-source-plugin</artifactId>
+                    <version>2.2.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>2.17</version>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+</project>
diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java
@@ -0,0 +1,34 @@
+package tech.clickhouse.benchmark;
+
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Collections;
+
+import org.openjdk.jmh.annotations.Benchmark;
+
+public class Basic extends JdbcBenchmark {
+    @Benchmark
+    public int selectOneRandomNumber(ClientState state) throws Throwable {
+        final int num = (int) (Math.random() * 1000);
+
+        try (Statement stmt = executeQuery(state, "select ? as n", num)) {
+            ResultSet rs = stmt.getResultSet();
+
+            rs.next();
+
+            if (num != rs.getInt(1)) {
+                throw new IllegalStateException();
+            }
+
+            return num;
+        }
+    }
+
+    @Benchmark
+    public int insertOneRandomNumber(ClientState state) throws Throwable {
+        final int num = (int) (Math.random() * 1000);
+
+        return executeInsert(state, "insert into test_insert(i) values(?)",
+                Collections.enumeration(Collections.singletonList(new Object[] { num })));
+    }
+}