update benchmarks

Author: leibale

Diff for: benchmark/.gitignore

@@ -0,0 +1 @@
+*.js.json

Diff for: benchmark/lib/defaults.yml

@@ -0,0 +1,32 @@
+version: 0.1
+
+remote:
+  - type: oss-standalone
+  - setup: redis-small
+
+setups:
+  - oss-standalone
+
+spec:
+  setups:
+  - name: oss-standalone
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    resources:
+      requests:
+        cpus: "1"
+        memory: "10g"
+
+exporter:
+  output_path: "./*.js.json"
+  redistimeseries:
+    timemetric: "$.timestamp"
+    metrics:
+      - "$.p0"
+      - "$.p50"
+      - "$.p95"
+      - "$.p99"
+      - "$.p100"
+      - "$.operationsPerSecond"

Diff for: benchmark/lib/index.js

@@ -4,6 +4,7 @@ import { promises as fs } from 'fs';
 import { fork } from 'child_process';
 import { URL, fileURLToPath } from 'url';
 import { once } from 'events';
+import { extname } from 'path';
 
 async function getPathChoices() {
     const dirents = await fs.readdir(new URL('.', import.meta.url), {
@@ -32,11 +33,23 @@ async function getName() {
 }
 
 const runnerPath = fileURLToPath(new URL('runner.js', import.meta.url)),
-    path = new URL(`${await getName()}/`, import.meta.url),
-    metadata = await import(new URL('index.js', path));
+    path = new URL(`${await getName()}/`, import.meta.url);
+
+async function getMetadata() {
+    try {
+        return await import(new URL('index.js', path));
+    } catch (err) {
+        if (err.code === 'ERR_MODULE_NOT_FOUND') return;
+
+        throw err;
+    }
+}
+
+const metadata = await getMetadata(),
+    timestamp = Date.now();
 
 for (const file of await fs.readdir(path)) {
-    if (file === 'index.js') continue;
+    if (file === 'index.js' || extname(file) !== '.js') continue;
 
     const benchmarkProcess = fork(runnerPath, [
         ...argv,
@@ -45,6 +58,9 @@ for (const file of await fs.readdir(path)) {
     ]);
 
     await once(benchmarkProcess, 'message');
-    benchmarkProcess.send(metadata);
+    benchmarkProcess.send({
+        metadata,
+        timestamp
+    });
     await once(benchmarkProcess, 'close');
 }

Diff for: benchmark/lib/ping/ioredis.js

@@ -0,0 +1,19 @@
+import Redis from 'ioredis';
+
+export default async (host) => {
+    const client = new Redis({
+        host,
+        lazyConnect: true
+    });
+
+    await client.connect();
+
+    return {
+        benchmark() {
+            return client.ping();
+        },
+        teardown() {
+            return client.disconnect();
+        }
+    }
+};

Diff for: benchmark/lib/ping/ping.yml

@@ -0,0 +1,9 @@
+name: "ping"
+
+clientconfig:
+  - command: |
+      npm install -ws
+      npm run build:tests-tools
+      cd benchmark
+      npm install
+      npm run start -- --name ping --redis-server-host ${server_private_ip}

Diff for: benchmark/lib/ping/v3.js

@@ -0,0 +1,21 @@
+import { createClient } from 'redis-v3';
+import { once } from 'events';
+import { promisify } from 'util';
+
+export default async (host) => {
+    const client = createClient({ host }),
+        pingAsync = promisify(client.ping).bind(client),
+        quitAsync = promisify(client.quit).bind(client);
+
+    await once(client, 'connect');
+
+    return {
+        benchmark() {
+            return pingAsync();
+        },
+        teardown() {
+            return quitAsync();
+        }
+    };
+
+};

Diff for: benchmark/lib/ping/v4.js

@@ -0,0 +1,16 @@
+import { createClient } from '@node-redis/client';
+
+export default async (host) => {
+    const client = createClient({ host });
+
+    await client.connect();
+
+    return {
+        benchmark() {
+            return client.ping();
+        },
+        teardown() {
+            return client.disconnect();
+        }
+    };
+};

Diff for: benchmark/lib/runner.js

@@ -1,10 +1,11 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import { basename } from 'path';
+import { promises as fs } from 'fs';
 import * as hdr from 'hdr-histogram-js';
 hdr.initWebAssemblySync();
 
-const { path, times, concurrency } = yargs(hideBin(process.argv))
+const { path, times, concurrency, 'redis-server-host': host } = yargs(hideBin(process.argv))
     .option('path', {
         type: 'string',
         demandOption: true
@@ -19,49 +20,40 @@ const { path, times, concurrency } = yargs(hideBin(process.argv))
         default: 100,
         demandOption: true
     })
+    .option('redis-server-host', {
+        type: 'string'
+    })
     .parseSync();
 
-async function setup() {
-    const module = await import(path);
-    await module.setup();
-    return module;
-}
-
-function getMetadata() {
-    return new Promise(resolve => {
-        process.once('message', resolve);
-        process.send('ready');
-    });
-}
-
-const [ { benchmark, teardown }, metadata ] = await Promise.all([
-    setup(),
-    getMetadata()
-]);
+const [ { metadata, timestamp }, module ] = await Promise.all([
+        new Promise(resolve => {
+            process.once('message', resolve);
+            process.send('ready');
+        }),
+        import(path)
+    ]),
+    { benchmark, teardown } = await module.default(host, metadata);
 
 async function run(times) {
     return new Promise(resolve => {
         const histogram = hdr.build({ useWebAssembly: true });
         let num = 0,
             inProgress = 0;
 
-        function run() {
+        async function run() {
             ++inProgress;
             ++num;
 
             const start = process.hrtime.bigint();
-            benchmark(metadata)
-                .catch(err => console.error(err))
-                .finally(() => {
-                    histogram.recordValue(Number(process.hrtime.bigint() - start));
-                    --inProgress;
+            await benchmark(metadata);
+            histogram.recordValue(Number(process.hrtime.bigint() - start));
+            --inProgress;
 
-                    if (num < times) {
-                        run();
-                    } else if (inProgress === 0) {
-                        resolve(histogram);
-                    }
-                });
+            if (num < times) {
+                run();
+            } else if (inProgress === 0) {
+                resolve(histogram);
+            }
         }
 
         const toInitiate = Math.min(concurrency, times);
@@ -75,8 +67,20 @@ async function run(times) {
 await run(Math.min(times * 0.1, 10_000));
 
 // benchmark
-const histogram = await run(times);
+const benchmarkStart = process.hrtime.bigint(),
+    histogram = await run(times),
+    benchmarkNanoseconds = process.hrtime.bigint() - benchmarkStart,
+    json = {
+        timestamp,
+        operationsPerSecond: times / Number(benchmarkNanoseconds) * 1_000_000_000,
+        p0: histogram.getValueAtPercentile(0),
+        p50: histogram.getValueAtPercentile(50),
+        p95: histogram.getValueAtPercentile(95),
+        p99: histogram.getValueAtPercentile(99),
+        p100: histogram.getValueAtPercentile(100)
+    };
 console.log(`[${basename(path)}]:`);
-console.table(histogram.toJSON());
+console.table(json);
+await fs.writeFile(`${path}.json`, JSON.stringify(json));
 
 await teardown();

Diff for: benchmark/lib/set-get-delete-string/1KB.yml

@@ -0,0 +1,9 @@
+name: "set-get-delete-string-1KB"
+
+clientconfig:
+  - command: |
+      npm install -ws
+      npm run build:tests-tools
+      cd benchmark
+      npm install
+      npm run start -- --name set-get-delete-string --size 1024 --redis-server-host ${server_private_ip}

Diff for: benchmark/lib/set-get-delete-string/1MB.yml

@@ -0,0 +1,9 @@
+name: "set-get-delete-string-1MB"
+
+clientconfig:
+  - command: |
+      npm install -ws
+      npm run build:tests-tools
+      cd benchmark
+      npm install
+      npm run start -- --name set-get-delete-string --size 1048576 --redis-server-host ${server_private_ip}

Diff for: benchmark/lib/set-get-delete-string/8B.yml

@@ -0,0 +1,9 @@
+name: "set-get-delete-string-8B"
+
+clientconfig:
+  - command: |
+      npm install -ws
+      npm run build:tests-tools
+      cd benchmark
+      npm install
+      npm run start -- --name set-get-delete-string --size 8 --redis-server-host ${server_private_ip}

Diff for: benchmark/lib/set-get-delete-string/ioredis.js

@@ -1,19 +1,23 @@
 import Redis from 'ioredis';
 
-const client = new Redis({ lazyConnect: true });
+export default async (host, { randomString }) => {
+    const client = new Redis({
+        host,
+        lazyConnect: true
+    });
 
-export function setup() {
-    return client.connect();
-}
+    await client.connect();
 
-export function benchmark({ randomString }) {
-    return Promise.all([
-        client.set(randomString, randomString),
-        client.get(randomString),
-        client.del(randomString)
-    ]);
-}
-
-export function teardown() {
-    return client.disconnect();
-}
+    return {
+        benchmark() {
+            return Promise.all([
+                client.set(randomString, randomString),
+                client.get(randomString),
+                client.del(randomString)
+            ]);
+        },
+        teardown() {
+            return client.disconnect();
+        }
+    }
+};

Diff for: benchmark/lib/set-get-delete-string/local.js

@@ -0,0 +1,27 @@
+import { createClient } from 'redis-v3';
+import { once } from 'events';
+import { promisify } from 'util';
+
+export default async (host, { randomString }) => {
+    const client = createClient({ host }),
+        setAsync = promisify(client.set).bind(client),
+        getAsync = promisify(client.get).bind(client),
+        delAsync = promisify(client.del).bind(client),
+        quitAsync = promisify(client.quit).bind(client);
+
+    await once(client, 'connect');
+
+    return {
+        benchmark() {
+            return Promise.all([
+                setAsync(randomString, randomString),
+                getAsync(randomString),
+                delAsync(randomString)
+            ]);
+        },
+        teardown() {
+            return quitAsync();
+        }
+    };
+
+};