Skip to content

Commit be90e62

Browse files
leibaleleibale
authored
Add support for T-Digest (#2214)
* wip * close #2216 - add support for TDIGEST.MERGESTORE and make compression optional on TDIGEST.CREATE * fix some tdigest commands, use bloom edge docker * fix index.ts * 2.4-RC2 (v2.4.1) * fix some commands and tests * clean code
1 parent 1c6d74f commit be90e62

33 files changed

+794
-20
lines changed

‎packages/bloom/lib/commands/index.ts

+10-8
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1-
import Bloom from './bloom';
2-
import CountMinSketch from './count-min-sketch';
3-
import Cuckoo from './cuckoo';
4-
import TopK from './top-k';
1+
import bf from './bloom';
2+
import cms from './count-min-sketch';
3+
import cf from './cuckoo';
4+
import tDigest from './t-digest';
5+
import topK from './top-k';
56

67
export default {
7-
bf: Bloom,
8-
cms: CountMinSketch,
9-
cf: Cuckoo,
10-
topK: TopK
8+
bf,
9+
cms,
10+
cf,
11+
tDigest,
12+
topK
1113
};

‎packages/bloom/lib/commands/t-digest/ADD.spec.ts

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { strict as assert } from 'assert';
2+
import testUtils, { GLOBAL } from '../../test-utils';
3+
import { transformArguments } from './ADD';
4+
5+
describe('TDIGEST.ADD', () => {
6+
it('transformArguments', () => {
7+
assert.deepEqual(
8+
transformArguments('key', [1, 2]),
9+
['TDIGEST.ADD', 'key', '1', '2']
10+
);
11+
});
12+
13+
testUtils.testWithClient('client.tDigest.add', async client => {
14+
const [ , reply ] = await Promise.all([
15+
client.tDigest.create('key'),
16+
client.tDigest.add('key', [1])
17+
]);
18+
19+
assert.equal(reply, 'OK');
20+
}, GLOBAL.SERVERS.OPEN);
21+
});

‎packages/bloom/lib/commands/t-digest/ADD.ts

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
2+
3+
export const FIRST_KEY_INDEX = 1;
4+
5+
export function transformArguments(
6+
key: RedisCommandArgument,
7+
values: Array<number>
8+
): RedisCommandArguments {
9+
const args = ['TDIGEST.ADD', key];
10+
for (const item of values) {
11+
args.push(item.toString());
12+
}
13+
14+
return args;
15+
}
16+
17+
export declare function transformReply(): 'OK';

‎packages/bloom/lib/commands/t-digest/BYRANK.spec.ts

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { strict as assert } from 'assert';
2+
import testUtils, { GLOBAL } from '../../test-utils';
3+
import { transformArguments } from './BYRANK';
4+
5+
describe('TDIGEST.BYRANK', () => {
6+
it('transformArguments', () => {
7+
assert.deepEqual(
8+
transformArguments('key', [1, 2]),
9+
['TDIGEST.BYRANK', 'key', '1', '2']
10+
);
11+
});
12+
13+
testUtils.testWithClient('client.tDigest.byRank', async client => {
14+
const [ , reply ] = await Promise.all([
15+
client.tDigest.create('key'),
16+
client.tDigest.byRank('key', [1])
17+
]);
18+
19+
assert.deepEqual(reply, [NaN]);
20+
}, GLOBAL.SERVERS.OPEN);
21+
});

‎packages/bloom/lib/commands/t-digest/BYRANK.ts

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
2+
3+
export const FIRST_KEY_INDEX = 1;
4+
5+
export const IS_READ_ONLY = true;
6+
7+
export function transformArguments(
8+
key: RedisCommandArgument,
9+
ranks: Array<number>
10+
): RedisCommandArguments {
11+
const args = ['TDIGEST.BYRANK', key];
12+
for (const rank of ranks) {
13+
args.push(rank.toString());
14+
}
15+
16+
return args;
17+
}
18+
19+
export { transformDoublesReply as transformReply } from '.';

‎packages/bloom/lib/commands/t-digest/BYREVRANK.spec.ts

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { strict as assert } from 'assert';
2+
import testUtils, { GLOBAL } from '../../test-utils';
3+
import { transformArguments } from './BYREVRANK';
4+
5+
describe('TDIGEST.BYREVRANK', () => {
6+
it('transformArguments', () => {
7+
assert.deepEqual(
8+
transformArguments('key', [1, 2]),
9+
['TDIGEST.BYREVRANK', 'key', '1', '2']
10+
);
11+
});
12+
13+
testUtils.testWithClient('client.tDigest.byRevRank', async client => {
14+
const [ , reply ] = await Promise.all([
15+
client.tDigest.create('key'),
16+
client.tDigest.byRevRank('key', [1])
17+
]);
18+
19+
assert.deepEqual(reply, [NaN]);
20+
}, GLOBAL.SERVERS.OPEN);
21+
});

‎packages/bloom/lib/commands/t-digest/BYREVRANK.ts

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
2+
3+
export const FIRST_KEY_INDEX = 1;
4+
5+
export const IS_READ_ONLY = true;
6+
7+
export function transformArguments(
8+
key: RedisCommandArgument,
9+
ranks: Array<number>
10+
): RedisCommandArguments {
11+
const args = ['TDIGEST.BYREVRANK', key];
12+
for (const rank of ranks) {
13+
args.push(rank.toString());
14+
}
15+
16+
return args;
17+
}
18+
19+
export { transformDoublesReply as transformReply } from '.';

‎packages/bloom/lib/commands/t-digest/CDF.spec.ts

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { strict as assert } from 'assert';
2+
import testUtils, { GLOBAL } from '../../test-utils';
3+
import { transformArguments } from './CDF';
4+
5+
describe('TDIGEST.CDF', () => {
6+
it('transformArguments', () => {
7+
assert.deepEqual(
8+
transformArguments('key', [1, 2]),
9+
['TDIGEST.CDF', 'key', '1', '2']
10+
);
11+
});
12+
13+
testUtils.testWithClient('client.tDigest.cdf', async client => {
14+
const [ , reply ] = await Promise.all([
15+
client.tDigest.create('key'),
16+
client.tDigest.cdf('key', [1])
17+
]);
18+
19+
assert.deepEqual(reply, [NaN]);
20+
}, GLOBAL.SERVERS.OPEN);
21+
});

‎packages/bloom/lib/commands/t-digest/CDF.ts

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
2+
3+
export const FIRST_KEY_INDEX = 1;
4+
5+
export const IS_READ_ONLY = true;
6+
7+
export function transformArguments(
8+
key: RedisCommandArgument,
9+
values: Array<number>
10+
): RedisCommandArguments {
11+
const args = ['TDIGEST.CDF', key];
12+
for (const item of values) {
13+
args.push(item.toString());
14+
}
15+
16+
return args;
17+
}
18+
19+
export { transformDoublesReply as transformReply } from '.';

‎packages/bloom/lib/commands/t-digest/CREATE.spec.ts

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { strict as assert } from 'assert';
2+
import testUtils, { GLOBAL } from '../../test-utils';
3+
import { transformArguments } from './CREATE';
4+
5+
describe('TDIGEST.CREATE', () => {
6+
describe('transformArguments', () => {
7+
it('without options', () => {
8+
assert.deepEqual(
9+
transformArguments('key'),
10+
['TDIGEST.CREATE', 'key']
11+
);
12+
});
13+
14+
it('with COMPRESSION', () => {
15+
assert.deepEqual(
16+
transformArguments('key', {
17+
COMPRESSION: 100
18+
}),
19+
['TDIGEST.CREATE', 'key', 'COMPRESSION', '100']
20+
);
21+
});
22+
});
23+
24+
testUtils.testWithClient('client.tDigest.create', async client => {
25+
assert.equal(
26+
await client.tDigest.create('key'),
27+
'OK'
28+
);
29+
}, GLOBAL.SERVERS.OPEN);
30+
});

‎packages/bloom/lib/commands/t-digest/CREATE.ts

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
2+
import { CompressionOption, pushCompressionArgument } from '.';
3+
4+
export const FIRST_KEY_INDEX = 1;
5+
6+
export function transformArguments(
7+
key: RedisCommandArgument,
8+
options?: CompressionOption
9+
): RedisCommandArguments {
10+
return pushCompressionArgument(
11+
['TDIGEST.CREATE', key],
12+
options
13+
);
14+
}
15+
16+
export declare function transformReply(): 'OK';

‎packages/bloom/lib/commands/t-digest/INFO.spec.ts

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { strict as assert } from 'assert';
2+
import testUtils, { GLOBAL } from '../../test-utils';
3+
import { transformArguments } from './INFO';
4+
5+
describe('TDIGEST.INFO', () => {
6+
it('transformArguments', () => {
7+
assert.deepEqual(
8+
transformArguments('key'),
9+
['TDIGEST.INFO', 'key']
10+
);
11+
});
12+
13+
testUtils.testWithClient('client.tDigest.info', async client => {
14+
await client.tDigest.create('key');
15+
16+
const info = await client.tDigest.info('key');
17+
assert(typeof info.capacity, 'number');
18+
assert(typeof info.mergedNodes, 'number');
19+
assert(typeof info.unmergedNodes, 'number');
20+
assert(typeof info.mergedWeight, 'number');
21+
assert(typeof info.unmergedWeight, 'number');
22+
assert(typeof info.totalCompression, 'number');
23+
assert(typeof info.totalCompression, 'number');
24+
}, GLOBAL.SERVERS.OPEN);
25+
});

‎packages/bloom/lib/commands/t-digest/INFO.ts

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
2+
3+
export const FIRST_KEY_INDEX = 1;
4+
5+
export const IS_READ_ONLY = true;
6+
7+
export function transformArguments(key: RedisCommandArgument): RedisCommandArguments {
8+
return [
9+
'TDIGEST.INFO',
10+
key
11+
];
12+
}
13+
14+
type InfoRawReply = [
15+
'Compression',
16+
number,
17+
'Capacity',
18+
number,
19+
'Merged nodes',
20+
number,
21+
'Unmerged nodes',
22+
number,
23+
'Merged weight',
24+
string,
25+
'Unmerged weight',
26+
string,
27+
'Total compressions',
28+
number
29+
];
30+
31+
interface InfoReply {
32+
comperssion: number;
33+
capacity: number;
34+
mergedNodes: number;
35+
unmergedNodes: number;
36+
mergedWeight: number;
37+
unmergedWeight: number;
38+
totalCompression: number;
39+
}
40+
41+
export function transformReply(reply: InfoRawReply): InfoReply {
42+
return {
43+
comperssion: reply[1],
44+
capacity: reply[3],
45+
mergedNodes: reply[5],
46+
unmergedNodes: reply[7],
47+
mergedWeight: Number(reply[9]),
48+
unmergedWeight: Number(reply[11]),
49+
totalCompression: reply[13]
50+
};
51+
}

‎packages/bloom/lib/commands/t-digest/MAX.spec.ts

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { strict as assert } from 'assert';
2+
import testUtils, { GLOBAL } from '../../test-utils';
3+
import { transformArguments, transformReply } from './MAX';
4+
5+
describe('TDIGEST.MAX', () => {
6+
it('transformArguments', () => {
7+
assert.deepEqual(
8+
transformArguments('key'),
9+
['TDIGEST.MAX', 'key']
10+
);
11+
});
12+
13+
testUtils.testWithClient('client.tDigest.max', async client => {
14+
const [ , reply ] = await Promise.all([
15+
client.tDigest.create('key'),
16+
client.tDigest.max('key')
17+
]);
18+
19+
assert.deepEqual(reply, NaN);
20+
}, GLOBAL.SERVERS.OPEN);
21+
});

‎packages/bloom/lib/commands/t-digest/MAX.ts

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
2+
3+
export const FIRST_KEY_INDEX = 1;
4+
5+
export const IS_READ_ONLY = true;
6+
7+
export function transformArguments(key: RedisCommandArgument): RedisCommandArguments {
8+
return [
9+
'TDIGEST.MAX',
10+
key
11+
];
12+
}
13+
14+
export { transformDoubleReply as transformReply } from '.';

0 commit comments

Comments
 (0)