Skip to content

Commit

Permalink
feat(clickhouse): move to clickhouse supported Python client
Browse files Browse the repository at this point in the history
BREAKING CHANGE: `external_tables` can no longer be passed in `ibis.clickhouse.connect`. Pass `external_tables` directly in `raw_sql`/`execute`/`to_pyarrow`/`to_pyarrow_batches()`.
  • Loading branch information
cpcloud authored and kszucs committed May 12, 2023
1 parent 30a4f7e commit 012557a
Show file tree
Hide file tree
Showing 27 changed files with 596 additions and 448 deletions.
35 changes: 17 additions & 18 deletions ci/schema/clickhouse.sql
Original file line number Diff line number Diff line change
@@ -1,25 +1,24 @@
-- NB: The paths in this file are all relative to /var/lib/clickhouse/user_files

CREATE OR REPLACE TABLE diamonds ENGINE = Memory AS
CREATE OR REPLACE TABLE ibis_testing.diamonds ENGINE = Memory AS
SELECT * FROM file('ibis/diamonds.parquet', 'Parquet');

CREATE OR REPLACE TABLE batting ENGINE = Memory AS
CREATE OR REPLACE TABLE ibis_testing.batting ENGINE = Memory AS
SELECT * FROM file('ibis/batting.parquet', 'Parquet');

CREATE OR REPLACE TABLE awards_players ENGINE = Memory AS
CREATE OR REPLACE TABLE ibis_testing.awards_players ENGINE = Memory AS
SELECT * FROM file('ibis/awards_players.parquet', 'Parquet');

CREATE OR REPLACE TABLE functional_alltypes ENGINE = Memory AS
CREATE OR REPLACE TABLE ibis_testing.functional_alltypes ENGINE = Memory AS
SELECT * REPLACE(CAST(timestamp_col AS Nullable(DateTime)) AS timestamp_col)
FROM file('ibis/functional_alltypes.parquet', 'Parquet');

CREATE OR REPLACE TABLE tzone (
CREATE OR REPLACE TABLE ibis_testing.tzone (
ts Nullable(DateTime),
key Nullable(String),
value Nullable(Float64)
) ENGINE = Memory;

CREATE OR REPLACE TABLE array_types (
CREATE OR REPLACE TABLE ibis_testing.array_types (
x Array(Nullable(Int64)),
y Array(Nullable(String)),
z Array(Nullable(Float64)),
Expand All @@ -28,39 +27,39 @@ CREATE OR REPLACE TABLE array_types (
multi_dim Array(Array(Nullable(Int64)))
) ENGINE = Memory;

INSERT INTO array_types VALUES
INSERT INTO ibis_testing.array_types VALUES
([1, 2, 3], ['a', 'b', 'c'], [1.0, 2.0, 3.0], 'a', 1.0, [[], [1, 2, 3], []]),
([4, 5], ['d', 'e'], [4.0, 5.0], 'a', 2.0, []),
([6, NULL], ['f', NULL], [6.0, NULL], 'a', 3.0, [[], [], []]),
([NULL, 1, NULL], [NULL, 'a', NULL], [], 'b', 4.0, [[1], [2], [], [3, 4, 5]]),
([2, NULL, 3], ['b', NULL, 'c'], NULL, 'b', 5.0, []),
([4, NULL, NULL, 5], ['d', NULL, NULL, 'e'], [4.0, NULL, NULL, 5.0], 'c', 6.0, [[1, 2, 3]]);

CREATE OR REPLACE TABLE time_df1 (
CREATE OR REPLACE TABLE ibis_testing.time_df1 (
time Int64,
value Nullable(Float64),
key Nullable(String)
) ENGINE = Memory;
INSERT INTO time_df1 VALUES
INSERT INTO ibis_testing.time_df1 VALUES
(1, 1.0, 'x'),
(20, 20.0, 'x'),
(30, 30.0, 'x'),
(40, 40.0, 'x'),
(50, 50.0, 'x');

CREATE OR REPLACE TABLE time_df2 (
CREATE OR REPLACE TABLE ibis_testing.time_df2 (
time Int64,
value Nullable(Float64),
key Nullable(String)
) ENGINE = Memory;
INSERT INTO time_df2 VALUES
INSERT INTO ibis_testing.time_df2 VALUES
(19, 19.0, 'x'),
(21, 21.0, 'x'),
(39, 39.0, 'x'),
(49, 49.0, 'x'),
(1000, 1000.0, 'x');

CREATE OR REPLACE TABLE struct (
CREATE OR REPLACE TABLE ibis_testing.struct (
abc Tuple(
a Nullable(Float64),
b Nullable(String),
Expand All @@ -70,7 +69,7 @@ CREATE OR REPLACE TABLE struct (

-- NULL is the same as tuple(NULL, NULL, NULL) because clickhouse doesn't
-- support Nullable(Tuple(...))
INSERT INTO struct VALUES
INSERT INTO ibis_testing.struct VALUES
(tuple(1.0, 'banana', 2)),
(tuple(2.0, 'apple', 3)),
(tuple(3.0, 'orange', 4)),
Expand All @@ -79,14 +78,14 @@ INSERT INTO struct VALUES
(tuple(NULL, NULL, NULL)),
(tuple(3.0, 'orange', NULL));

CREATE OR REPLACE TABLE map (kv Map(String, Nullable(Int64))) ENGINE = Memory;
CREATE OR REPLACE TABLE ibis_testing.map (kv Map(String, Nullable(Int64))) ENGINE = Memory;

INSERT INTO map VALUES
INSERT INTO ibis_testing.map VALUES
(map('a', 1, 'b', 2, 'c', 3)),
(map('d', 4, 'e', 5, 'c', 6));

CREATE OR REPLACE TABLE win (g String, x Int64, y Int64) ENGINE = Memory;
INSERT INTO win VALUES
CREATE OR REPLACE TABLE ibis_testing.win (g String, x Int64, y Int64) ENGINE = Memory;
INSERT INTO ibis_testing.win VALUES
('a', 0, 3),
('a', 1, 2),
('a', 2, 0),
Expand Down
3 changes: 2 additions & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ services:
clickhouse:
image: clickhouse/clickhouse-server:23.4.2.11-alpine
ports:
- 8123:8123
- 9000:9000
healthcheck:
interval: 1s
retries: 10
test:
- CMD-SHELL
- nc -z 127.0.0.1 9000
- nc -z 127.0.0.1 8123 && nc -z 127.0.0.1 9000
timeout: 10s
volumes:
- clickhouse:/var/lib/clickhouse/user_files/ibis
Expand Down
Loading

0 comments on commit 012557a

Please sign in to comment.