From 6f258f3e8cbd5992f16d481230c333f4fd25f4a7 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Wed, 20 Mar 2024 23:39:25 -0700 Subject: [PATCH 01/52] chore: bump version to 0.15.0 (#369) --- Cargo.toml | 10 +++++----- bindings/nodejs/npm/darwin-arm64/package.json | 2 +- bindings/nodejs/npm/darwin-x64/package.json | 2 +- bindings/nodejs/npm/linux-arm64-gnu/package.json | 2 +- bindings/nodejs/npm/linux-x64-gnu/package.json | 2 +- bindings/nodejs/npm/win32-x64-msvc/package.json | 2 +- bindings/nodejs/package.json | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 972a5aaf..7b1be17e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.14.0" +version = "0.15.0" edition = "2021" license = "Apache-2.0" authors = ["Databend Authors "] @@ -21,7 +21,7 @@ keywords = ["databend", "database"] repository = "https://github.com/datafuselabs/bendsql" [workspace.dependencies] -databend-client = { path = "core", version = "0.14.0" } -databend-driver = { path = "driver", version = "0.14.0" } -databend-driver-macros = { path = "macros", version = "0.14.0" } -databend-sql = { path = "sql", version = "0.14.0" } +databend-client = { path = "core", version = "0.15.0" } +databend-driver = { path = "driver", version = "0.15.0" } +databend-driver-macros = { path = "macros", version = "0.15.0" } +databend-sql = { path = "sql", version = "0.15.0" } diff --git a/bindings/nodejs/npm/darwin-arm64/package.json b/bindings/nodejs/npm/darwin-arm64/package.json index 57bd0173..00db8e37 100644 --- a/bindings/nodejs/npm/darwin-arm64/package.json +++ b/bindings/nodejs/npm/darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-arm64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.14.0", + "version": "0.15.0", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/darwin-x64/package.json b/bindings/nodejs/npm/darwin-x64/package.json index 44746585..cf6f7082 100644 --- a/bindings/nodejs/npm/darwin-x64/package.json +++ b/bindings/nodejs/npm/darwin-x64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-x64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.14.0", + "version": "0.15.0", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/linux-arm64-gnu/package.json b/bindings/nodejs/npm/linux-arm64-gnu/package.json index 57fba4b1..4ca89b92 100644 --- a/bindings/nodejs/npm/linux-arm64-gnu/package.json +++ b/bindings/nodejs/npm/linux-arm64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-arm64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.14.0", + "version": "0.15.0", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/linux-x64-gnu/package.json b/bindings/nodejs/npm/linux-x64-gnu/package.json index eed2c128..f738fe90 100644 --- a/bindings/nodejs/npm/linux-x64-gnu/package.json +++ b/bindings/nodejs/npm/linux-x64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-x64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.14.0", + "version": "0.15.0", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/win32-x64-msvc/package.json b/bindings/nodejs/npm/win32-x64-msvc/package.json index 385e3af6..6ec18660 100644 --- a/bindings/nodejs/npm/win32-x64-msvc/package.json +++ b/bindings/nodejs/npm/win32-x64-msvc/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-win32-x64-msvc", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.14.0", + "version": "0.15.0", "os": [ "win32" ], diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index dc70aa5b..55f7685d 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "databend-driver", "author": "Databend Authors ", - "version": "0.14.0", + "version": "0.15.0", "license": "Apache-2.0", "main": "index.js", "types": "index.d.ts", From 3cc3100e0e6b8d343509b01702d908ef087f5e0a Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 21 Mar 2024 15:04:35 +0800 Subject: [PATCH 02/52] fix(bindings/python): add blocking client related to pyi (#368) --- .github/workflows/bindings.python.yml | 2 +- bindings/python/Pipfile | 4 +- bindings/python/Pipfile.lock | 164 +++++------------- .../package/databend_driver/__init__.py | 1 + .../package/databend_driver/__init__.pyi | 16 +- 5 files changed, 66 insertions(+), 121 deletions(-) diff --git a/.github/workflows/bindings.python.yml b/.github/workflows/bindings.python.yml index 7c0700c8..72f6586c 100644 --- a/.github/workflows/bindings.python.yml +++ b/.github/workflows/bindings.python.yml @@ -40,7 +40,7 @@ jobs: run: pipenv install --dev - name: Check format working-directory: bindings/python - run: pipenv run black --check . + run: pipenv run ruff format --check . - name: Setup develop working-directory: bindings/python run: pipenv run maturin develop diff --git a/bindings/python/Pipfile b/bindings/python/Pipfile index 185b339d..549b6439 100644 --- a/bindings/python/Pipfile +++ b/bindings/python/Pipfile @@ -4,12 +4,12 @@ verify_ssl = true name = "pypi" [packages] +databend-driver = {path = ".", editable = true} [dev-packages] maturin = "*" behave = "*" -black = "*" -flake8 = "*" +ruff = "*" [requires] python_version = "3.11" diff --git a/bindings/python/Pipfile.lock b/bindings/python/Pipfile.lock index 8b646c3a..fbb03438 100644 --- a/bindings/python/Pipfile.lock +++ b/bindings/python/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "668b937cec20b273b3e03cb58dce3f54808655219b3e18def516d266d3d9d0b0" + "sha256": "f4bf1b937ddea6c07969baabf98a5a7e324ddd55b719522f72a5d2d827a67f6a" }, "pipfile-spec": 6, "requires": { @@ -15,7 +15,13 @@ } ] }, - "default": {}, + "default": { + "databend-driver": { + "editable": true, + "markers": "python_version >= '3.7'", + "path": "." + } + }, "develop": { "behave": { "hashes": [ @@ -23,148 +29,74 @@ "sha256:ebda1a6c9e5bfe95c5f9f0a2794e01c7098b3dde86c10a95d8621c5907ff6f1c" ], "index": "pypi", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.6" }, - "black": { - "hashes": [ - "sha256:01ede61aac8c154b55f35301fac3e730baf0c9cf8120f65a9cd61a81cfb4a0c3", - "sha256:022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb", - "sha256:25cc308838fe71f7065df53aedd20327969d05671bac95b38fdf37ebe70ac087", - "sha256:27eb7a0c71604d5de083757fbdb245b1a4fae60e9596514c6ec497eb63f95320", - "sha256:327a8c2550ddc573b51e2c352adb88143464bb9d92c10416feb86b0f5aee5ff6", - "sha256:47e56d83aad53ca140da0af87678fb38e44fd6bc0af71eebab2d1f59b1acf1d3", - "sha256:501387a9edcb75d7ae8a4412bb8749900386eaef258f1aefab18adddea1936bc", - "sha256:552513d5cd5694590d7ef6f46e1767a4df9af168d449ff767b13b084c020e63f", - "sha256:5c4bc552ab52f6c1c506ccae05681fab58c3f72d59ae6e6639e8885e94fe2587", - "sha256:642496b675095d423f9b8448243336f8ec71c9d4d57ec17bf795b67f08132a91", - "sha256:6d1c6022b86f83b632d06f2b02774134def5d4d4f1dac8bef16d90cda18ba28a", - "sha256:7f3bf2dec7d541b4619b8ce526bda74a6b0bffc480a163fed32eb8b3c9aed8ad", - "sha256:831d8f54c3a8c8cf55f64d0422ee875eecac26f5f649fb6c1df65316b67c8926", - "sha256:8417dbd2f57b5701492cd46edcecc4f9208dc75529bcf76c514864e48da867d9", - "sha256:86cee259349b4448adb4ef9b204bb4467aae74a386bce85d56ba4f5dc0da27be", - "sha256:893695a76b140881531062d48476ebe4a48f5d1e9388177e175d76234ca247cd", - "sha256:9fd59d418c60c0348505f2ddf9609c1e1de8e7493eab96198fc89d9f865e7a96", - "sha256:ad0014efc7acf0bd745792bd0d8857413652979200ab924fbf239062adc12491", - "sha256:b5b0ee6d96b345a8b420100b7d71ebfdd19fab5e8301aff48ec270042cd40ac2", - "sha256:c333286dc3ddca6fdff74670b911cccedacb4ef0a60b34e491b8a67c833b343a", - "sha256:f9062af71c59c004cd519e2fb8f5d25d39e46d3af011b41ab43b9c74e27e236f", - "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995" - ], - "index": "pypi", - "version": "==23.7.0" - }, - "click": { - "hashes": [ - "sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367", - "sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548" - ], - "markers": "python_version >= '3.7'", - "version": "==8.1.5" - }, - "flake8": { - "hashes": [ - "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", - "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" - ], - "index": "pypi", - "version": "==6.0.0" - }, "maturin": { "hashes": [ - "sha256:19332aa0830f23813e461f61e4d2b7d0c6f8ec84c335040232aafea3f068ac31", - "sha256:32e95212e6b334caf68d278bcc5137dde020a8e84095c2728d9b08d2311e5d64", - "sha256:3f6f1e8e970f7728c26eeb55b4c38103f2c5d830b9df4c12fd00903eef7a4114", - "sha256:4252241c196abf0ede0088e38bc3c181fe0bf073a974d6594493f3ae7232545c", - "sha256:4650aeaa8debd004b55aae7afb75248cbd4d61cd7da2dcf4ead8b22b58cecae0", - "sha256:47705b6c5b5ec9d883f6b4fb6ae2480a07a77aabbfbc658d6327c9d6cd74c7cf", - "sha256:6f63dc6f9dba2081346f0ff40019e67a72eb20af7ee4847dc21174dd3636a981", - "sha256:7a2398df2d2f5ba4970ac7f670108e7c7e2da39185caf9b9f9e67d14ae4315e2", - "sha256:9042ea6538529e22954c0a4eefbe55026a9eba45484423882dd8eb97854b29dd", - "sha256:a36efcca897b356deee56c7a3c399c595201518a892ec0f20f0f20abb3064ccb", - "sha256:c0fecd5ae4f8bad703ee648873d837e3bd6eb846f48ff8607bfb0556a2010adc", - "sha256:eb04f3473b9f283eba51a5dc20dc0bab6f8741048c1bbc5e45f39cc29ad69aa6", - "sha256:eee3c9b2cd80adee6938ea1828882795e261a7fc6b5ebaed15003ed4d713adaa" + "sha256:0b976116b7cfaafbc8c3f0acfaec6702520c49e86e48ea80e2c282b7f8118c1a", + "sha256:1b29bf8771f27d2e6b2685c82de952b5732ee79e5c0030ffd5dab5ccb99137a1", + "sha256:2e4c01370a5c10b6c4887bee66d3582bdb38c3805168c1393f072bd266da08d4", + "sha256:76e3270ff87b5484976d23e3d88475cd64acf41b54f561263f253d8fca0baab3", + "sha256:9cba3737cb92ce5c1bd82cbb9b1fde412b2aac8882ac38b8340980f5eb858d8c", + "sha256:a5c038ded82c7595d99e94a208aa8af2b5c94eef4c8fcf5ef6e841957e506201", + "sha256:b3a499ff5960e46115488e68011809ce99857864ce3a91cf5d0fff3adbd89e8c", + "sha256:d277adf9b27143627ba7be7ea254513d3e85008fb16a94638b56884a41b4e5a2", + "sha256:d6a314472e07b6bdfa4cdf97d24cda1defe008d36d4b75de2efd3383e7a2d7bf", + "sha256:e046ea2aed687991d58c42f6276dfcc0c037092934654f538b5877fd57dd3a9c", + "sha256:eb35dfe5994ad2c34d2874a73720847ecc2adb28f934e9a7cbcdb8826b240e60", + "sha256:f271f315fb78d2ff5fdf60f8d3ada2a04a66ac6fbd3cbb318c4eb4e9766449bc", + "sha256:faa0d099a8045afc9977284cb3a1c26e5ebc9a7f0fe4d53b7ee17f62fd279f4a" ], "index": "pypi", - "version": "==1.1.0" - }, - "mccabe": { - "hashes": [ - "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", - "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" - ], - "markers": "python_version >= '3.6'", - "version": "==0.7.0" - }, - "mypy-extensions": { - "hashes": [ - "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", - "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.0" - }, - "packaging": { - "hashes": [ - "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", - "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" - ], "markers": "python_version >= '3.7'", - "version": "==23.1" + "version": "==1.5.0" }, "parse": { "hashes": [ - "sha256:371ed3800dc63983832159cc9373156613947707bc448b5215473a219dbd4362", - "sha256:cc3a47236ff05da377617ddefa867b7ba983819c664e1afe46249e5b469be464" + "sha256:09002ca350ad42e76629995f71f7b518670bcf93548bdde3684fd55d2be51975", + "sha256:76ddd5214255ae711db4c512be636151fbabaa948c6f30115aecc440422ca82c" ], - "version": "==1.19.1" + "version": "==1.20.1" }, "parse-type": { "hashes": [ "sha256:06d39a8b70fde873eb2a131141a0e79bb34a432941fb3d66fad247abafc9766c", "sha256:79b1f2497060d0928bc46016793f1fca1057c4aacdf15ef876aa48d75a73a355" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.6.2" }, - "pathspec": { + "ruff": { "hashes": [ - "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687", - "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293" - ], - "markers": "python_version >= '3.7'", - "version": "==0.11.1" - }, - "platformdirs": { - "hashes": [ - "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421", - "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f" + "sha256:0171aab5fecdc54383993389710a3d1227f2da124d76a2784a7098e818f92d61", + "sha256:0da458989ce0159555ef224d5b7c24d3d2e4bf4c300b85467b08c3261c6bc6a8", + "sha256:1eca7ff7a47043cf6ce5c7f45f603b09121a7cc047447744b029d1b719278eb5", + "sha256:2700a804d5336bcffe063fd789ca2c7b02b552d2e323a336700abb8ae9e6a3f8", + "sha256:352e95ead6964974b234e16ba8a66dad102ec7bf8ac064a23f95371d8b198aab", + "sha256:38671be06f57a2f8aba957d9f701ea889aa5736be806f18c0cd03d6ff0cbca8d", + "sha256:45817af234605525cdf6317005923bf532514e1ea3d9270acf61ca2440691376", + "sha256:5a6cbf216b69c7090f0fe4669501a27326c34e119068c1494f35aaf4cc683778", + "sha256:79bca3a03a759cc773fca69e0bdeac8abd1c13c31b798d5bb3c9da4a03144a9f", + "sha256:8d6ab88c81c4040a817aa432484e838aaddf8bfd7ca70e4e615482757acb64f8", + "sha256:973a0e388b7bc2e9148c7f9be8b8c6ae7471b9be37e1cc732f8f44a6f6d7720d", + "sha256:b24c19e8598916d9c6f5a5437671f55ee93c212a2c4c569605dc3842b6820386", + "sha256:be90bcae57c24d9f9d023b12d627e958eb55f595428bafcb7fec0791ad25ddfc", + "sha256:cfa60d23269d6e2031129b053fdb4e5a7b0637fc6c9c0586737b962b2f834493", + "sha256:e7d3f6762217c1da954de24b4a1a70515630d29f71e268ec5000afe81377642d", + "sha256:f2831ec6a580a97f1ea82ea1eda0401c3cdf512cf2045fa3c85e8ef109e87de0", + "sha256:fd66469f1a18fdb9d32e22b79f486223052ddf057dc56dea0caaf1a47bdfaf4e" ], + "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==3.9.1" - }, - "pycodestyle": { - "hashes": [ - "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", - "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" - ], - "markers": "python_version >= '3.6'", - "version": "==2.10.0" - }, - "pyflakes": { - "hashes": [ - "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", - "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" - ], - "markers": "python_version >= '3.6'", - "version": "==3.0.1" + "version": "==0.3.3" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" } } diff --git a/bindings/python/package/databend_driver/__init__.py b/bindings/python/package/databend_driver/__init__.py index fdafd8eb..356fd9d5 100644 --- a/bindings/python/package/databend_driver/__init__.py +++ b/bindings/python/package/databend_driver/__init__.py @@ -13,4 +13,5 @@ # limitations under the License. # flake8: noqa + from ._databend_driver import * diff --git a/bindings/python/package/databend_driver/__init__.pyi b/bindings/python/package/databend_driver/__init__.pyi index c526b453..4e32645d 100644 --- a/bindings/python/package/databend_driver/__init__.pyi +++ b/bindings/python/package/databend_driver/__init__.pyi @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# flake8: noqa + class ServerStats: @property def total_rows(self) -> int: ... @@ -59,7 +61,6 @@ class RowIterator: async def __anext__(self) -> Row: ... def schema(self) -> Schema: ... -# flake8: noqa class AsyncDatabendConnection: async def info(self) -> ConnectionInfo: ... async def version(self) -> str: ... @@ -68,7 +69,18 @@ class AsyncDatabendConnection: async def query_iter(self, sql: str) -> RowIterator: ... async def stream_load(self, sql: str, data: list[list[str]]) -> ServerStats: ... -# flake8: noqa class AsyncDatabendClient: def __init__(self, dsn: str): ... async def get_conn(self) -> AsyncDatabendConnection: ... + +class BlockingDatabendConnection: + def info(self) -> ConnectionInfo: ... + def version(self) -> str: ... + def exec(self, sql: str) -> int: ... + def query_row(self, sql: str) -> Row: ... + def query_iter(self, sql: str) -> RowIterator: ... + def stream_load(self, sql: str, data: list[list[str]]) -> ServerStats: ... + +class BlockingDatabendClient: + def __init__(self, dsn: str): ... + def get_conn(self) -> BlockingDatabendConnection: ... From bdcb14981fa9e1675dc9bd8838fb5aab03594472 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 21 Mar 2024 16:19:19 +0800 Subject: [PATCH 03/52] feat(bindings/python): convert decimal values to python Decimal (#370) --- bindings/nodejs/tests/binding.js | 7 +++++ bindings/python/.gitignore | 2 ++ bindings/python/src/types.rs | 31 +++++++++++++++---- .../python/tests/asyncio/steps/binding.py | 10 ++++++ .../python/tests/blocking/steps/binding.py | 9 ++++++ bindings/tests/features/binding.feature | 4 +++ 6 files changed, 57 insertions(+), 6 deletions(-) diff --git a/bindings/nodejs/tests/binding.js b/bindings/nodejs/tests/binding.js index 80fb2ada..d6b14e78 100644 --- a/bindings/nodejs/tests/binding.js +++ b/bindings/nodejs/tests/binding.js @@ -33,6 +33,13 @@ Then("Select string {string} should be equal to {string}", async function (input assert.equal(output, value); }); +Then("Select types should be expected native types", async function () { + // NumberValue::Decimal + const row = await this.conn.queryRow(`SELECT 15.7563::Decimal(8,4), 2.0+3.0`); + const excepted = ["15.7563", "5.0"]; + assert.deepEqual(row.values(), excepted); +}); + Then("Select numbers should iterate all rows", async function () { let rows = await this.conn.queryIter("SELECT number FROM numbers(5)"); let ret = []; diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index 65036982..d106553c 100644 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -73,3 +73,5 @@ docs/_build/ # Generated docs docs + +.ruff_cache/ diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs index 9dd0d479..2884f4a4 100644 --- a/bindings/python/src/types.rs +++ b/bindings/python/src/types.rs @@ -16,8 +16,9 @@ use std::sync::Arc; use once_cell::sync::Lazy; use pyo3::exceptions::{PyException, PyStopAsyncIteration, PyStopIteration}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyList, PyTuple}; +use pyo3::sync::GILOnceCell; +use pyo3::types::{PyDict, PyList, PyTuple, PyType}; +use pyo3::{intern, prelude::*}; use pyo3_asyncio::tokio::future_into_py; use tokio::sync::Mutex; use tokio_stream::StreamExt; @@ -29,6 +30,18 @@ pub static VERSION: Lazy = Lazy::new(|| { version.to_string() }); +pub static DECIMAL_CLS: GILOnceCell> = GILOnceCell::new(); + +fn get_decimal_cls(py: Python<'_>) -> PyResult<&PyType> { + DECIMAL_CLS + .get_or_try_init(py, || { + py.import(intern!(py, "decimal"))? + .getattr(intern!(py, "Decimal"))? + .extract() + }) + .map(|ty| ty.as_ref(py)) +} + pub struct Value(databend_driver::Value); impl IntoPy for Value { @@ -97,12 +110,18 @@ impl IntoPy for NumberValue { databend_driver::NumberValue::Float32(i) => i.into_py(py), databend_driver::NumberValue::Float64(i) => i.into_py(py), databend_driver::NumberValue::Decimal128(_, _) => { - let s = self.0.to_string(); - s.into_py(py) + let dec_cls = get_decimal_cls(py).expect("failed to load decimal.Decimal"); + let ret = dec_cls + .call1((self.0.to_string(),)) + .expect("failed to call decimal.Decimal(value)"); + ret.to_object(py) } databend_driver::NumberValue::Decimal256(_, _) => { - let s = self.0.to_string(); - s.into_py(py) + let dec_cls = get_decimal_cls(py).expect("failed to load decimal.Decimal"); + let ret = dec_cls + .call1((self.0.to_string(),)) + .expect("failed to call decimal.Decimal(value)"); + ret.to_object(py) } } } diff --git a/bindings/python/tests/asyncio/steps/binding.py b/bindings/python/tests/asyncio/steps/binding.py index 5d13b68f..eec63532 100644 --- a/bindings/python/tests/asyncio/steps/binding.py +++ b/bindings/python/tests/asyncio/steps/binding.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from decimal import Decimal from behave import given, when, then from behave.api.async_step import async_run_until_complete @@ -56,6 +57,15 @@ async def _(context, input, output): assert output == value +@then("Select types should be expected native types") +@async_run_until_complete +async def _(context): + # NumberValue::Decimal + row = await context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") + expected = (Decimal("15.7563"), Decimal("5.0")) + assert row.values() == expected + + @then("Select numbers should iterate all rows") @async_run_until_complete async def _(context): diff --git a/bindings/python/tests/blocking/steps/binding.py b/bindings/python/tests/blocking/steps/binding.py index 071d733a..67ccf27f 100644 --- a/bindings/python/tests/blocking/steps/binding.py +++ b/bindings/python/tests/blocking/steps/binding.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from decimal import Decimal from behave import given, when, then import databend_driver @@ -52,6 +53,14 @@ def _(context, input, output): assert output == value +@then("Select types should be expected native types") +async def _(context): + # NumberValue::Decimal + row = context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") + expected = (Decimal("15.7563"), Decimal("5.0")) + assert row.values() == expected + + @then("Select numbers should iterate all rows") def _(context): rows = context.conn.query_iter("SELECT number FROM numbers(5)") diff --git a/bindings/tests/features/binding.feature b/bindings/tests/features/binding.feature index 35953ec6..1db9dfb2 100644 --- a/bindings/tests/features/binding.feature +++ b/bindings/tests/features/binding.feature @@ -18,6 +18,10 @@ Feature: Databend Driver Given A new Databend Driver Client Then Select string "Hello, Databend!" should be equal to "Hello, Databend!" + Scenario: Select Types + Given A new Databend Driver Client + Then Select types should be expected native types + Scenario: Select Iter Given A new Databend Driver Client Then Select numbers should iterate all rows From d7ec1e987c29d4e3ed2de857e9a0268feaa4cc90 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 21 Mar 2024 18:04:47 +0800 Subject: [PATCH 04/52] fix(driver): do not kill query for query_row (#371) --- driver/src/conn.rs | 12 +++++++++--- driver/src/flight_sql.rs | 6 ------ driver/src/rest_api.rs | 40 ---------------------------------------- 3 files changed, 9 insertions(+), 49 deletions(-) diff --git a/driver/src/conn.rs b/driver/src/conn.rs index f7a58e54..7708f881 100644 --- a/driver/src/conn.rs +++ b/driver/src/conn.rs @@ -105,13 +105,19 @@ pub trait Connection: DynClone + Send + Sync { } async fn exec(&self, sql: &str) -> Result; - async fn query_row(&self, sql: &str) -> Result>; + async fn query_iter(&self, sql: &str) -> Result; + async fn query_iter_ext(&self, sql: &str) -> Result; + + async fn query_row(&self, sql: &str) -> Result> { + let rows = self.query_all(sql).await?; + let row = rows.into_iter().next(); + Ok(row) + } + async fn query_all(&self, sql: &str) -> Result> { let rows = self.query_iter(sql).await?; rows.collect().await } - async fn query_iter(&self, sql: &str) -> Result; - async fn query_iter_ext(&self, sql: &str) -> Result; /// Get presigned url for a given operation and stage location. /// The operation can be "UPLOAD" or "DOWNLOAD". diff --git a/driver/src/flight_sql.rs b/driver/src/flight_sql.rs index db62361c..d80f054c 100644 --- a/driver/src/flight_sql.rs +++ b/driver/src/flight_sql.rs @@ -65,12 +65,6 @@ impl Connection for FlightSQLConnection { Ok(affected_rows) } - async fn query_row(&self, sql: &str) -> Result> { - let mut rows = self.query_iter(sql).await?; - let row = rows.try_next().await?; - Ok(row) - } - async fn query_iter(&self, sql: &str) -> Result { let rows_with_progress = self.query_iter_ext(sql).await?; let rows = rows_with_progress.filter_rows().await; diff --git a/driver/src/rest_api.rs b/driver/src/rest_api.rs index 133ed08f..d4c55427 100644 --- a/driver/src/rest_api.rs +++ b/driver/src/rest_api.rs @@ -76,27 +76,6 @@ impl Connection for RestAPIConnection { Ok(RowStatsIterator::new(Arc::new(schema), Box::pin(rows))) } - async fn query_row(&self, sql: &str) -> Result> { - info!("query row: {}", sql); - let resp = self.client.start_query(sql).await?; - let resp = self.wait_for_data(resp).await?; - match resp.kill_uri { - Some(uri) => self - .client - .kill_query(&resp.id, &uri) - .await - .map_err(|e| e.into()), - None => Err(Error::InvalidResponse("kill_uri is empty".to_string())), - }?; - let schema = resp.schema.try_into()?; - if resp.data.is_empty() { - Ok(None) - } else { - let row = Row::try_from((Arc::new(schema), &resp.data[0]))?; - Ok(Some(row)) - } - } - async fn get_presigned_url(&self, operation: &str, stage: &str) -> Result { info!("get presigned url: {} {}", operation, stage); let sql = format!("PRESIGN {} {}", operation, stage); @@ -196,25 +175,6 @@ impl<'o> RestAPIConnection { Ok(Self { client }) } - async fn wait_for_data(&self, pre: QueryResponse) -> Result { - if !pre.data.is_empty() { - return Ok(pre); - } - // preserve schema since it is not included in the final response in old servers - let pre_schema = pre.schema.clone(); - let mut result = pre; - while let Some(next_uri) = result.next_uri { - result = self.client.query_page(&result.id, &next_uri).await?; - if !result.data.is_empty() { - break; - } - } - if result.schema.is_empty() { - result.schema = pre_schema; - } - Ok(result) - } - async fn wait_for_schema(&self, pre: QueryResponse) -> Result { if !pre.data.is_empty() || !pre.schema.is_empty() { return Ok(pre); From 4ee348d7a7e5232210eedf50feadb8a9495d13f5 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Mon, 25 Mar 2024 14:09:37 +0800 Subject: [PATCH 05/52] chore: bump version to 0.16.0 (#372) --- Cargo.toml | 10 +++++----- bindings/nodejs/npm/darwin-arm64/package.json | 2 +- bindings/nodejs/npm/darwin-x64/package.json | 2 +- bindings/nodejs/npm/linux-arm64-gnu/package.json | 2 +- bindings/nodejs/npm/linux-x64-gnu/package.json | 2 +- bindings/nodejs/npm/win32-x64-msvc/package.json | 2 +- bindings/nodejs/package.json | 2 +- scripts/bump.sh | 7 +++++++ 8 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7b1be17e..a375a7a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.15.0" +version = "0.16.0" edition = "2021" license = "Apache-2.0" authors = ["Databend Authors "] @@ -21,7 +21,7 @@ keywords = ["databend", "database"] repository = "https://github.com/datafuselabs/bendsql" [workspace.dependencies] -databend-client = { path = "core", version = "0.15.0" } -databend-driver = { path = "driver", version = "0.15.0" } -databend-driver-macros = { path = "macros", version = "0.15.0" } -databend-sql = { path = "sql", version = "0.15.0" } +databend-client = { path = "core", version = "0.16.0" } +databend-driver = { path = "driver", version = "0.16.0" } +databend-driver-macros = { path = "macros", version = "0.16.0" } +databend-sql = { path = "sql", version = "0.16.0" } diff --git a/bindings/nodejs/npm/darwin-arm64/package.json b/bindings/nodejs/npm/darwin-arm64/package.json index 00db8e37..371e2007 100644 --- a/bindings/nodejs/npm/darwin-arm64/package.json +++ b/bindings/nodejs/npm/darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-arm64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.15.0", + "version": "0.16.0", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/darwin-x64/package.json b/bindings/nodejs/npm/darwin-x64/package.json index cf6f7082..199abba4 100644 --- a/bindings/nodejs/npm/darwin-x64/package.json +++ b/bindings/nodejs/npm/darwin-x64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-x64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.15.0", + "version": "0.16.0", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/linux-arm64-gnu/package.json b/bindings/nodejs/npm/linux-arm64-gnu/package.json index 4ca89b92..f949104c 100644 --- a/bindings/nodejs/npm/linux-arm64-gnu/package.json +++ b/bindings/nodejs/npm/linux-arm64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-arm64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.15.0", + "version": "0.16.0", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/linux-x64-gnu/package.json b/bindings/nodejs/npm/linux-x64-gnu/package.json index f738fe90..693fc1bf 100644 --- a/bindings/nodejs/npm/linux-x64-gnu/package.json +++ b/bindings/nodejs/npm/linux-x64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-x64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.15.0", + "version": "0.16.0", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/win32-x64-msvc/package.json b/bindings/nodejs/npm/win32-x64-msvc/package.json index 6ec18660..01dd2f7a 100644 --- a/bindings/nodejs/npm/win32-x64-msvc/package.json +++ b/bindings/nodejs/npm/win32-x64-msvc/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-win32-x64-msvc", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.15.0", + "version": "0.16.0", "os": [ "win32" ], diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index 55f7685d..413637ed 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "databend-driver", "author": "Databend Authors ", - "version": "0.15.0", + "version": "0.16.0", "license": "Apache-2.0", "main": "index.js", "types": "index.d.ts", diff --git a/scripts/bump.sh b/scripts/bump.sh index f545b212..728ca7eb 100755 --- a/scripts/bump.sh +++ b/scripts/bump.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + VERSION=$1 if [ -z "$VERSION" ]; then @@ -33,6 +35,11 @@ else sed -i "s/\"version\": \".*\"/\"version\": \"$VERSION\"/g" bindings/nodejs/npm/*/package.json fi +git status +git checkout main +git fetch upstream +git rebase upstream/main +git checkout -b "bump-$VERSION" git status git add Cargo.toml bindings/nodejs/package.json bindings/nodejs/npm/*/package.json git commit -m "chore: bump version to $VERSION" From a415abb9aa89a0bfa6317ac40f4f5e92767c482a Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 3 Apr 2024 13:22:04 +0800 Subject: [PATCH 06/52] feat: compress with zstd for loading data (#376) --- cli/src/main.rs | 4 ---- driver/Cargo.toml | 1 + driver/src/rest_api.rs | 30 ++++++++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 0b5415ce..c1892aae 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -66,24 +66,20 @@ impl InputFormat { options.insert("field_delimiter", ","); options.insert("quote", "\""); options.insert("skip_header", "0"); - options.insert("compression", "NONE"); } InputFormat::TSV => { options.insert("type", "TSV"); options.insert("record_delimiter", "\n"); options.insert("field_delimiter", "\t"); - options.insert("compression", "NONE"); } InputFormat::NDJSON => { options.insert("type", "NDJSON"); - options.insert("compression", "NONE"); } InputFormat::Parquet => { options.insert("type", "Parquet"); } InputFormat::XML => { options.insert("type", "XML"); - options.insert("compression", "NONE"); options.insert("row_tag", "row"); } } diff --git a/driver/Cargo.toml b/driver/Cargo.toml index 8b029b38..df02a234 100644 --- a/driver/Cargo.toml +++ b/driver/Cargo.toml @@ -30,6 +30,7 @@ databend-client = { workspace = true } databend-driver-macros = { workspace = true } databend-sql = { workspace = true } +async-compression = { version = "0.4", features = ["tokio", "zstd"] } async-trait = "0.1" chrono = { version = "0.4.35", default-features = false, features = ["clock"] } csv = "1.3" diff --git a/driver/src/rest_api.rs b/driver/src/rest_api.rs index d4c55427..eafbd583 100644 --- a/driver/src/rest_api.rs +++ b/driver/src/rest_api.rs @@ -14,14 +14,17 @@ use std::collections::{BTreeMap, VecDeque}; use std::future::Future; +use std::io::Cursor; use std::path::Path; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; +use async_compression::tokio::write::ZstdEncoder; use async_trait::async_trait; use log::info; use tokio::fs::File; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_stream::Stream; use databend_client::error::Error as ClientError; @@ -113,10 +116,33 @@ impl Connection for RestAPIConnection { .timestamp_nanos_opt() .ok_or_else(|| Error::IO("Failed to get current timestamp".to_string()))?; let stage = format!("@~/client/load/{}", now); - self.upload_to_stage(&stage, data, size).await?; - let file_format_options = + + let mut file_format_options = file_format_options.unwrap_or_else(Self::default_file_format_options); let copy_options = copy_options.unwrap_or_else(Self::default_copy_options); + + let mut data = data; + let mut size = size; + + if file_format_options.get("compression").is_none() { + let mut buffer = Vec::new(); + let real_size = data.read_to_end(&mut buffer).await?; + if real_size != size as usize && size != 0 { + return Err(Error::IO(format!( + "Failed to read all data, expected: {}, read: {}", + size, real_size + ))); + } + let mut encoder = ZstdEncoder::new(Vec::new()); + encoder.write_all(&buffer).await?; + encoder.shutdown().await?; + file_format_options.insert("compression", "ZSTD"); + let output = encoder.into_inner(); + size = output.len() as u64; + data = Box::new(Cursor::new(output)) + } + + self.upload_to_stage(&stage, data, size).await?; let resp = self .client .insert_with_stage(sql, &stage, file_format_options, copy_options) From b01f9b84d688334d42a59e853a62a3f218fb6d33 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Wed, 3 Apr 2024 06:04:37 -0700 Subject: [PATCH 07/52] feat: add more commands (#377) * feat: add more commands * update --- README.md | 20 ++++++++++++---- cli/Cargo.toml | 1 + cli/src/session.rs | 60 +++++++++++++++++++++++++++++++++------------- 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index e44ba957..ec5048a8 100644 --- a/README.md +++ b/README.md @@ -144,15 +144,25 @@ prompt = ":) " | `replace_newline` | whether replace '\n' with '\\\n'. | -## Control commands in REPL +## Commands in REPL -We can use `.CMD_NAME VAL` to update the `Settings` above in runtime, example: +| Commands | Description | +|---|---| +| `!exit` | Exit bendsql | +| `!quit` | Exit bendsql | +| `!configs` | Show current settings | +| `!set` | Set settings | +| `!source file` | Source file and execute | + +## Setting commands in REPL + +We can use `!set CMD_NAME VAL` to update the `Settings` above in runtime, example: ``` ❯ bendsql -:) .display_pretty_sql false -:) .max_display_rows 10 -:) .expand auto +:) !set display_pretty_sql false +:) !set max_display_rows 10 +:) !set expand auto ``` ## DSN diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4d42879d..41631b3b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -42,6 +42,7 @@ toml = "0.8" tracing-appender = "0.2" unicode-segmentation = "1.10" url = { version = "2.5", default-features = false } +async-recursion = "1.1.0" [build-dependencies] vergen = { version = "8.2", features = ["build", "git", "gix"] } diff --git a/cli/src/session.rs b/cli/src/session.rs index e127f2df..f2c7ad22 100644 --- a/cli/src/session.rs +++ b/cli/src/session.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use anyhow::anyhow; use anyhow::Result; +use async_recursion::async_recursion; use chrono::NaiveDateTime; use databend_driver::ServerStats; use databend_driver::{Client, Connection}; @@ -319,7 +320,7 @@ impl Session { } if self.query.is_empty() - && (line.starts_with('.') + && (line.starts_with('!') || line == "exit" || line == "quit" || line.to_uppercase().starts_with("PUT")) @@ -388,28 +389,16 @@ impl Session { queries } + #[async_recursion] pub async fn handle_query( &mut self, is_repl: bool, query: &str, ) -> Result> { let query = query.trim_end_matches(';').trim(); - if is_repl && (query == "exit" || query == "quit") { - return Ok(None); - } - if is_repl && query.starts_with('.') { - let query = query - .trim_start_matches('.') - .split_whitespace() - .collect::>(); - if query.len() != 2 { - return Err(anyhow!( - "Control command error, must be syntax of `.cmd_name cmd_value`." - )); - } - self.settings.inject_ctrl_cmd(query[0], query[1])?; - return Ok(Some(ServerStats::default())); + if is_repl && query.starts_with('!') { + return self.handle_commands(query).await; } let start = Instant::now(); @@ -466,6 +455,45 @@ impl Session { } } + #[async_recursion] + pub async fn handle_commands(&mut self, query: &str) -> Result> { + if query == "!exit" || query == "!quit" { + return Ok(None); + } + + match query { + "!exit" | "!quit" => { + return Ok(None); + } + "!configs" => { + println!("{:#?}", self.settings); + } + other => { + if other.starts_with("!set") { + let query = query[4..].split_whitespace().collect::>(); + if query.len() != 2 { + return Err(anyhow!( + "Control command error, must be syntax of `.cmd_name cmd_value`." + )); + } + self.settings.inject_ctrl_cmd(query[0], query[1])?; + } else if other.starts_with("!source") { + let query = query[7..].trim(); + let path = Path::new(query); + if !path.exists() { + return Err(anyhow!("File not found: {}", query)); + } + let file = std::fs::File::open(path)?; + let reader = std::io::BufReader::new(file); + self.handle_reader(reader).await?; + } else { + return Err(anyhow!("Unknown commands: {}", other)); + } + } + } + Ok(Some(ServerStats::default())) + } + pub async fn stream_load_stdin( &mut self, query: &str, From d9f7454c99d93f53e6ff01e678e2abbb0b0e29c0 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Sun, 7 Apr 2024 11:40:29 +0800 Subject: [PATCH 08/52] fix(core): make client scheme public (#378) --- core/src/client.rs | 5 +++-- driver/tests/driver/load.rs | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/client.rs b/core/src/client.rs index c155e0e0..05a44c54 100644 --- a/core/src/client.rs +++ b/core/src/client.rs @@ -50,11 +50,12 @@ static VERSION: Lazy = Lazy::new(|| { #[derive(Clone)] pub struct APIClient { pub cli: HttpClient, - scheme: String, - endpoint: Url, + pub scheme: String, pub host: String, pub port: u16, + endpoint: Url, + auth: Arc, tenant: Option, diff --git a/driver/tests/driver/load.rs b/driver/tests/driver/load.rs index ccc90869..488ae9da 100644 --- a/driver/tests/driver/load.rs +++ b/driver/tests/driver/load.rs @@ -36,9 +36,9 @@ async fn prepare_client(presigned: bool) -> Option { Some(client) } -async fn prepare_table(client: &Client) -> String { +async fn prepare_table(client: &Client, prefix: &str) -> String { let conn = client.get_conn().await.unwrap(); - let table = format!("books_{}", Utc::now().format("%Y%m%d%H%M%S%9f")); + let table = format!("books_{}_{}", prefix, Utc::now().format("%Y%m%d%H%M%S%9f")); let sql = format!( "CREATE TABLE `{}` ( title VARCHAR NULL, @@ -121,7 +121,7 @@ async fn check_result(table: &str, client: &Client) { #[tokio::test] async fn load_csv_with_presign() { if let Some(client) = prepare_client(true).await { - let table = prepare_table(&client).await; + let table = prepare_table(&client, "load_csv_with_presign").await; prepare_data_with_file(&table, "csv", &client).await; check_result(&table, &client).await; } @@ -130,7 +130,7 @@ async fn load_csv_with_presign() { #[tokio::test] async fn load_csv_without_presign() { if let Some(client) = prepare_client(false).await { - let table = prepare_table(&client).await; + let table = prepare_table(&client, "load_csv_without_presign").await; prepare_data_with_file(&table, "csv", &client).await; check_result(&table, &client).await; } @@ -139,7 +139,7 @@ async fn load_csv_without_presign() { #[tokio::test] async fn stream_load_with_presign() { if let Some(client) = prepare_client(true).await { - let table = prepare_table(&client).await; + let table = prepare_table(&client, "stream_load_with_presign").await; prepare_data(&table, &client).await; check_result(&table, &client).await; } From 79d17034c0e15baa6d778c0205ca433bb346ceb5 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Sun, 7 Apr 2024 11:56:43 +0800 Subject: [PATCH 09/52] chore: bump version to 0.16.1 (#379) --- Cargo.toml | 10 +++++----- bindings/nodejs/npm/darwin-arm64/package.json | 2 +- bindings/nodejs/npm/darwin-x64/package.json | 2 +- bindings/nodejs/npm/linux-arm64-gnu/package.json | 2 +- bindings/nodejs/npm/linux-x64-gnu/package.json | 2 +- bindings/nodejs/npm/win32-x64-msvc/package.json | 2 +- bindings/nodejs/package.json | 2 +- scripts/bump.sh | 11 ++++++----- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a375a7a8..7a1ecd62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.16.0" +version = "0.16.1" edition = "2021" license = "Apache-2.0" authors = ["Databend Authors "] @@ -21,7 +21,7 @@ keywords = ["databend", "database"] repository = "https://github.com/datafuselabs/bendsql" [workspace.dependencies] -databend-client = { path = "core", version = "0.16.0" } -databend-driver = { path = "driver", version = "0.16.0" } -databend-driver-macros = { path = "macros", version = "0.16.0" } -databend-sql = { path = "sql", version = "0.16.0" } +databend-client = { path = "core", version = "0.16.1" } +databend-driver = { path = "driver", version = "0.16.1" } +databend-driver-macros = { path = "macros", version = "0.16.1" } +databend-sql = { path = "sql", version = "0.16.1" } diff --git a/bindings/nodejs/npm/darwin-arm64/package.json b/bindings/nodejs/npm/darwin-arm64/package.json index 371e2007..3d56b072 100644 --- a/bindings/nodejs/npm/darwin-arm64/package.json +++ b/bindings/nodejs/npm/darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-arm64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.0", + "version": "0.16.1", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/darwin-x64/package.json b/bindings/nodejs/npm/darwin-x64/package.json index 199abba4..aacb2315 100644 --- a/bindings/nodejs/npm/darwin-x64/package.json +++ b/bindings/nodejs/npm/darwin-x64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-x64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.0", + "version": "0.16.1", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/linux-arm64-gnu/package.json b/bindings/nodejs/npm/linux-arm64-gnu/package.json index f949104c..0cc18a2a 100644 --- a/bindings/nodejs/npm/linux-arm64-gnu/package.json +++ b/bindings/nodejs/npm/linux-arm64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-arm64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.0", + "version": "0.16.1", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/linux-x64-gnu/package.json b/bindings/nodejs/npm/linux-x64-gnu/package.json index 693fc1bf..ad920d45 100644 --- a/bindings/nodejs/npm/linux-x64-gnu/package.json +++ b/bindings/nodejs/npm/linux-x64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-x64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.0", + "version": "0.16.1", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/win32-x64-msvc/package.json b/bindings/nodejs/npm/win32-x64-msvc/package.json index 01dd2f7a..839cea7a 100644 --- a/bindings/nodejs/npm/win32-x64-msvc/package.json +++ b/bindings/nodejs/npm/win32-x64-msvc/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-win32-x64-msvc", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.0", + "version": "0.16.1", "os": [ "win32" ], diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index 413637ed..ef13fa14 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "databend-driver", "author": "Databend Authors ", - "version": "0.16.0", + "version": "0.16.1", "license": "Apache-2.0", "main": "index.js", "types": "index.d.ts", diff --git a/scripts/bump.sh b/scripts/bump.sh index 728ca7eb..57c58c1d 100755 --- a/scripts/bump.sh +++ b/scripts/bump.sh @@ -25,6 +25,12 @@ fi echo "Bumping version to $VERSION" +git status +git checkout main +git fetch upstream +git rebase upstream/main +git checkout -b "bump-$VERSION" + if [[ "$OSTYPE" == "darwin"* ]]; then sed -i '' "s/version = \".*\"/version = \"$VERSION\"/g" Cargo.toml sed -i '' "s/\"version\": \".*\"/\"version\": \"$VERSION\"/g" bindings/nodejs/package.json @@ -35,11 +41,6 @@ else sed -i "s/\"version\": \".*\"/\"version\": \"$VERSION\"/g" bindings/nodejs/npm/*/package.json fi -git status -git checkout main -git fetch upstream -git rebase upstream/main -git checkout -b "bump-$VERSION" git status git add Cargo.toml bindings/nodejs/package.json bindings/nodejs/npm/*/package.json git commit -m "chore: bump version to $VERSION" From 1e744f8f61f08eeee0ac5e9371d5f83e402d145b Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 11 Apr 2024 14:42:28 +0800 Subject: [PATCH 10/52] chore: ignore return code for bendsql test (#382) --- cli/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/test.sh b/cli/test.sh index b4bfe260..0321a8f6 100755 --- a/cli/test.sh +++ b/cli/test.sh @@ -47,10 +47,10 @@ for tf in cli/tests/*.{sql,sh}; do echo " Running test -- ${tf}" if [[ $tf == *.sh ]]; then suite=$(basename "${tf}" | sed -e 's#.sh##') - bash "${tf}" >"cli/tests/${suite}.output" 2>&1 + bash "${tf}" >"cli/tests/${suite}.output" 2>&1 || true elif [[ $tf == *.sql ]]; then suite=$(basename "${tf}" | sed -e 's#.sql##') - "${BENDSQL}" --output tsv <"${tf}" >"cli/tests/${suite}.output" 2>&1 + "${BENDSQL}" --output tsv <"${tf}" >"cli/tests/${suite}.output" 2>&1 || true fi diff "cli/tests/${suite}.output" "cli/tests/${suite}.result" done @@ -61,10 +61,10 @@ for tf in cli/tests/"$TEST_HANDLER"/*.{sql,sh}; do echo " Running test -- ${tf}" if [[ $tf == *.sh ]]; then suite=$(basename "${tf}" | sed -e 's#.sh##') - bash "${tf}" >"cli/tests/${TEST_HANDLER}/${suite}.output" 2>&1 + bash "${tf}" >"cli/tests/${TEST_HANDLER}/${suite}.output" 2>&1 || true elif [[ $tf == *.sql ]]; then suite=$(basename "${tf}" | sed -e 's#.sql##') - "${BENDSQL}" --output tsv <"${tf}" >"cli/tests/${TEST_HANDLER}/${suite}.output" 2>&1 + "${BENDSQL}" --output tsv <"${tf}" >"cli/tests/${TEST_HANDLER}/${suite}.output" 2>&1 || true fi diff "cli/tests/${TEST_HANDLER}/${suite}.output" "cli/tests/${TEST_HANDLER}/${suite}.result" done From 062a1899490cabd1c0a9d4268c0a9fe6ce2c97d2 Mon Sep 17 00:00:00 2001 From: baishen Date: Thu, 11 Apr 2024 16:45:55 +0800 Subject: [PATCH 11/52] feat: support decode nested types from string to values (#381) --- cli/tests/00-base.result | 1 + cli/tests/00-base.sql | 2 + driver/tests/driver/select_simple.rs | 103 ++++++- sql/Cargo.toml | 3 + sql/src/cursor_ext/cursor_checkpoint_ext.rs | 33 ++ sql/src/cursor_ext/cursor_read_bytes_ext.rs | 171 +++++++++++ sql/src/cursor_ext/cursor_read_number_ext.rs | 150 +++++++++ sql/src/cursor_ext/cursor_read_string_ext.rs | 78 +++++ sql/src/cursor_ext/mod.rs | 25 ++ sql/src/lib.rs | 1 + sql/src/schema.rs | 48 ++- sql/src/value.rs | 305 ++++++++++++++++++- 12 files changed, 899 insertions(+), 21 deletions(-) create mode 100644 sql/src/cursor_ext/cursor_checkpoint_ext.rs create mode 100644 sql/src/cursor_ext/cursor_read_bytes_ext.rs create mode 100644 sql/src/cursor_ext/cursor_read_number_ext.rs create mode 100644 sql/src/cursor_ext/cursor_read_string_ext.rs create mode 100644 sql/src/cursor_ext/mod.rs diff --git a/cli/tests/00-base.result b/cli/tests/00-base.result index c6b8b920..4962e053 100644 --- a/cli/tests/00-base.result +++ b/cli/tests/00-base.result @@ -20,4 +20,5 @@ Asia/Shanghai NULL {'k1':'v1','k2':'v2'} (2,NULL) 1 NULL 1 ab NULL v1 2 NULL +{'k1':'v1','k2':'v2'} [6162,78797A] ('[1,2]','SRID=4326;POINT(1 2)','2024-04-10') bye diff --git a/cli/tests/00-base.sql b/cli/tests/00-base.sql index 14c9a2aa..e07c3270 100644 --- a/cli/tests/00-base.sql +++ b/cli/tests/00-base.sql @@ -46,6 +46,8 @@ insert into test_nested values([1,2,3], null, (1, 'ab')), (null, {'k1':'v1', 'k2 select * from test_nested; select a[1], b['k1'], c:x, c:y from test_nested; +select {'k1':'v1','k2':'v2'}, [to_binary('ab'), to_binary('xyz')], (parse_json('[1,2]'), st_geometryfromwkt('SRID=4326;POINT(1.0 2.0)'), to_date('2024-04-10')); + select 'bye'; drop table test; drop table test_decimal; diff --git a/driver/tests/driver/select_simple.rs b/driver/tests/driver/select_simple.rs index 4a927a7c..25c62c52 100644 --- a/driver/tests/driver/select_simple.rs +++ b/driver/tests/driver/select_simple.rs @@ -177,16 +177,99 @@ async fn select_nullable_u64() { assert_eq!(val, Some(4950)); } -// TODO:(everpcoc) parse to real array -// #[tokio::test] -// async fn select_array() { -// let mut conn = prepare().await; -// let row = conn.query_row("select [1, 2, 3, 4, 5]").await.unwrap(); -// assert!(row.is_some()); -// let row = row.unwrap(); -// let (val,): (String,) = row.try_into().unwrap(); -// assert_eq!(val, "[1,2,3,4,5]"); -// } +#[tokio::test] +async fn select_array() { + let conn = prepare().await; + let row = conn + .query_row("select [], [1, 2, 3, 4, 5], [10::Decimal(15,2), 1.1+2.3], [to_binary('xyz')]") + .await + .unwrap(); + assert!(row.is_some()); + let row = row.unwrap(); + assert_eq!( + row.values().to_owned(), + vec![ + Value::EmptyArray, + Value::Array(vec![ + Value::Number(NumberValue::UInt8(1)), + Value::Number(NumberValue::UInt8(2)), + Value::Number(NumberValue::UInt8(3)), + Value::Number(NumberValue::UInt8(4)), + Value::Number(NumberValue::UInt8(5)), + ]), + Value::Array(vec![ + Value::Number(NumberValue::Decimal128( + 1000, + DecimalSize { + precision: 4, + scale: 2 + } + )), + Value::Number(NumberValue::Decimal128( + 340, + DecimalSize { + precision: 4, + scale: 2 + } + )) + ]), + Value::Array(vec![Value::Binary(vec![120, 121, 122])]), + ] + ); +} + +#[tokio::test] +async fn select_map() { + let conn = prepare().await; + let row = conn + .query_row("select {}, {'k1':'v1','k2':'v2'}, {'xx':to_date('2020-01-01')}") + .await + .unwrap(); + assert!(row.is_some()); + let row = row.unwrap(); + assert_eq!( + row.values().to_owned(), + vec![ + Value::EmptyMap, + Value::Map(vec![ + ( + Value::String("k1".to_string()), + Value::String("v1".to_string()) + ), + ( + Value::String("k2".to_string()), + Value::String("v2".to_string()) + ), + ]), + Value::Map(vec![(Value::String("xx".to_string()), Value::Date(18262)),]), + ] + ); +} + +#[tokio::test] +async fn select_tuple() { + let conn = prepare().await; + let row = conn.query_row("select (parse_json('[1,2]'), [1,2], true), (st_geometryfromwkt('SRID=4126;POINT(3.0 5.0)'), to_timestamp('2024-10-22 10:11:12'))").await.unwrap(); + assert!(row.is_some()); + let row = row.unwrap(); + assert_eq!( + row.values().to_owned(), + vec![ + Value::Tuple(vec![ + Value::Variant("[1,2]".to_string()), + Value::Array(vec![ + Value::Number(NumberValue::UInt8(1)), + Value::Number(NumberValue::UInt8(2)), + ]), + Value::Boolean(true), + ]), + Value::Tuple(vec![ + Value::Geometry("SRID=4126;POINT(3 5)".to_string()), + Value::Timestamp(1729591872000000) + ]), + ] + ); +} #[tokio::test] async fn select_multiple_columns() { diff --git a/sql/Cargo.toml b/sql/Cargo.toml index 304e4afa..c7c390a6 100644 --- a/sql/Cargo.toml +++ b/sql/Cargo.toml @@ -16,11 +16,14 @@ flight-sql = ["dep:arrow-array", "dep:arrow-schema", "dep:tonic"] [dependencies] databend-client = { workspace = true } +memchr = "2.7.2" chrono = { version = "0.4.35", default-features = false } geozero = { version = "0.12.0", features = ["default", "with-wkb"] } glob = "0.3" itertools = "0.12" jsonb = "0.3" +lexical-core = "0.8.5" + roaring = { version = "0.10", features = ["serde"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false, features = ["std"] } diff --git a/sql/src/cursor_ext/cursor_checkpoint_ext.rs b/sql/src/cursor_ext/cursor_checkpoint_ext.rs new file mode 100644 index 00000000..a6309ae2 --- /dev/null +++ b/sql/src/cursor_ext/cursor_checkpoint_ext.rs @@ -0,0 +1,33 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Cursor; + +pub trait ReadCheckPointExt { + fn checkpoint(&self) -> u64; + fn rollback(&mut self, checkpoint: u64); +} + +impl ReadCheckPointExt for Cursor +where + T: AsRef<[u8]>, +{ + fn checkpoint(&self) -> u64 { + self.position() + } + + fn rollback(&mut self, checkpoint: u64) { + self.set_position(checkpoint) + } +} diff --git a/sql/src/cursor_ext/cursor_read_bytes_ext.rs b/sql/src/cursor_ext/cursor_read_bytes_ext.rs new file mode 100644 index 00000000..10926ed5 --- /dev/null +++ b/sql/src/cursor_ext/cursor_read_bytes_ext.rs @@ -0,0 +1,171 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use memchr::memchr; +use std::io::BufRead; +use std::io::Cursor; +use std::io::ErrorKind; +use std::io::Result; + +pub trait ReadBytesExt { + fn peek(&mut self) -> Option; + fn peek_byte(&mut self) -> Option; + fn ignore(&mut self, f: impl Fn(u8) -> bool) -> bool; + fn ignores(&mut self, f: impl Fn(u8) -> bool) -> usize; + fn ignore_byte(&mut self, b: u8) -> bool; + fn ignore_bytes(&mut self, bs: &[u8]) -> bool; + fn ignore_white_spaces(&mut self) -> bool; + fn until(&mut self, delim: u8, buf: &mut Vec) -> usize; + fn keep_read(&mut self, buf: &mut Vec, f: impl Fn(u8) -> bool) -> usize; + fn must_ignore(&mut self, f: impl Fn(u8) -> bool) -> Result<()> { + if !self.ignore(f) { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "Expected to ignore a byte", + )); + } + Ok(()) + } + + fn must_ignore_byte(&mut self, b: u8) -> Result<()>; +} + +impl ReadBytesExt for Cursor +where + T: AsRef<[u8]>, +{ + fn peek(&mut self) -> Option { + let buf = self.fill_buf().ok()?; + if buf.is_empty() { + None + } else { + Some(buf[0] as char) + } + } + + fn peek_byte(&mut self) -> Option { + let buf = self.fill_buf().ok()?; + if buf.is_empty() { + None + } else { + Some(buf[0]) + } + } + + fn ignore(&mut self, f: impl Fn(u8) -> bool) -> bool { + match self.fill_buf() { + Ok(available) => { + if available.is_empty() { + false + } else if f(available[0]) { + self.consume(1); + true + } else { + false + } + } + Err(_) => false, + } + } + + fn ignores(&mut self, f: impl Fn(u8) -> bool) -> usize { + match self.fill_buf() { + Ok(available) => { + if available.is_empty() { + return 0; + } + for (index, byt) in available.iter().enumerate() { + if !f(*byt) { + self.consume(index); + return index; + } + } + let len = available.len(); + self.consume(len); + len + } + Err(_) => 0, + } + } + + fn ignore_byte(&mut self, b: u8) -> bool { + self.ignore(|c| c == b) + } + + fn ignore_bytes(&mut self, bs: &[u8]) -> bool { + match self.fill_buf() { + Ok(available) => { + let len = bs.len(); + if available.len() < len { + return false; + } + let eq = available[..len].iter().zip(bs).all(|(x, y)| x == y); + if eq { + self.consume(len); + } + eq + } + Err(_) => false, + } + } + + fn must_ignore_byte(&mut self, b: u8) -> Result<()> { + if !self.ignore_byte(b) { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + format!( + "Expected to have char '{}', got '{:?}' at pos {}", + b as char, + self.peek(), + self.position() + ), + )); + } + Ok(()) + } + + fn ignore_white_spaces(&mut self) -> bool { + self.ignores(|c| c.is_ascii_whitespace()) > 0 + } + + fn until(&mut self, delim: u8, buf: &mut Vec) -> usize { + match self.fill_buf() { + Ok(remaining_slice) => { + let to_read = memchr(delim, remaining_slice).map_or(buf.len(), |n| n + 1); + buf.extend_from_slice(&remaining_slice[..to_read]); + self.consume(to_read); + to_read + } + Err(_) => 0, + } + } + + fn keep_read(&mut self, buf: &mut Vec, f: impl Fn(u8) -> bool) -> usize { + match self.fill_buf() { + Ok(remaining_slice) => { + let mut to_read = remaining_slice.len(); + for (i, b) in remaining_slice.iter().enumerate() { + if !f(*b) { + to_read = i; + break; + } + } + buf.extend_from_slice(&remaining_slice[..to_read]); + self.consume(to_read); + to_read + } + Err(_) => 0, + } + } +} diff --git a/sql/src/cursor_ext/cursor_read_number_ext.rs b/sql/src/cursor_ext/cursor_read_number_ext.rs new file mode 100644 index 00000000..43f71fd4 --- /dev/null +++ b/sql/src/cursor_ext/cursor_read_number_ext.rs @@ -0,0 +1,150 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::BufRead; +use std::io::Cursor; + +use std::io::ErrorKind; +use std::io::Result; + +use lexical_core::FromLexical; + +pub trait ReadNumberExt { + fn read_int_text(&mut self) -> Result; + fn read_float_text(&mut self) -> Result; +} + +pub fn collect_binary_number(buffer: &[u8]) -> usize { + let mut index = 0; + let len = buffer.len(); + + for _ in 0..len { + match buffer[index] { + b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' => { + index += 1; + } + _ => { + break; + } + } + } + index +} + +pub fn collect_number(buffer: &[u8]) -> (usize, usize) { + let mut has_number = false; + let mut index = 0; + let len = buffer.len(); + let mut point_pos = len; + + for _ in 0..len { + match buffer[index] { + b'0'..=b'9' => { + has_number = true; + } + + b'-' | b'+' => { + if has_number { + break; + } + } + b'.' => { + point_pos = index; + index += 1; + break; + } + _ => { + break; + } + } + index += 1; + } + if point_pos < len { + while index < len && buffer[index].is_ascii_digit() { + index += 1; + } + } + + let mut is_scientific = false; + if has_number && index < len && (buffer[index] == b'e' || buffer[index] == b'E') { + is_scientific = true; + index += 1; + if index < len && (buffer[index] == b'-' || buffer[index] == b'+') { + index += 1 + } + while index < len && buffer[index].is_ascii_digit() { + index += 1; + } + } + + let effective = if !is_scientific + && point_pos < len + && buffer[point_pos + 1..index].iter().all(|x| *x == b'0') + { + point_pos + } else { + index + }; + (index, effective) +} + +#[inline] +fn read_num_text_exact(buf: &[u8]) -> Result { + match FromLexical::from_lexical(buf) { + Ok(value) => Ok(value), + Err(cause) => Err(std::io::Error::new( + ErrorKind::InvalidData, + format!( + "Cannot parse value:{:?} to number type, cause: {:?}", + String::from_utf8_lossy(buf), + cause + ), + )), + } +} + +impl ReadNumberExt for Cursor +where + B: AsRef<[u8]>, +{ + fn read_int_text(&mut self) -> Result { + let buf = self.fill_buf()?; + let (n_in, n_out) = collect_number(buf); + if n_in == 0 { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "Failed to parse number from text: input does not contain a valid numeric format." + .to_string(), + )); + } + let n = read_num_text_exact(&buf[..n_out])?; + self.consume(n_in); + Ok(n) + } + + fn read_float_text(&mut self) -> Result { + let buf = self.fill_buf()?; + let (n_in, n_out) = collect_number(buf); + if n_in == 0 { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "Unable to parse float: provided text is not in a recognizable floating-point format.".to_string() + )); + } + let buf = self.fill_buf()?; + let n = read_num_text_exact(&buf[..n_out])?; + self.consume(n_in); + Ok(n) + } +} diff --git a/sql/src/cursor_ext/cursor_read_string_ext.rs b/sql/src/cursor_ext/cursor_read_string_ext.rs new file mode 100644 index 00000000..97cdba48 --- /dev/null +++ b/sql/src/cursor_ext/cursor_read_string_ext.rs @@ -0,0 +1,78 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::BufRead; +use std::io::Cursor; +use std::io::ErrorKind; +use std::io::Result; + +use crate::cursor_ext::cursor_read_bytes_ext::ReadBytesExt; + +pub trait BufferReadStringExt { + fn read_quoted_text(&mut self, buf: &mut Vec, quota: u8) -> Result<()>; +} + +impl BufferReadStringExt for Cursor +where + T: AsRef<[u8]>, +{ + fn read_quoted_text(&mut self, buf: &mut Vec, quote: u8) -> Result<()> { + self.must_ignore_byte(quote)?; + + loop { + self.keep_read(buf, |b| b != quote && b != b'\\'); + if self.ignore_byte(quote) { + if self.peek_byte() == Some(quote) { + buf.push(quote); + self.consume(1); + } else { + return Ok(()); + } + } else if self.ignore_byte(b'\\') { + let b = self.fill_buf()?; + if b.is_empty() { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "Expected to have terminated string literal after escaped char '\' ." + .to_string(), + )); + } + let c = b[0]; + self.ignore_byte(c); + + match c { + b'n' => buf.push(b'\n'), + b't' => buf.push(b'\t'), + b'r' => buf.push(b'\r'), + b'0' => buf.push(b'\0'), + b'\'' => buf.push(b'\''), + b'\\' => buf.push(b'\\'), + _ => { + buf.push(b'\\'); + buf.push(c); + } + } + } else { + break; + } + } + Err(std::io::Error::new( + ErrorKind::InvalidData, + format!( + "Expected to have terminated string literal after quota {:?}, while consumed buf: {:?}", + quote as char, buf + ), + )) + } +} diff --git a/sql/src/cursor_ext/mod.rs b/sql/src/cursor_ext/mod.rs new file mode 100644 index 00000000..6991d408 --- /dev/null +++ b/sql/src/cursor_ext/mod.rs @@ -0,0 +1,25 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod cursor_checkpoint_ext; +mod cursor_read_bytes_ext; +mod cursor_read_number_ext; +mod cursor_read_string_ext; + +pub use cursor_checkpoint_ext::ReadCheckPointExt; +pub use cursor_read_bytes_ext::ReadBytesExt; +pub use cursor_read_number_ext::collect_binary_number; +pub use cursor_read_number_ext::collect_number; +pub use cursor_read_number_ext::ReadNumberExt; +pub use cursor_read_string_ext::BufferReadStringExt; diff --git a/sql/src/lib.rs b/sql/src/lib.rs index f1937426..9ce252ec 100644 --- a/sql/src/lib.rs +++ b/sql/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod cursor_ext; pub mod error; pub mod from_row; pub mod rows; diff --git a/sql/src/schema.rs b/sql/src/schema.rs index 7a28263d..a310402e 100644 --- a/sql/src/schema.rs +++ b/sql/src/schema.rs @@ -131,7 +131,12 @@ impl std::fmt::Display for DataType { DataType::Date => write!(f, "Date"), DataType::Nullable(inner) => write!(f, "Nullable({})", inner), DataType::Array(inner) => write!(f, "Array({})", inner), - DataType::Map(inner) => write!(f, "Map({})", inner), + DataType::Map(inner) => match inner.as_ref() { + DataType::Tuple(tys) => { + write!(f, "Map({}, {})", tys[0], tys[1]) + } + _ => unreachable!(), + }, DataType::Tuple(inner) => { let inner = inner .iter() @@ -173,7 +178,19 @@ impl TryFrom<&TypeDesc<'_>> for DataType { fn try_from(desc: &TypeDesc) -> Result { let dt = match desc.name { - "Null" | "NULL" => DataType::Null, + "Null" | "NULL" => { + if desc.args.is_empty() { + DataType::Null + } else { + if desc.args.len() != 1 { + return Err(Error::Parsing( + "Nullable type must have one argument".to_string(), + )); + } + let inner = Self::try_from(&desc.args[0])?; + DataType::Nullable(Box::new(inner)) + } + } "Boolean" => DataType::Boolean, "Binary" => DataType::Binary, "String" => DataType::String, @@ -423,6 +440,10 @@ fn parse_type_desc(s: &str) -> Result { } ' ' => { if depth == 0 { + let s = &s[start..i]; + if !s.is_empty() { + args.push(parse_type_desc(s)?); + } start = i + 1; } } @@ -566,6 +587,29 @@ mod test { ], }, }, + TestCase { + desc: "map nullable value args", + input: "Nullable(Map(String, String NULL))", + output: TypeDesc { + name: "Nullable", + args: vec![TypeDesc { + name: "Map", + args: vec![ + TypeDesc { + name: "String", + args: vec![], + }, + TypeDesc { + name: "NULL", + args: vec![TypeDesc { + name: "String", + args: vec![], + }], + }, + ], + }], + }, + }, ]; for case in test_cases { let output = parse_type_desc(case.input).unwrap(); diff --git a/sql/src/value.rs b/sql/src/value.rs index 7f8ea27e..3f4b9ab2 100644 --- a/sql/src/value.rs +++ b/sql/src/value.rs @@ -12,9 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::io::BufRead; +use std::io::Cursor; + use arrow::datatypes::{i256, ArrowNativeTypeOp}; use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime}; +use crate::cursor_ext::{ + collect_binary_number, collect_number, BufferReadStringExt, ReadBytesExt, ReadCheckPointExt, + ReadNumberExt, +}; + use crate::{ error::{ConvertError, Error, Result}, schema::{DecimalDataType, DecimalSize}, @@ -28,6 +36,8 @@ use std::fmt::Write; // Thu 1970-01-01 is R.D. 719163 const DAYS_FROM_CE: i32 = 719_163; const NULL_VALUE: &str = "NULL"; +const TRUE_VALUE: &str = "1"; +const FALSE_VALUE: &str = "0"; #[cfg(feature = "flight-sql")] use { @@ -146,7 +156,6 @@ impl TryFrom<(&DataType, &str)> for Value { DataType::Boolean => Ok(Self::Boolean(v == "1")), DataType::Binary => Ok(Self::Binary(hex::decode(v)?)), DataType::String => Ok(Self::String(v.to_string())), - DataType::Number(NumberDataType::Int8) => { Ok(Self::Number(NumberValue::Int8(v.parse()?))) } @@ -177,7 +186,6 @@ impl TryFrom<(&DataType, &str)> for Value { DataType::Number(NumberDataType::Float64) => { Ok(Self::Number(NumberValue::Float64(v.parse()?))) } - DataType::Decimal(DecimalDataType::Decimal128(size)) => { let d = parse_decimal(v, *size)?; Ok(Self::Number(d)) @@ -186,7 +194,6 @@ impl TryFrom<(&DataType, &str)> for Value { let d = parse_decimal(v, *size)?; Ok(Self::Number(d)) } - DataType::Timestamp => Ok(Self::Timestamp( chrono::NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S%.6f")? .and_utc() @@ -199,6 +206,11 @@ impl TryFrom<(&DataType, &str)> for Value { DataType::Variant => Ok(Self::Variant(v.to_string())), DataType::Geometry => Ok(Self::Geometry(v.to_string())), + DataType::Array(_) | DataType::Map(_) | DataType::Tuple(_) => { + let mut reader = Cursor::new(v); + let decoder = ValueDecoder {}; + decoder.read_field(t, &mut reader) + } DataType::Nullable(inner) => { if v == NULL_VALUE { Ok(Self::Null) @@ -206,9 +218,6 @@ impl TryFrom<(&DataType, &str)> for Value { Self::try_from((inner.as_ref(), v)) } } - - // TODO:(everpcpc) handle complex types - _ => Ok(Self::String(v.to_string())), } } } @@ -352,12 +361,12 @@ impl TryFrom<(&ArrowField, &Arc, usize)> for Value { } ArrowDataType::Binary => match array.as_any().downcast_ref::() { - Some(array) => Ok(Value::String(String::from_utf8(array.value(seq).to_vec())?)), + Some(array) => Ok(Value::Binary(array.value(seq).to_vec())), None => Err(ConvertError::new("binary", format!("{:?}", array)).into()), }, ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => { match array.as_any().downcast_ref::() { - Some(array) => Ok(Value::String(String::from_utf8(array.value(seq).to_vec())?)), + Some(array) => Ok(Value::Binary(array.value(seq).to_vec())), None => Err(ConvertError::new("large binary", format!("{:?}", array)).into()), } } @@ -621,7 +630,13 @@ fn encode_value(f: &mut std::fmt::Formatter<'_>, val: &Value, raw: bool) -> std: Value::Null => write!(f, "NULL"), Value::EmptyArray => write!(f, "[]"), Value::EmptyMap => write!(f, "{{}}"), - Value::Boolean(b) => write!(f, "{}", b), + Value::Boolean(b) => { + if *b { + write!(f, "true") + } else { + write!(f, "false") + } + } Value::Number(n) => write!(f, "{}", n), Value::Binary(s) => write!(f, "{}", hex::encode_upper(s)), Value::String(s) | Value::Bitmap(s) | Value::Variant(s) | Value::Geometry(s) => { @@ -826,3 +841,275 @@ pub fn parse_geometry(raw_data: &[u8]) -> Result { let wkt = Ewkt::from_wkb(&mut data, WkbDialect::Ewkb); wkt.map(|g| g.0).map_err(|e| e.into()) } + +struct ValueDecoder {} + +impl ValueDecoder { + fn read_field>(&self, ty: &DataType, reader: &mut Cursor) -> Result { + match ty { + DataType::Null => self.read_null(reader), + DataType::EmptyArray => self.read_empty_array(reader), + DataType::EmptyMap => self.read_empty_map(reader), + DataType::Boolean => self.read_bool(reader), + DataType::Number(NumberDataType::Int8) => self.read_int8(reader), + DataType::Number(NumberDataType::Int16) => self.read_int16(reader), + DataType::Number(NumberDataType::Int32) => self.read_int32(reader), + DataType::Number(NumberDataType::Int64) => self.read_int64(reader), + DataType::Number(NumberDataType::UInt8) => self.read_uint8(reader), + DataType::Number(NumberDataType::UInt16) => self.read_uint16(reader), + DataType::Number(NumberDataType::UInt32) => self.read_uint32(reader), + DataType::Number(NumberDataType::UInt64) => self.read_uint64(reader), + DataType::Number(NumberDataType::Float32) => self.read_float32(reader), + DataType::Number(NumberDataType::Float64) => self.read_float64(reader), + DataType::Decimal(DecimalDataType::Decimal128(size)) => self.read_decimal(size, reader), + DataType::Decimal(DecimalDataType::Decimal256(size)) => self.read_decimal(size, reader), + DataType::String => self.read_string(reader), + DataType::Binary => self.read_binary(reader), + DataType::Timestamp => self.read_timestamp(reader), + DataType::Date => self.read_date(reader), + DataType::Bitmap => self.read_bitmap(reader), + DataType::Variant => self.read_variant(reader), + DataType::Geometry => self.read_geometry(reader), + DataType::Array(inner_ty) => self.read_array(inner_ty.as_ref(), reader), + DataType::Map(inner_ty) => self.read_map(inner_ty.as_ref(), reader), + DataType::Tuple(inner_tys) => self.read_tuple(inner_tys.as_ref(), reader), + DataType::Nullable(inner_ty) => self.read_nullable(inner_ty.as_ref(), reader), + } + } + + fn match_bytes>(&self, reader: &mut Cursor, bs: &[u8]) -> bool { + let pos = reader.checkpoint(); + if reader.ignore_bytes(bs) { + true + } else { + reader.rollback(pos); + false + } + } + + fn read_null>(&self, reader: &mut Cursor) -> Result { + if self.match_bytes(reader, NULL_VALUE.as_bytes()) { + Ok(Value::Null) + } else { + let buf = reader.fill_buf()?; + Err(ConvertError::new("null", String::from_utf8_lossy(buf).to_string()).into()) + } + } + + fn read_bool>(&self, reader: &mut Cursor) -> Result { + if self.match_bytes(reader, TRUE_VALUE.as_bytes()) { + Ok(Value::Boolean(true)) + } else if self.match_bytes(reader, FALSE_VALUE.as_bytes()) { + Ok(Value::Boolean(false)) + } else { + let buf = reader.fill_buf()?; + Err(ConvertError::new("boolean", String::from_utf8_lossy(buf).to_string()).into()) + } + } + + fn read_int8>(&self, reader: &mut Cursor) -> Result { + let v: i8 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::Int8(v))) + } + + fn read_int16>(&self, reader: &mut Cursor) -> Result { + let v: i16 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::Int16(v))) + } + + fn read_int32>(&self, reader: &mut Cursor) -> Result { + let v: i32 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::Int32(v))) + } + + fn read_int64>(&self, reader: &mut Cursor) -> Result { + let v: i64 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::Int64(v))) + } + + fn read_uint8>(&self, reader: &mut Cursor) -> Result { + let v: u8 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::UInt8(v))) + } + + fn read_uint16>(&self, reader: &mut Cursor) -> Result { + let v: u16 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::UInt16(v))) + } + + fn read_uint32>(&self, reader: &mut Cursor) -> Result { + let v: u32 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::UInt32(v))) + } + + fn read_uint64>(&self, reader: &mut Cursor) -> Result { + let v: u64 = reader.read_int_text()?; + Ok(Value::Number(NumberValue::UInt64(v))) + } + + fn read_float32>(&self, reader: &mut Cursor) -> Result { + let v: f32 = reader.read_float_text()?; + Ok(Value::Number(NumberValue::Float32(v))) + } + + fn read_float64>(&self, reader: &mut Cursor) -> Result { + let v: f64 = reader.read_float_text()?; + Ok(Value::Number(NumberValue::Float64(v))) + } + + fn read_decimal>( + &self, + size: &DecimalSize, + reader: &mut Cursor, + ) -> Result { + let buf = reader.fill_buf()?; + // parser decimal need fractional part. + // 10.00 and 10 is different value. + let (n_in, _) = collect_number(buf); + let v = unsafe { std::str::from_utf8_unchecked(&buf[..n_in]) }; + let d = parse_decimal(v, *size)?; + reader.consume(n_in); + Ok(Value::Number(d)) + } + + fn read_string>(&self, reader: &mut Cursor) -> Result { + let mut buf = Vec::new(); + reader.read_quoted_text(&mut buf, b'\'')?; + Ok(Value::String(unsafe { String::from_utf8_unchecked(buf) })) + } + + fn read_binary>(&self, reader: &mut Cursor) -> Result { + let buf = reader.fill_buf()?; + let n = collect_binary_number(buf); + let v = buf[..n].to_vec(); + reader.consume(n); + Ok(Value::Binary(hex::decode(v)?)) + } + + fn read_date>(&self, reader: &mut Cursor) -> Result { + let mut buf = Vec::new(); + reader.read_quoted_text(&mut buf, b'\'')?; + let v = unsafe { std::str::from_utf8_unchecked(&buf) }; + let days = + chrono::NaiveDate::parse_from_str(v, "%Y-%m-%d")?.num_days_from_ce() - DAYS_FROM_CE; + Ok(Value::Date(days)) + } + + fn read_timestamp>(&self, reader: &mut Cursor) -> Result { + let mut buf = Vec::new(); + reader.read_quoted_text(&mut buf, b'\'')?; + let v = unsafe { std::str::from_utf8_unchecked(&buf) }; + let ts = chrono::NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S%.6f")? + .and_utc() + .timestamp_micros(); + Ok(Value::Timestamp(ts)) + } + + fn read_bitmap>(&self, reader: &mut Cursor) -> Result { + let mut buf = Vec::new(); + reader.read_quoted_text(&mut buf, b'\'')?; + Ok(Value::Bitmap(unsafe { String::from_utf8_unchecked(buf) })) + } + + fn read_variant>(&self, reader: &mut Cursor) -> Result { + let mut buf = Vec::new(); + reader.read_quoted_text(&mut buf, b'\'')?; + Ok(Value::Variant(unsafe { String::from_utf8_unchecked(buf) })) + } + + fn read_geometry>(&self, reader: &mut Cursor) -> Result { + let mut buf = Vec::new(); + reader.read_quoted_text(&mut buf, b'\'')?; + Ok(Value::Geometry(unsafe { String::from_utf8_unchecked(buf) })) + } + + fn read_nullable>( + &self, + ty: &DataType, + reader: &mut Cursor, + ) -> Result { + match self.read_null(reader) { + Ok(val) => Ok(val), + Err(_) => self.read_field(ty, reader), + } + } + + fn read_empty_array>(&self, reader: &mut Cursor) -> Result { + reader.must_ignore_byte(b'[')?; + reader.must_ignore_byte(b']')?; + Ok(Value::EmptyArray) + } + + fn read_empty_map>(&self, reader: &mut Cursor) -> Result { + reader.must_ignore_byte(b'{')?; + reader.must_ignore_byte(b'}')?; + Ok(Value::EmptyArray) + } + + fn read_array>(&self, ty: &DataType, reader: &mut Cursor) -> Result { + let mut vals = Vec::new(); + reader.must_ignore_byte(b'[')?; + for idx in 0.. { + let _ = reader.ignore_white_spaces(); + if reader.ignore_byte(b']') { + break; + } + if idx != 0 { + reader.must_ignore_byte(b',')?; + } + let _ = reader.ignore_white_spaces(); + let val = self.read_field(ty, reader)?; + vals.push(val); + } + Ok(Value::Array(vals)) + } + + fn read_map>(&self, ty: &DataType, reader: &mut Cursor) -> Result { + const KEY: usize = 0; + const VALUE: usize = 1; + let mut kvs = Vec::new(); + reader.must_ignore_byte(b'{')?; + match ty { + DataType::Tuple(inner_tys) => { + for idx in 0.. { + let _ = reader.ignore_white_spaces(); + if reader.ignore_byte(b'}') { + break; + } + if idx != 0 { + reader.must_ignore_byte(b',')?; + } + let _ = reader.ignore_white_spaces(); + let key = self.read_field(&inner_tys[KEY], reader)?; + let _ = reader.ignore_white_spaces(); + reader.must_ignore_byte(b':')?; + let _ = reader.ignore_white_spaces(); + let val = self.read_field(&inner_tys[VALUE], reader)?; + kvs.push((key, val)); + } + Ok(Value::Map(kvs)) + } + _ => unreachable!(), + } + } + + fn read_tuple>( + &self, + tys: &[DataType], + reader: &mut Cursor, + ) -> Result { + let mut vals = Vec::new(); + reader.must_ignore_byte(b'(')?; + for (idx, ty) in tys.iter().enumerate() { + let _ = reader.ignore_white_spaces(); + if idx != 0 { + reader.must_ignore_byte(b',')?; + } + let _ = reader.ignore_white_spaces(); + let val = self.read_field(ty, reader)?; + vals.push(val); + } + reader.must_ignore_byte(b')')?; + Ok(Value::Tuple(vals)) + } +} From d4f8b6b750eacd8853e378e64d1fc46242cbf7d5 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 11 Apr 2024 17:00:29 +0800 Subject: [PATCH 12/52] chore: bump version to 0.16.2 (#383) --- Cargo.toml | 10 +++++----- bindings/nodejs/npm/darwin-arm64/package.json | 2 +- bindings/nodejs/npm/darwin-x64/package.json | 2 +- bindings/nodejs/npm/linux-arm64-gnu/package.json | 2 +- bindings/nodejs/npm/linux-x64-gnu/package.json | 2 +- bindings/nodejs/npm/win32-x64-msvc/package.json | 2 +- bindings/nodejs/package.json | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7a1ecd62..ec820cf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.16.1" +version = "0.16.2" edition = "2021" license = "Apache-2.0" authors = ["Databend Authors "] @@ -21,7 +21,7 @@ keywords = ["databend", "database"] repository = "https://github.com/datafuselabs/bendsql" [workspace.dependencies] -databend-client = { path = "core", version = "0.16.1" } -databend-driver = { path = "driver", version = "0.16.1" } -databend-driver-macros = { path = "macros", version = "0.16.1" } -databend-sql = { path = "sql", version = "0.16.1" } +databend-client = { path = "core", version = "0.16.2" } +databend-driver = { path = "driver", version = "0.16.2" } +databend-driver-macros = { path = "macros", version = "0.16.2" } +databend-sql = { path = "sql", version = "0.16.2" } diff --git a/bindings/nodejs/npm/darwin-arm64/package.json b/bindings/nodejs/npm/darwin-arm64/package.json index 3d56b072..861d48e1 100644 --- a/bindings/nodejs/npm/darwin-arm64/package.json +++ b/bindings/nodejs/npm/darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-arm64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.1", + "version": "0.16.2", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/darwin-x64/package.json b/bindings/nodejs/npm/darwin-x64/package.json index aacb2315..f5f7ccac 100644 --- a/bindings/nodejs/npm/darwin-x64/package.json +++ b/bindings/nodejs/npm/darwin-x64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-x64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.1", + "version": "0.16.2", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/linux-arm64-gnu/package.json b/bindings/nodejs/npm/linux-arm64-gnu/package.json index 0cc18a2a..cacbb9dd 100644 --- a/bindings/nodejs/npm/linux-arm64-gnu/package.json +++ b/bindings/nodejs/npm/linux-arm64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-arm64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.1", + "version": "0.16.2", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/linux-x64-gnu/package.json b/bindings/nodejs/npm/linux-x64-gnu/package.json index ad920d45..0949b49b 100644 --- a/bindings/nodejs/npm/linux-x64-gnu/package.json +++ b/bindings/nodejs/npm/linux-x64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-x64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.1", + "version": "0.16.2", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/win32-x64-msvc/package.json b/bindings/nodejs/npm/win32-x64-msvc/package.json index 839cea7a..f6e977d1 100644 --- a/bindings/nodejs/npm/win32-x64-msvc/package.json +++ b/bindings/nodejs/npm/win32-x64-msvc/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-win32-x64-msvc", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.1", + "version": "0.16.2", "os": [ "win32" ], diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index ef13fa14..8d79a827 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "databend-driver", "author": "Databend Authors ", - "version": "0.16.1", + "version": "0.16.2", "license": "Apache-2.0", "main": "index.js", "types": "index.d.ts", From f8fdb97ac53c8a8dcc36d424e3642c2048b7acec Mon Sep 17 00:00:00 2001 From: everpcpc Date: Fri, 12 Apr 2024 17:36:37 +0800 Subject: [PATCH 13/52] feat: impl TryFrom for HashMap, Vec, Tuple (#384) --- deny.toml | 27 +++-- driver/tests/driver/select_simple.rs | 161 +++++++++++++++------------ sql/Cargo.toml | 8 +- sql/src/from_row.rs | 78 ------------- sql/src/lib.rs | 1 - sql/src/rows.rs | 64 ++++++++++- sql/src/value.rs | 112 +++++++++++++++++++ 7 files changed, 286 insertions(+), 165 deletions(-) delete mode 100644 sql/src/from_row.rs diff --git a/deny.toml b/deny.toml index 64cb8972..dbe2de22 100644 --- a/deny.toml +++ b/deny.toml @@ -1,26 +1,31 @@ [advisories] +version = 2 db-path = "~/.cargo/advisory-db" db-urls = ["https://github.com/rustsec/advisory-db"] -vulnerability = "deny" -unmaintained = "warn" -yanked = "warn" -notice = "warn" ignore = [ #"RUSTSEC-0000-0000", ] [licenses] -unlicensed = "warn" +version = 2 allow = [ - "MIT", - "CC0-1.0", + "Apache-2.0 WITH LLVM-exception", + "Apache-2.0", "BSD-3-Clause", - "Unicode-DFS-2016", + "BSL-1.0", + "CC0-1.0", "ISC", + "MIT", "MPL-2.0", - "BSL-1.0", - "Apache-2.0", - "Apache-2.0 WITH LLVM-exception", + "OpenSSL", + "Unicode-DFS-2016", +] + +[[licenses.clarify]] +name = "ring" +expression = "MIT AND ISC AND OpenSSL" +license-files = [ + { path = "LICENSE", hash = 0xbd0eed23 } ] [bans] diff --git a/driver/tests/driver/select_simple.rs b/driver/tests/driver/select_simple.rs index 25c62c52..f44c72fd 100644 --- a/driver/tests/driver/select_simple.rs +++ b/driver/tests/driver/select_simple.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::assert_eq; +use std::collections::HashMap; use chrono::{DateTime, NaiveDate, NaiveDateTime}; use databend_driver::{Client, Connection, DecimalSize, NumberValue, Value}; @@ -180,94 +181,116 @@ async fn select_nullable_u64() { #[tokio::test] async fn select_array() { let conn = prepare().await; - let row = conn - .query_row("select [], [1, 2, 3, 4, 5], [10::Decimal(15,2), 1.1+2.3], [to_binary('xyz')]") + + let row1 = conn.query_row("select []").await.unwrap().unwrap(); + let (val1,): (Vec,) = row1.try_into().unwrap(); + assert_eq!(val1, Vec::::new()); + + let row2 = conn + .query_row("select [1, 2, 3, 4, 5]") .await + .unwrap() .unwrap(); - assert!(row.is_some()); - let row = row.unwrap(); - assert_eq!( - row.values().to_owned(), - vec![ - Value::EmptyArray, - Value::Array(vec![ - Value::Number(NumberValue::UInt8(1)), - Value::Number(NumberValue::UInt8(2)), - Value::Number(NumberValue::UInt8(3)), - Value::Number(NumberValue::UInt8(4)), - Value::Number(NumberValue::UInt8(5)), - ]), - Value::Array(vec![ - Value::Number(NumberValue::Decimal128( - 1000, - DecimalSize { - precision: 4, - scale: 2 - } - )), - Value::Number(NumberValue::Decimal128( - 340, - DecimalSize { - precision: 4, - scale: 2 - } - )) - ]), - Value::Array(vec![Value::Binary(vec![120, 121, 122])]), - ] - ); + let (val2,): (Vec,) = row2.try_into().unwrap(); + assert_eq!(val2, vec![1, 2, 3, 4, 5]); + + let row3 = conn + .query_row("select [10::Decimal(15,2), 1.1+2.3]") + .await + .unwrap() + .unwrap(); + let (val3,): (Vec,) = row3.try_into().unwrap(); + assert_eq!(val3, vec!["10.00".to_string(), "3.40".to_string()]); + + let row4 = conn + .query_row("select [to_binary('xyz')]") + .await + .unwrap() + .unwrap(); + let (val4,): (Vec>,) = row4.try_into().unwrap(); + assert_eq!(val4, vec![vec![120, 121, 122]]); } #[tokio::test] async fn select_map() { let conn = prepare().await; - let row = conn - .query_row("select {}, {'k1':'v1','k2':'v2'}, {'xx':to_date('2020-01-01')}") + + let row1 = conn.query_row("select {}").await.unwrap().unwrap(); + let (val1,): (HashMap,) = row1.try_into().unwrap(); + assert_eq!(val1, HashMap::new()); + + let row2 = conn + .query_row("select {'k1':'v1','k2':'v2'}") .await + .unwrap() .unwrap(); - assert!(row.is_some()); - let row = row.unwrap(); + let (val2,): (HashMap,) = row2.try_into().unwrap(); assert_eq!( - row.values().to_owned(), + val2, vec![ - Value::EmptyMap, - Value::Map(vec![ - ( - Value::String("k1".to_string()), - Value::String("v1".to_string()) - ), - ( - Value::String("k2".to_string()), - Value::String("v2".to_string()) - ), - ]), - Value::Map(vec![(Value::String("xx".to_string()), Value::Date(18262)),]), + ("k1".to_string(), "v1".to_string()), + ("k2".to_string(), "v2".to_string()) ] + .into_iter() + .collect() + ); + + let row3 = conn + .query_row("select {'xx':to_date('2020-01-01')}") + .await + .unwrap() + .unwrap(); + let (val3,): (HashMap,) = row3.try_into().unwrap(); + assert_eq!( + val3, + vec![( + "xx".to_string(), + NaiveDate::from_ymd_opt(2020, 1, 1).unwrap() + )] + .into_iter() + .collect() + ); + + let row4 = conn + .query_row("select {1: 'a', 2: 'b'}") + .await + .unwrap() + .unwrap(); + let (val4,): (HashMap,) = row4.try_into().unwrap(); + assert_eq!( + val4, + vec![(1, "a".to_string()), (2, "b".to_string())] + .into_iter() + .collect() ); } #[tokio::test] async fn select_tuple() { let conn = prepare().await; - let row = conn.query_row("select (parse_json('[1,2]'), [1,2], true), (st_geometryfromwkt('SRID=4126;POINT(3.0 5.0)'), to_timestamp('2024-10-22 10:11:12'))").await.unwrap(); - assert!(row.is_some()); - let row = row.unwrap(); + + let row1 = conn + .query_row("select (parse_json('[1,2]'), [1,2], true)") + .await + .unwrap() + .unwrap(); + let (val1,): ((String, Vec, bool),) = row1.try_into().unwrap(); + assert_eq!(val1, ("[1,2]".to_string(), vec![1, 2], true,)); + + let row2 = conn + .query_row("select (st_geometryfromwkt('SRID=4126;POINT(3.0 5.0)'), to_timestamp('2024-10-22 10:11:12'))") + .await + .unwrap() + .unwrap(); + let (val2,): ((String, NaiveDateTime),) = row2.try_into().unwrap(); assert_eq!( - row.values().to_owned(), - vec![ - Value::Tuple(vec![ - Value::Variant("[1,2]".to_string()), - Value::Array(vec![ - Value::Number(NumberValue::UInt8(1)), - Value::Number(NumberValue::UInt8(2)), - ]), - Value::Boolean(true), - ]), - Value::Tuple(vec![ - Value::Geometry("SRID=4126;POINT(3 5)".to_string()), - Value::Timestamp(1729591872000000) - ]), - ] + val2, + ( + "SRID=4126;POINT(3 5)".to_string(), + DateTime::parse_from_rfc3339("2024-10-22T10:11:12Z") + .unwrap() + .naive_utc() + ) ); } diff --git a/sql/Cargo.toml b/sql/Cargo.toml index c7c390a6..395c4f51 100644 --- a/sql/Cargo.toml +++ b/sql/Cargo.toml @@ -16,13 +16,13 @@ flight-sql = ["dep:arrow-array", "dep:arrow-schema", "dep:tonic"] [dependencies] databend-client = { workspace = true } -memchr = "2.7.2" chrono = { version = "0.4.35", default-features = false } -geozero = { version = "0.12.0", features = ["default", "with-wkb"] } +geozero = { version = "0.12", features = ["default", "with-wkb"] } glob = "0.3" itertools = "0.12" jsonb = "0.3" -lexical-core = "0.8.5" +lexical-core = "0.8" +memchr = "2.7" roaring = { version = "0.10", features = ["serde"] } serde = { version = "1.0", default-features = false, features = ["derive"] } @@ -33,6 +33,7 @@ url = { version = "2.5", default-features = false } arrow = { version = "47.0" } arrow-array = { version = "47.0", optional = true } arrow-schema = { version = "47.0", optional = true } +hex = "0.4.3" tonic = { version = "0.10", default-features = false, features = [ "transport", "codegen", @@ -40,4 +41,3 @@ tonic = { version = "0.10", default-features = false, features = [ "tls-webpki-roots", "prost", ], optional = true } -hex = "0.4.3" diff --git a/sql/src/from_row.rs b/sql/src/from_row.rs deleted file mode 100644 index 8b267335..00000000 --- a/sql/src/from_row.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::error::Result; -use crate::rows::Row; -use crate::value::Value; - -macro_rules! replace_expr { - ($_t:tt $sub:expr) => { - $sub - }; -} - -// This macro implements TryFrom for tuple of types -macro_rules! impl_tuple_from_row { - ( $($Ti:tt),+ ) => { - impl<$($Ti),+> TryFrom for ($($Ti,)+) - where - $($Ti: TryFrom),+ - { - type Error = String; - fn try_from(row: Row) -> Result { - // It is not possible yet to get the number of metavariable repetitions - // ref: https://github.com/rust-lang/lang-team/issues/28#issue-644523674 - // This is a workaround - let expected_len = <[()]>::len(&[$(replace_expr!(($Ti) ())),*]); - - if expected_len != row.len() { - return Err(format!("row size mismatch: expected {} columns, got {}", expected_len, row.len())); - } - let mut vals_iter = row.into_iter().enumerate(); - - Ok(( - $( - { - let (col_ix, col_value) = vals_iter - .next() - .unwrap(); // vals_iter size is checked before this code is reached, - // so it is safe to unwrap - let t = col_value.get_type(); - $Ti::try_from(col_value) - .map_err(|_| format!("failed converting column {} from type({:?}) to type({})", col_ix, t, std::any::type_name::<$Ti>()))? - } - ,)+ - )) - } - } - } -} - -// Implement FromRow for tuples of size up to 16 -impl_tuple_from_row!(T1); -impl_tuple_from_row!(T1, T2); -impl_tuple_from_row!(T1, T2, T3); -impl_tuple_from_row!(T1, T2, T3, T4); -impl_tuple_from_row!(T1, T2, T3, T4, T5); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15); -impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16); diff --git a/sql/src/lib.rs b/sql/src/lib.rs index 9ce252ec..05f9a349 100644 --- a/sql/src/lib.rs +++ b/sql/src/lib.rs @@ -14,7 +14,6 @@ mod cursor_ext; pub mod error; -pub mod from_row; pub mod rows; pub mod schema; pub mod value; diff --git a/sql/src/rows.rs b/sql/src/rows.rs index 2ddf199e..953f3802 100644 --- a/sql/src/rows.rs +++ b/sql/src/rows.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::pin::Pin; -use std::sync::Arc; use std::task::Context; use std::task::Poll; @@ -149,7 +148,7 @@ impl TryFrom for Rows { rows.push(Row(values)); } Ok(Self { - schema: Arc::new(schema.try_into()?), + schema: std::sync::Arc::new(schema.try_into()?), rows, }) } @@ -182,6 +181,67 @@ impl Rows { } } +macro_rules! replace_expr { + ($_t:tt $sub:expr) => { + $sub + }; +} + +// This macro implements TryFrom for tuple of types +macro_rules! impl_tuple_from_row { + ( $($Ti:tt),+ ) => { + impl<$($Ti),+> TryFrom for ($($Ti,)+) + where + $($Ti: TryFrom),+ + { + type Error = String; + fn try_from(row: Row) -> Result { + // It is not possible yet to get the number of metavariable repetitions + // ref: https://github.com/rust-lang/lang-team/issues/28#issue-644523674 + // This is a workaround + let expected_len = <[()]>::len(&[$(replace_expr!(($Ti) ())),*]); + + if expected_len != row.len() { + return Err(format!("row size mismatch: expected {} columns, got {}", expected_len, row.len())); + } + let mut vals_iter = row.into_iter().enumerate(); + + Ok(( + $( + { + let (col_ix, col_value) = vals_iter + .next() + .unwrap(); // vals_iter size is checked before this code is reached, + // so it is safe to unwrap + let t = col_value.get_type(); + $Ti::try_from(col_value) + .map_err(|_| format!("failed converting column {} from type({:?}) to type({})", col_ix, t, std::any::type_name::<$Ti>()))? + } + ,)+ + )) + } + } + } +} + +// Implement FromRow for tuples of size up to 16 +impl_tuple_from_row!(T1); +impl_tuple_from_row!(T1, T2); +impl_tuple_from_row!(T1, T2, T3); +impl_tuple_from_row!(T1, T2, T3, T4); +impl_tuple_from_row!(T1, T2, T3, T4, T5); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15); +impl_tuple_from_row!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16); + pub struct RowIterator { schema: SchemaRef, it: Pin> + Send>>, diff --git a/sql/src/value.rs b/sql/src/value.rs index 3f4b9ab2..13a3b3b9 100644 --- a/sql/src/value.rs +++ b/sql/src/value.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; +use std::hash::Hash; use std::io::BufRead; use std::io::Cursor; @@ -472,6 +474,9 @@ impl TryFrom for String { match val { Value::String(s) => Ok(s), Value::Bitmap(s) => Ok(s), + Value::Number(NumberValue::Decimal128(v, s)) => Ok(display_decimal_128(v, s.scale)), + Value::Number(NumberValue::Decimal256(v, s)) => Ok(display_decimal_256(v, s.scale)), + Value::Geometry(s) => Ok(s), Value::Variant(s) => Ok(s), _ => Err(ConvertError::new("string", format!("{:?}", val)).into()), } @@ -563,6 +568,113 @@ impl TryFrom for NaiveDate { } } +impl TryFrom for Vec +where + V: TryFrom, +{ + type Error = Error; + fn try_from(val: Value) -> Result { + match val { + Value::Binary(vals) => vals + .into_iter() + .map(|v| V::try_from(Value::Number(NumberValue::UInt8(v)))) + .collect(), + Value::Array(vals) => vals.into_iter().map(V::try_from).collect(), + Value::EmptyArray => Ok(vec![]), + _ => Err(ConvertError::new("Vec", format!("{}", val)).into()), + } + } +} + +impl TryFrom for HashMap +where + K: TryFrom + Eq + Hash, + V: TryFrom, +{ + type Error = Error; + fn try_from(val: Value) -> Result { + match val { + Value::Map(kvs) => { + let mut map = HashMap::new(); + for (k, v) in kvs { + let k = K::try_from(k)?; + let v = V::try_from(v)?; + map.insert(k, v); + } + Ok(map) + } + Value::EmptyMap => Ok(HashMap::new()), + _ => Err(ConvertError::new("HashMap", format!("{}", val)).into()), + } + } +} + +macro_rules! replace_expr { + ($_t:tt $sub:expr) => { + $sub + }; +} + +// This macro implements TryFrom for tuple of types +macro_rules! impl_tuple_from_value { + ( $($Ti:tt),+ ) => { + impl<$($Ti),+> TryFrom for ($($Ti,)+) + where + $($Ti: TryFrom),+ + { + type Error = String; + fn try_from(val: Value) -> Result { + // It is not possible yet to get the number of metavariable repetitions + // ref: https://github.com/rust-lang/lang-team/issues/28#issue-644523674 + // This is a workaround + let expected_len = <[()]>::len(&[$(replace_expr!(($Ti) ())),*]); + + match val { + Value::Tuple(vals) => { + if expected_len != vals.len() { + return Err(format!("value tuple size mismatch: expected {} columns, got {}", expected_len, vals.len())); + } + let mut vals_iter = vals.into_iter().enumerate(); + + Ok(( + $( + { + let (col_ix, col_value) = vals_iter + .next() + .unwrap(); // vals_iter size is checked before this code is reached, + // so it is safe to unwrap + let t = col_value.get_type(); + $Ti::try_from(col_value) + .map_err(|_| format!("failed converting column {} from type({:?}) to type({})", col_ix, t, std::any::type_name::<$Ti>()))? + } + ,)+ + )) + } + _ => Err(format!("expected tuple, got {:?}", val)), + } + } + } + } +} + +// Implement From Value for tuples of size up to 16 +impl_tuple_from_value!(T1); +impl_tuple_from_value!(T1, T2); +impl_tuple_from_value!(T1, T2, T3); +impl_tuple_from_value!(T1, T2, T3, T4); +impl_tuple_from_value!(T1, T2, T3, T4, T5); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15); +impl_tuple_from_value!(T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16); + // This macro implements TryFrom to Option for Nullable column macro_rules! impl_try_from_to_option { ($($t:ty),*) => { From c36e44ab118febf122813f9943b69bd5c60d74ef Mon Sep 17 00:00:00 2001 From: everpcpc Date: Fri, 12 Apr 2024 17:53:24 +0800 Subject: [PATCH 14/52] chore: bump version to 0.16.3 (#385) --- Cargo.toml | 10 +++++----- bindings/nodejs/npm/darwin-arm64/package.json | 2 +- bindings/nodejs/npm/darwin-x64/package.json | 2 +- bindings/nodejs/npm/linux-arm64-gnu/package.json | 2 +- bindings/nodejs/npm/linux-x64-gnu/package.json | 2 +- bindings/nodejs/npm/win32-x64-msvc/package.json | 2 +- bindings/nodejs/package.json | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ec820cf5..44c247a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.16.2" +version = "0.16.3" edition = "2021" license = "Apache-2.0" authors = ["Databend Authors "] @@ -21,7 +21,7 @@ keywords = ["databend", "database"] repository = "https://github.com/datafuselabs/bendsql" [workspace.dependencies] -databend-client = { path = "core", version = "0.16.2" } -databend-driver = { path = "driver", version = "0.16.2" } -databend-driver-macros = { path = "macros", version = "0.16.2" } -databend-sql = { path = "sql", version = "0.16.2" } +databend-client = { path = "core", version = "0.16.3" } +databend-driver = { path = "driver", version = "0.16.3" } +databend-driver-macros = { path = "macros", version = "0.16.3" } +databend-sql = { path = "sql", version = "0.16.3" } diff --git a/bindings/nodejs/npm/darwin-arm64/package.json b/bindings/nodejs/npm/darwin-arm64/package.json index 861d48e1..60deaab5 100644 --- a/bindings/nodejs/npm/darwin-arm64/package.json +++ b/bindings/nodejs/npm/darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-arm64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.2", + "version": "0.16.3", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/darwin-x64/package.json b/bindings/nodejs/npm/darwin-x64/package.json index f5f7ccac..a3ff9979 100644 --- a/bindings/nodejs/npm/darwin-x64/package.json +++ b/bindings/nodejs/npm/darwin-x64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-x64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.2", + "version": "0.16.3", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/linux-arm64-gnu/package.json b/bindings/nodejs/npm/linux-arm64-gnu/package.json index cacbb9dd..3e15a619 100644 --- a/bindings/nodejs/npm/linux-arm64-gnu/package.json +++ b/bindings/nodejs/npm/linux-arm64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-arm64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.2", + "version": "0.16.3", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/linux-x64-gnu/package.json b/bindings/nodejs/npm/linux-x64-gnu/package.json index 0949b49b..26bd72e6 100644 --- a/bindings/nodejs/npm/linux-x64-gnu/package.json +++ b/bindings/nodejs/npm/linux-x64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-x64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.2", + "version": "0.16.3", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/win32-x64-msvc/package.json b/bindings/nodejs/npm/win32-x64-msvc/package.json index f6e977d1..2afbcdc1 100644 --- a/bindings/nodejs/npm/win32-x64-msvc/package.json +++ b/bindings/nodejs/npm/win32-x64-msvc/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-win32-x64-msvc", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.2", + "version": "0.16.3", "os": [ "win32" ], diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index 8d79a827..979f80a9 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "databend-driver", "author": "Databend Authors ", - "version": "0.16.2", + "version": "0.16.3", "license": "Apache-2.0", "main": "index.js", "types": "index.d.ts", From b7bc39ebee2121f170cdfb437ff8705e632669fd Mon Sep 17 00:00:00 2001 From: everpcpc Date: Mon, 15 Apr 2024 11:14:59 +0800 Subject: [PATCH 15/52] fix(cli): display text for rows speed (#386) --- cli/src/display.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/src/display.rs b/cli/src/display.rs index 85b37ee4..9283865c 100644 --- a/cli/src/display.rs +++ b/cli/src/display.rs @@ -298,9 +298,14 @@ impl<'a> FormatDisplay<'a> { stats.write_bytes, ), }; + let mut rows_speed_str = rows_str; if rows <= 1 { rows_str = rows_str.trim_end_matches('s'); } + let rows_speed = total_rows as f64 / self.start.elapsed().as_secs_f64(); + if rows_speed <= 1.0 { + rows_speed_str = rows_speed_str.trim_end_matches('s'); + } eprintln!( "{} {} {} in {:.3} sec. Processed {} {}, {} ({} {}/s, {}/s)", rows, @@ -310,8 +315,8 @@ impl<'a> FormatDisplay<'a> { humanize_count(total_rows as f64), rows_str, HumanBytes(total_bytes as u64), - humanize_count(total_rows as f64 / self.start.elapsed().as_secs_f64()), - rows_str, + humanize_count(rows_speed), + rows_speed_str, HumanBytes((total_bytes as f64 / self.start.elapsed().as_secs_f64()) as u64), ); eprintln!(); From 7c7fa94cf2026ce163a1a1aee8c19ba0ca250176 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 10:34:15 +0800 Subject: [PATCH 16/52] feat(bindings/nodejs): support nested types Array,Map,Tuple (#388) --- bindings/nodejs/src/lib.rs | 37 ++++++++++++++++++++++++++------ bindings/nodejs/tests/binding.js | 17 ++++++++++++--- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs index 8e470fff..8f4a4355 100644 --- a/bindings/nodejs/src/lib.rs +++ b/bindings/nodejs/src/lib.rs @@ -16,7 +16,7 @@ extern crate napi_derive; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; -use napi::bindgen_prelude::*; +use napi::{bindgen_prelude::*, Env}; use once_cell::sync::Lazy; use tokio_stream::StreamExt; @@ -71,10 +71,17 @@ pub struct Value(databend_driver::Value); impl ToNapiValue for Value { unsafe fn to_napi_value(env: sys::napi_env, val: Self) -> Result { + let ctx = Env::from(env); match val.0 { databend_driver::Value::Null => Null::to_napi_value(env, Null), - databend_driver::Value::EmptyArray => String::to_napi_value(env, "[]".to_string()), - databend_driver::Value::EmptyMap => String::to_napi_value(env, "{}".to_string()), + databend_driver::Value::EmptyArray => { + let arr = ctx.create_array(0)?; + Array::to_napi_value(env, arr) + } + databend_driver::Value::EmptyMap => { + let obj = ctx.create_object()?; + Object::to_napi_value(env, obj) + } databend_driver::Value::Boolean(b) => bool::to_napi_value(env, b), databend_driver::Value::Binary(b) => Buffer::to_napi_value(env, b.into()), databend_driver::Value::String(s) => String::to_napi_value(env, s), @@ -90,9 +97,27 @@ impl ToNapiValue for Value { NaiveDateTime::new(v, NaiveTime::from_hms_opt(0, 0, 0).unwrap()), ) } - databend_driver::Value::Array(_) => String::to_napi_value(env, format!("{}", val.0)), - databend_driver::Value::Map(_) => String::to_napi_value(env, format!("{}", val.0)), - databend_driver::Value::Tuple(_) => String::to_napi_value(env, format!("{}", val.0)), + databend_driver::Value::Array(inner) => { + let mut arr = ctx.create_array(inner.len() as u32)?; + for (i, v) in inner.into_iter().enumerate() { + arr.set(i as u32, Value(v))?; + } + Array::to_napi_value(env, arr) + } + databend_driver::Value::Map(inner) => { + let mut obj = ctx.create_object()?; + for (k, v) in inner.into_iter() { + obj.set(k.to_string(), Value(v))?; + } + Object::to_napi_value(env, obj) + } + databend_driver::Value::Tuple(inner) => { + let mut arr = ctx.create_array(inner.len() as u32)?; + for (i, v) in inner.into_iter().enumerate() { + arr.set(i as u32, Value(v))?; + } + Array::to_napi_value(env, arr) + } databend_driver::Value::Bitmap(s) => String::to_napi_value(env, s), databend_driver::Value::Variant(s) => String::to_napi_value(env, s), databend_driver::Value::Geometry(s) => String::to_napi_value(env, s), diff --git a/bindings/nodejs/tests/binding.js b/bindings/nodejs/tests/binding.js index d6b14e78..bbf68ff9 100644 --- a/bindings/nodejs/tests/binding.js +++ b/bindings/nodejs/tests/binding.js @@ -35,9 +35,20 @@ Then("Select string {string} should be equal to {string}", async function (input Then("Select types should be expected native types", async function () { // NumberValue::Decimal - const row = await this.conn.queryRow(`SELECT 15.7563::Decimal(8,4), 2.0+3.0`); - const excepted = ["15.7563", "5.0"]; - assert.deepEqual(row.values(), excepted); + const row1 = await this.conn.queryRow(`SELECT 15.7563::Decimal(8,4), 2.0+3.0`); + assert.deepEqual(row1.values(), ["15.7563", "5.0"]); + + // Array + const row2 = await this.conn.queryRow(`SELECT [10::Decimal(15,2), 1.1+2.3]`); + assert.deepEqual(row2.values(), [["10.00", "3.40"]]); + + // Map + const row3 = await this.conn.queryRow(`SELECT {'xx':to_date('2020-01-01')}`); + assert.deepEqual(row3.values(), [{ xx: new Date("2020-01-01") }]); + + // Tuple + const row4 = await this.conn.queryRow(`SELECT (10, '20', to_datetime('2024-04-16 12:34:56.789'))`); + assert.deepEqual(row4.values(), [[10, "20", new Date("2024-04-16T12:34:56.789Z")]]); }); Then("Select numbers should iterate all rows", async function () { From 9e8de918f660a4da6efd37a4c69a8a0f573bd629 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 11:16:15 +0800 Subject: [PATCH 17/52] feat(bindings/python): support nested types Array,Map,Tuple (#390) --- .github/actions/setup/action.yml | 4 ++-- .github/workflows/bindings.nodejs.yml | 12 +++++++----- .github/workflows/bindings.python.yml | 18 ++++++++++++++---- bindings/python/pyproject.toml | 12 +++++++----- bindings/python/tests/asyncio/steps/binding.py | 18 +++++++++++++++--- .../python/tests/blocking/steps/binding.py | 18 +++++++++++++++--- 6 files changed, 60 insertions(+), 22 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 899c710f..bcb03c95 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -20,10 +20,10 @@ runs: targets: ${{ inputs.target }} - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.4 - name: Cache Cargo - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo/registry/ diff --git a/.github/workflows/bindings.nodejs.yml b/.github/workflows/bindings.nodejs.yml index 4fd0bfdf..56c3b60b 100644 --- a/.github/workflows/bindings.nodejs.yml +++ b/.github/workflows/bindings.nodejs.yml @@ -26,7 +26,7 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup with: - cache-key: bindings-nodejs + cache-key: bindings-nodejs-integration - name: Setup node uses: actions/setup-node@v4 with: @@ -65,7 +65,7 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup with: - cache-key: bindings-nodejs + cache-key: bindings-nodejs-${{ matrix.os }}-${{ matrix.arch }} target: ${{ matrix.target }} - name: Setup node uses: actions/setup-node@v4 @@ -103,9 +103,9 @@ jobs: working-directory: bindings/nodejs run: | strip -x *.node - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: bindings-nodejs + name: bindings-nodejs-${{ matrix.os }}-${{ matrix.arch }} path: bindings/nodejs/*.node publish: @@ -131,9 +131,11 @@ jobs: working-directory: bindings/nodejs run: yarn install --immutable - name: Download all artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: bindings/nodejs/artifacts + pattern: bindings-nodejs-* + merge-multiple: true - name: Move artifacts shell: bash working-directory: bindings/nodejs diff --git a/.github/workflows/bindings.python.yml b/.github/workflows/bindings.python.yml index 72f6586c..59e45173 100644 --- a/.github/workflows/bindings.python.yml +++ b/.github/workflows/bindings.python.yml @@ -26,7 +26,7 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup with: - cache-key: bindings-python + cache-key: bindings-python-integration - name: Setup Python uses: actions/setup-python@v4 with: @@ -69,6 +69,11 @@ jobs: else echo "BUILD_ARGS=--release --strip --out dist" >> $GITHUB_OUTPUT fi + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + cache-key: bindings-python-${{ matrix.os }}-${{ matrix.arch }} + target: ${{ matrix.target }} - name: Build wheels uses: PyO3/maturin-action@v1 with: @@ -77,9 +82,9 @@ jobs: sccache: 'true' args: ${{ steps.opts.outputs.BUILD_ARGS }} - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: bindings-python-${{ matrix.os }}-${{ matrix.arch }} path: bindings/python/dist/*.whl publish: @@ -94,7 +99,12 @@ jobs: url: https://pypi.org/p/databend-driver/ steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 + with: + path: bindings/python/artifacts + pattern: bindings-python-* + merge-multiple: true - uses: pypa/gh-action-pypi-publish@release/v1 with: skip-existing: true + packages-dir: bindings/python/artifacts diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 48cd43fb..d1bd600d 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +build-backend = "maturin" +requires = ["maturin>=1.0,<2.0"] + [project] classifiers = [ "Programming Language :: Rust", @@ -8,7 +12,9 @@ description = "Databend Driver Python Binding" license = { text = "Apache-2.0" } name = "databend-driver" readme = "README.md" -requires-python = ">=3.7" +# PyO3 doesn't support python 3.13 yet. +# ref: https://github.com/apache/opendal/issues/4268 +requires-python = ">=3.7, < 3.13" [project.optional-dependencies] docs = ["pdoc"] @@ -22,7 +28,3 @@ Repository = "https://github.com/datafuselabs/bendsql" features = ["pyo3/extension-module"] module-name = "databend_driver._databend_driver" python-source = "package" - -[build-system] -build-backend = "maturin" -requires = ["maturin>=1.0,<2.0"] diff --git a/bindings/python/tests/asyncio/steps/binding.py b/bindings/python/tests/asyncio/steps/binding.py index eec63532..62cb1ddc 100644 --- a/bindings/python/tests/asyncio/steps/binding.py +++ b/bindings/python/tests/asyncio/steps/binding.py @@ -62,8 +62,21 @@ async def _(context, input, output): async def _(context): # NumberValue::Decimal row = await context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") - expected = (Decimal("15.7563"), Decimal("5.0")) - assert row.values() == expected + assert row.values() == (Decimal("15.7563"), Decimal("5.0")) + + # Array + row = await context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") + assert row.values() == ([Decimal("10.00"), Decimal("3.40")],) + + # Map + row = await context.conn.query_row("select {'xx':to_date('2020-01-01')}") + assert row.values() == ({"xx": "2020-01-01"},) + + # Tuple + row = await context.conn.query_row( + "select (10, '20', to_datetime('2024-04-16 12:34:56.789'))" + ) + assert row.values() == ((10, "20", "2024-04-16 12:34:56.789"),) @then("Select numbers should iterate all rows") @@ -121,5 +134,4 @@ async def _(context): (-2, 2, 2.0, "2", "2", "2012-05-31", "2012-05-31 11:20:00"), (-3, 3, 3.0, "3", "2", "2016-04-04", "2016-04-04 11:30:00"), ] - print("==>", ret) assert ret == expected diff --git a/bindings/python/tests/blocking/steps/binding.py b/bindings/python/tests/blocking/steps/binding.py index 67ccf27f..61ee9063 100644 --- a/bindings/python/tests/blocking/steps/binding.py +++ b/bindings/python/tests/blocking/steps/binding.py @@ -57,8 +57,21 @@ def _(context, input, output): async def _(context): # NumberValue::Decimal row = context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") - expected = (Decimal("15.7563"), Decimal("5.0")) - assert row.values() == expected + assert row.values() == (Decimal("15.7563"), Decimal("5.0")) + + # Array + row = context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") + assert row.values() == ([Decimal("10.00"), Decimal("3.40")],) + + # Map + row = context.conn.query_row("select {'xx':to_date('2020-01-01')}") + assert row.values() == ({"xx": "2020-01-01"},) + + # Tuple + row = context.conn.query_row( + "select (10, '20', to_datetime('2024-04-16 12:34:56.789'))" + ) + assert row.values() == ((10, "20", "2024-04-16 12:34:56.789"),) @then("Select numbers should iterate all rows") @@ -113,5 +126,4 @@ def _(context): (-2, 2, 2.0, "2", "2", "2012-05-31", "2012-05-31 11:20:00"), (-3, 3, 3.0, "3", "2", "2016-04-04", "2016-04-04 11:30:00"), ] - print("==>", ret) assert ret == expected From 167663e4aeb1dd1079afb3fc36ccc3d193795b84 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 12:32:16 +0800 Subject: [PATCH 18/52] chore(bindings/python): upgrade pyo3 to 0.21 (#392) --- Cargo.toml | 3 +++ bindings/python/Cargo.toml | 6 ++--- bindings/python/src/asyncio.rs | 18 ++++++------- bindings/python/src/lib.rs | 2 +- bindings/python/src/types.rs | 48 ++++++++++++++++------------------ deny.toml | 3 +++ 6 files changed, 41 insertions(+), 39 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 44c247a5..3c808fb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,6 @@ databend-client = { path = "core", version = "0.16.3" } databend-driver = { path = "driver", version = "0.16.3" } databend-driver-macros = { path = "macros", version = "0.16.3" } databend-sql = { path = "sql", version = "0.16.3" } + +[patch.crates-io] +pyo3-asyncio = { git = "https://github.com/everpcpc/pyo3-asyncio", rev = "42af887" } diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 61cfcf39..0042ff07 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -8,7 +8,7 @@ license = { workspace = true } authors = { workspace = true } [lib] -crate-type = ["cdylib", "rlib"] +crate-type = ["cdylib"] name = "databend_driver" doc = false @@ -16,7 +16,7 @@ doc = false ctor = "0.2.5" databend-driver = { workspace = true, features = ["rustls", "flight-sql"] } once_cell = "1.18" -pyo3 = { version = "0.20", features = ["abi3-py37"] } -pyo3-asyncio = { version = "0.20", features = ["tokio-runtime"] } +pyo3 = { version = "0.21", features = ["abi3-py37"] } +pyo3-asyncio = { version = "0.21", features = ["tokio-runtime"] } tokio = "1.34" tokio-stream = "0.1" diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs index 28725d21..d95c034e 100644 --- a/bindings/python/src/asyncio.rs +++ b/bindings/python/src/asyncio.rs @@ -30,7 +30,7 @@ impl AsyncDatabendClient { Ok(Self(client)) } - pub fn get_conn<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { + pub fn get_conn<'p>(&'p self, py: Python<'p>) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let conn = this.get_conn().await.map_err(DriverError::new)?; @@ -44,7 +44,7 @@ pub struct AsyncDatabendConnection(Box); #[pymethods] impl AsyncDatabendConnection { - pub fn info<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { + pub fn info<'p>(&'p self, py: Python<'p>) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let info = this.info().await; @@ -52,7 +52,7 @@ impl AsyncDatabendConnection { }) } - pub fn version<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { + pub fn version<'p>(&'p self, py: Python<'p>) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let version = this.version().await.map_err(DriverError::new)?; @@ -60,7 +60,7 @@ impl AsyncDatabendConnection { }) } - pub fn exec<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult<&'p PyAny> { + pub fn exec<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let res = this.exec(&sql).await.map_err(DriverError::new)?; @@ -68,7 +68,7 @@ impl AsyncDatabendConnection { }) } - pub fn query_row<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult<&'p PyAny> { + pub fn query_row<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let row = this.query_row(&sql).await.map_err(DriverError::new)?; @@ -76,7 +76,7 @@ impl AsyncDatabendConnection { }) } - pub fn query_all<'p>(&self, py: Python<'p>, sql: String) -> PyResult<&'p PyAny> { + pub fn query_all<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let rows: Vec = this @@ -90,7 +90,7 @@ impl AsyncDatabendConnection { }) } - pub fn query_iter<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult<&'p PyAny> { + pub fn query_iter<'p>(&'p self, py: Python<'p>, sql: String) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let streamer = this.query_iter(&sql).await.map_err(DriverError::new)?; @@ -99,11 +99,11 @@ impl AsyncDatabendConnection { } pub fn stream_load<'p>( - &self, + &'p self, py: Python<'p>, sql: String, data: Vec>, - ) -> PyResult<&'p PyAny> { + ) -> PyResult> { let this = self.0.clone(); future_into_py(py, async move { let data = data diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index d8f3b8e3..20111b94 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -24,7 +24,7 @@ use crate::blocking::{BlockingDatabendClient, BlockingDatabendConnection}; use crate::types::{ConnectionInfo, Field, Row, RowIterator, Schema, ServerStats}; #[pymodule] -fn _databend_driver(_py: Python, m: &PyModule) -> PyResult<()> { +fn _databend_driver(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs index 2884f4a4..6755433d 100644 --- a/bindings/python/src/types.rs +++ b/bindings/python/src/types.rs @@ -16,9 +16,10 @@ use std::sync::Arc; use once_cell::sync::Lazy; use pyo3::exceptions::{PyException, PyStopAsyncIteration, PyStopIteration}; +use pyo3::intern; +use pyo3::prelude::*; use pyo3::sync::GILOnceCell; use pyo3::types::{PyDict, PyList, PyTuple, PyType}; -use pyo3::{intern, prelude::*}; use pyo3_asyncio::tokio::future_into_py; use tokio::sync::Mutex; use tokio_stream::StreamExt; @@ -32,14 +33,14 @@ pub static VERSION: Lazy = Lazy::new(|| { pub static DECIMAL_CLS: GILOnceCell> = GILOnceCell::new(); -fn get_decimal_cls(py: Python<'_>) -> PyResult<&PyType> { +fn get_decimal_cls(py: Python<'_>) -> PyResult<&Bound> { DECIMAL_CLS .get_or_try_init(py, || { - py.import(intern!(py, "decimal"))? + py.import_bound(intern!(py, "decimal"))? .getattr(intern!(py, "Decimal"))? .extract() }) - .map(|ty| ty.as_ref(py)) + .map(|ty| ty.bind(py)) } pub struct Value(databend_driver::Value); @@ -49,11 +50,11 @@ impl IntoPy for Value { match self.0 { databend_driver::Value::Null => py.None(), databend_driver::Value::EmptyArray => { - let list = PyList::empty(py); + let list = PyList::empty_bound(py); list.into_py(py) } databend_driver::Value::EmptyMap => { - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); dict.into_py(py) } databend_driver::Value::Boolean(b) => b.into_py(py), @@ -72,11 +73,11 @@ impl IntoPy for Value { s.into_py(py) } databend_driver::Value::Array(inner) => { - let list = PyList::new(py, inner.into_iter().map(|v| Value(v).into_py(py))); + let list = PyList::new_bound(py, inner.into_iter().map(|v| Value(v).into_py(py))); list.into_py(py) } databend_driver::Value::Map(inner) => { - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); for (k, v) in inner { dict.set_item(Value(k).into_py(py), Value(v).into_py(py)) .unwrap(); @@ -84,7 +85,7 @@ impl IntoPy for Value { dict.into_py(py) } databend_driver::Value::Tuple(inner) => { - let tuple = PyTuple::new(py, inner.into_iter().map(|v| Value(v).into_py(py))); + let tuple = PyTuple::new_bound(py, inner.into_iter().map(|v| Value(v).into_py(py))); tuple.into_py(py) } databend_driver::Value::Bitmap(s) => s.into_py(py), @@ -138,12 +139,9 @@ impl Row { #[pymethods] impl Row { - pub fn values<'p>(&'p self, py: Python<'p>) -> PyResult { - let res = PyTuple::new( - py, - self.0.values().iter().map(|v| Value(v.clone()).into_py(py)), // FIXME: do not clone - ); - Ok(res.into_py(py)) + pub fn values<'p>(&'p self, py: Python<'p>) -> PyResult> { + let vals = self.0.values().iter().map(|v| Value(v.clone()).into_py(py)); + Ok(PyTuple::new_bound(py, vals)) } } @@ -158,7 +156,7 @@ impl RowIterator { #[pymethods] impl RowIterator { - fn schema<'p>(&self, py: Python) -> PyResult { + pub fn schema(&self, py: Python) -> PyResult { let streamer = self.0.clone(); let ret = wait_for_future(py, async move { let schema = streamer.lock().await.schema(); @@ -170,9 +168,9 @@ impl RowIterator { fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } - fn __next__(&self, py: Python) -> PyResult> { + fn __next__(&self, py: Python) -> PyResult { let streamer = self.0.clone(); - let ret = wait_for_future(py, async move { + wait_for_future(py, async move { match streamer.lock().await.next().await { Some(val) => match val { Err(e) => Err(PyException::new_err(format!("{}", e))), @@ -180,16 +178,15 @@ impl RowIterator { }, None => Err(PyStopIteration::new_err("The iterator is exhausted")), } - }); - ret.map(Some) + }) } fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } - fn __anext__(&self, py: Python<'_>) -> PyResult> { + fn __anext__<'p>(&'p self, py: Python<'p>) -> PyResult> { let streamer = self.0.clone(); - let future = future_into_py(py, async move { + future_into_py(py, async move { match streamer.lock().await.next().await { Some(val) => match val { Err(e) => Err(PyException::new_err(format!("{}", e))), @@ -197,8 +194,7 @@ impl RowIterator { }, None => Err(PyStopAsyncIteration::new_err("The iterator is exhausted")), } - }); - Ok(Some(future?.into())) + }) } } @@ -207,13 +203,13 @@ pub struct Schema(databend_driver::SchemaRef); #[pymethods] impl Schema { - pub fn fields<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { + pub fn fields<'p>(&'p self, py: Python<'p>) -> PyResult> { let fields = self .0 .fields() .into_iter() .map(|f| Field(f.clone()).into_py(py)); - Ok(PyList::new(py, fields)) + Ok(PyList::new_bound(py, fields)) } } diff --git a/deny.toml b/deny.toml index dbe2de22..510cf03e 100644 --- a/deny.toml +++ b/deny.toml @@ -35,3 +35,6 @@ highlight = "all" [sources] unknown-git = "deny" +allow-git = [ + "https://github.com/everpcpc/pyo3-asyncio.git", +] From cfa3ece53995cc1876d60a92a4975e2afc1ffe88 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 12:46:54 +0800 Subject: [PATCH 19/52] feat(bindings/python): convert Binary to python bytes (#393) --- bindings/python/src/types.rs | 7 +++++-- bindings/python/tests/asyncio/steps/binding.py | 4 ++++ bindings/python/tests/blocking/steps/binding.py | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs index 6755433d..824f3ffd 100644 --- a/bindings/python/src/types.rs +++ b/bindings/python/src/types.rs @@ -19,7 +19,7 @@ use pyo3::exceptions::{PyException, PyStopAsyncIteration, PyStopIteration}; use pyo3::intern; use pyo3::prelude::*; use pyo3::sync::GILOnceCell; -use pyo3::types::{PyDict, PyList, PyTuple, PyType}; +use pyo3::types::{PyBytes, PyDict, PyList, PyTuple, PyType}; use pyo3_asyncio::tokio::future_into_py; use tokio::sync::Mutex; use tokio_stream::StreamExt; @@ -58,7 +58,10 @@ impl IntoPy for Value { dict.into_py(py) } databend_driver::Value::Boolean(b) => b.into_py(py), - databend_driver::Value::Binary(b) => b.into_py(py), + databend_driver::Value::Binary(b) => { + let buf = PyBytes::new_bound(py, &b); + buf.into_py(py) + } databend_driver::Value::String(s) => s.into_py(py), databend_driver::Value::Number(n) => { let v = NumberValue(n); diff --git a/bindings/python/tests/asyncio/steps/binding.py b/bindings/python/tests/asyncio/steps/binding.py index 62cb1ddc..0ba230fb 100644 --- a/bindings/python/tests/asyncio/steps/binding.py +++ b/bindings/python/tests/asyncio/steps/binding.py @@ -64,6 +64,10 @@ async def _(context): row = await context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") assert row.values() == (Decimal("15.7563"), Decimal("5.0")) + # Binary + row = await context.conn.query_row("select to_binary('xyz')") + assert row.values() == (b"xyz",) + # Array row = await context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") assert row.values() == ([Decimal("10.00"), Decimal("3.40")],) diff --git a/bindings/python/tests/blocking/steps/binding.py b/bindings/python/tests/blocking/steps/binding.py index 61ee9063..42d632af 100644 --- a/bindings/python/tests/blocking/steps/binding.py +++ b/bindings/python/tests/blocking/steps/binding.py @@ -59,6 +59,10 @@ async def _(context): row = context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") assert row.values() == (Decimal("15.7563"), Decimal("5.0")) + # Binary + row = context.conn.query_row("select to_binary('xyz')") + assert row.values() == (b"xyz",) + # Array row = context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") assert row.values() == ([Decimal("10.00"), Decimal("3.40")],) From b0f1c456a6f6e5fdfcfb9c39a8bcaf4991b6129a Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 15:25:25 +0800 Subject: [PATCH 20/52] feat(bindings/python): support convert Date,Timestamp to datetime types (#389) --- bindings/python/Cargo.toml | 3 +- bindings/python/src/types.rs | 9 ++-- .../python/tests/asyncio/steps/binding.py | 45 +++++++++++-------- .../python/tests/blocking/steps/binding.py | 45 +++++++++++-------- sql/src/value.rs | 15 +++---- 5 files changed, 67 insertions(+), 50 deletions(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 0042ff07..ac5f2cf8 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -13,10 +13,11 @@ name = "databend_driver" doc = false [dependencies] +chrono = { version = "0.4.35", default-features = false } ctor = "0.2.5" databend-driver = { workspace = true, features = ["rustls", "flight-sql"] } once_cell = "1.18" -pyo3 = { version = "0.21", features = ["abi3-py37"] } +pyo3 = { version = "0.21", features = ["abi3-py37", "chrono"] } pyo3-asyncio = { version = "0.21", features = ["tokio-runtime"] } tokio = "1.34" tokio-stream = "0.1" diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs index 824f3ffd..dd71ad18 100644 --- a/bindings/python/src/types.rs +++ b/bindings/python/src/types.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use chrono::{NaiveDate, NaiveDateTime}; use once_cell::sync::Lazy; use pyo3::exceptions::{PyException, PyStopAsyncIteration, PyStopIteration}; use pyo3::intern; @@ -68,12 +69,12 @@ impl IntoPy for Value { v.into_py(py) } databend_driver::Value::Timestamp(_) => { - let s = self.0.to_string(); - s.into_py(py) + let t = NaiveDateTime::try_from(self.0).unwrap(); + t.into_py(py) } databend_driver::Value::Date(_) => { - let s = self.0.to_string(); - s.into_py(py) + let d = NaiveDate::try_from(self.0).unwrap(); + d.into_py(py) } databend_driver::Value::Array(inner) => { let list = PyList::new_bound(py, inner.into_iter().map(|v| Value(v).into_py(py))); diff --git a/bindings/python/tests/asyncio/steps/binding.py b/bindings/python/tests/asyncio/steps/binding.py index 0ba230fb..3c2924b6 100644 --- a/bindings/python/tests/asyncio/steps/binding.py +++ b/bindings/python/tests/asyncio/steps/binding.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from datetime import datetime, date from decimal import Decimal from behave import given, when, then @@ -24,7 +25,8 @@ @async_run_until_complete async def _(context): dsn = os.getenv( - "TEST_DATABEND_DSN", "databend+http://root:root@localhost:8000/?sslmode=disable" + "TEST_DATABEND_DSN", + "databend+http://root:root@localhost:8000/?sslmode=disable", ) client = databend_driver.AsyncDatabendClient(dsn) context.conn = await client.get_conn() @@ -54,7 +56,7 @@ async def _(context): async def _(context, input, output): row = await context.conn.query_row(f"SELECT '{input}'") value = row.values()[0] - assert output == value + assert output == value, f"output: {output}" @then("Select types should be expected native types") @@ -62,25 +64,32 @@ async def _(context, input, output): async def _(context): # NumberValue::Decimal row = await context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") - assert row.values() == (Decimal("15.7563"), Decimal("5.0")) + assert row.values() == ( + Decimal("15.7563"), + Decimal("5.0"), + ), f"Decimal: {row.values()}" # Binary row = await context.conn.query_row("select to_binary('xyz')") - assert row.values() == (b"xyz",) + assert row.values() == (b"xyz",), f"Binary: {row.values()}" # Array row = await context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") - assert row.values() == ([Decimal("10.00"), Decimal("3.40")],) + assert row.values() == ( + [Decimal("10.00"), Decimal("3.40")], + ), f"Array: {row.values()}" # Map row = await context.conn.query_row("select {'xx':to_date('2020-01-01')}") - assert row.values() == ({"xx": "2020-01-01"},) + assert row.values() == ({"xx": date(2020, 1, 1)},), f"Map: {row.values()}" # Tuple row = await context.conn.query_row( "select (10, '20', to_datetime('2024-04-16 12:34:56.789'))" ) - assert row.values() == ((10, "20", "2024-04-16 12:34:56.789"),) + assert row.values() == ( + (10, "20", datetime(2024, 4, 16, 12, 34, 56, 789000)), + ), f"Tuple: {row.values()}" @then("Select numbers should iterate all rows") @@ -91,7 +100,7 @@ async def _(context): async for row in rows: ret.append(row.values()[0]) expected = [0, 1, 2, 3, 4] - assert ret == expected + assert ret == expected, f"ret: {ret}" @then("Insert and Select should be equal") @@ -110,11 +119,11 @@ async def _(context): async for row in rows: ret.append(row.values()) expected = [ - (-1, 1, 1.0, "1", "1", "2011-03-06", "2011-03-06 06:20:00"), - (-2, 2, 2.0, "2", "2", "2012-05-31", "2012-05-31 11:20:00"), - (-3, 3, 3.0, "3", "2", "2016-04-04", "2016-04-04 11:30:00"), + (-1, 1, 1.0, "1", "1", date(2011, 3, 6), datetime(2011, 3, 6, 6, 20)), + (-2, 2, 2.0, "2", "2", date(2012, 5, 31), datetime(2012, 5, 31, 11, 20)), + (-3, 3, 3.0, "3", "2", date(2016, 4, 4), datetime(2016, 4, 4, 11, 30)), ] - assert ret == expected + assert ret == expected, f"ret: {ret}" @then("Stream load and Select should be equal") @@ -126,16 +135,16 @@ async def _(context): ["-3", "3", "3.0", "3", "2", "2016-04-04", "2016-04-04T11:30:00Z"], ] progress = await context.conn.stream_load("INSERT INTO test VALUES", values) - assert progress.write_rows == 3 - assert progress.write_bytes == 185 + assert progress.write_rows == 3, f"progress.write_rows: {progress.write_rows}" + assert progress.write_bytes == 185, f"progress.write_bytes: {progress.write_bytes}" rows = await context.conn.query_iter("SELECT * FROM test") ret = [] async for row in rows: ret.append(row.values()) expected = [ - (-1, 1, 1.0, "1", "1", "2011-03-06", "2011-03-06 06:20:00"), - (-2, 2, 2.0, "2", "2", "2012-05-31", "2012-05-31 11:20:00"), - (-3, 3, 3.0, "3", "2", "2016-04-04", "2016-04-04 11:30:00"), + (-1, 1, 1.0, "1", "1", date(2011, 3, 6), datetime(2011, 3, 6, 6, 20)), + (-2, 2, 2.0, "2", "2", date(2012, 5, 31), datetime(2012, 5, 31, 11, 20)), + (-3, 3, 3.0, "3", "2", date(2016, 4, 4), datetime(2016, 4, 4, 11, 30)), ] - assert ret == expected + assert ret == expected, f"ret: {ret}" diff --git a/bindings/python/tests/blocking/steps/binding.py b/bindings/python/tests/blocking/steps/binding.py index 42d632af..15c4353e 100644 --- a/bindings/python/tests/blocking/steps/binding.py +++ b/bindings/python/tests/blocking/steps/binding.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from datetime import datetime, date from decimal import Decimal from behave import given, when, then @@ -22,7 +23,8 @@ @given("A new Databend Driver Client") def _(context): dsn = os.getenv( - "TEST_DATABEND_DSN", "databend+http://root:root@localhost:8000/?sslmode=disable" + "TEST_DATABEND_DSN", + "databend+http://root:root@localhost:8000/?sslmode=disable", ) client = databend_driver.BlockingDatabendClient(dsn) context.conn = client.get_conn() @@ -50,32 +52,39 @@ def _(context): def _(context, input, output): row = context.conn.query_row(f"SELECT '{input}'") value = row.values()[0] - assert output == value + assert output == value, f"output: {output}" @then("Select types should be expected native types") async def _(context): # NumberValue::Decimal row = context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") - assert row.values() == (Decimal("15.7563"), Decimal("5.0")) + assert row.values() == ( + Decimal("15.7563"), + Decimal("5.0"), + ), f"Decimal: {row.values()}" # Binary row = context.conn.query_row("select to_binary('xyz')") - assert row.values() == (b"xyz",) + assert row.values() == (b"xyz",), f"Binary: {row.values()}" # Array row = context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") - assert row.values() == ([Decimal("10.00"), Decimal("3.40")],) + assert row.values() == ( + [Decimal("10.00"), Decimal("3.40")], + ), f"Array: {row.values()}" # Map row = context.conn.query_row("select {'xx':to_date('2020-01-01')}") - assert row.values() == ({"xx": "2020-01-01"},) + assert row.values() == ({"xx": date(2020, 1, 1)},), f"Map: {row.values()}" # Tuple row = context.conn.query_row( "select (10, '20', to_datetime('2024-04-16 12:34:56.789'))" ) - assert row.values() == ((10, "20", "2024-04-16 12:34:56.789"),) + assert row.values() == ( + (10, "20", datetime(2024, 4, 16, 12, 34, 56, 789000)), + ), f"Tuple: {row.values()}" @then("Select numbers should iterate all rows") @@ -85,7 +94,7 @@ def _(context): for row in rows: ret.append(row.values()[0]) expected = [0, 1, 2, 3, 4] - assert ret == expected + assert ret == expected, f"ret: {ret}" @then("Insert and Select should be equal") @@ -103,11 +112,11 @@ def _(context): for row in rows: ret.append(row.values()) expected = [ - (-1, 1, 1.0, "1", "1", "2011-03-06", "2011-03-06 06:20:00"), - (-2, 2, 2.0, "2", "2", "2012-05-31", "2012-05-31 11:20:00"), - (-3, 3, 3.0, "3", "2", "2016-04-04", "2016-04-04 11:30:00"), + (-1, 1, 1.0, "1", "1", date(2011, 3, 6), datetime(2011, 3, 6, 6, 20)), + (-2, 2, 2.0, "2", "2", date(2012, 5, 31), datetime(2012, 5, 31, 11, 20)), + (-3, 3, 3.0, "3", "2", date(2016, 4, 4), datetime(2016, 4, 4, 11, 30)), ] - assert ret == expected + assert ret == expected, f"ret: {ret}" @then("Stream load and Select should be equal") @@ -118,16 +127,16 @@ def _(context): ["-3", "3", "3.0", "3", "2", "2016-04-04", "2016-04-04T11:30:00Z"], ] progress = context.conn.stream_load("INSERT INTO test VALUES", values) - assert progress.write_rows == 3 - assert progress.write_bytes == 185 + assert progress.write_rows == 3, f"progress.write_rows: {progress.write_rows}" + assert progress.write_bytes == 185, f"progress.write_bytes: {progress.write_bytes}" rows = context.conn.query_iter("SELECT * FROM test") ret = [] for row in rows: ret.append(row.values()) expected = [ - (-1, 1, 1.0, "1", "1", "2011-03-06", "2011-03-06 06:20:00"), - (-2, 2, 2.0, "2", "2", "2012-05-31", "2012-05-31 11:20:00"), - (-3, 3, 3.0, "3", "2", "2016-04-04", "2016-04-04 11:30:00"), + (-1, 1, 1.0, "1", "1", date(2011, 3, 6), datetime(2011, 3, 6, 6, 20)), + (-2, 2, 2.0, "2", "2", date(2012, 5, 31), datetime(2012, 5, 31, 11, 20)), + (-3, 3, 3.0, "3", "2", date(2016, 4, 4), datetime(2016, 4, 4, 11, 30)), ] - assert ret == expected + assert ret == expected, f"ret: {ret}" diff --git a/sql/src/value.rs b/sql/src/value.rs index 13a3b3b9..68d2acc4 100644 --- a/sql/src/value.rs +++ b/sql/src/value.rs @@ -197,12 +197,12 @@ impl TryFrom<(&DataType, &str)> for Value { Ok(Self::Number(d)) } DataType::Timestamp => Ok(Self::Timestamp( - chrono::NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S%.6f")? + NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S%.6f")? .and_utc() .timestamp_micros(), )), DataType::Date => Ok(Self::Date( - chrono::NaiveDate::parse_from_str(v, "%Y-%m-%d")?.num_days_from_ce() - DAYS_FROM_CE, + NaiveDate::parse_from_str(v, "%Y-%m-%d")?.num_days_from_ce() - DAYS_FROM_CE, )), DataType::Bitmap => Ok(Self::Bitmap(v.to_string())), DataType::Variant => Ok(Self::Variant(v.to_string())), @@ -540,8 +540,7 @@ impl TryFrom for NaiveDateTime { Value::Timestamp(i) => { let secs = i / 1_000_000; let nanos = ((i % 1_000_000) * 1000) as u32; - let t = DateTime::from_timestamp(secs, nanos); - match t { + match DateTime::from_timestamp(secs, nanos) { Some(t) => Ok(t.naive_utc()), None => Err(ConvertError::new("NaiveDateTime", "".to_string()).into()), } @@ -557,8 +556,7 @@ impl TryFrom for NaiveDate { match val { Value::Date(i) => { let days = i + DAYS_FROM_CE; - let d = NaiveDate::from_num_days_from_ce_opt(days); - match d { + match NaiveDate::from_num_days_from_ce_opt(days) { Some(d) => Ok(d), None => Err(ConvertError::new("NaiveDate", "".to_string()).into()), } @@ -1102,8 +1100,7 @@ impl ValueDecoder { let mut buf = Vec::new(); reader.read_quoted_text(&mut buf, b'\'')?; let v = unsafe { std::str::from_utf8_unchecked(&buf) }; - let days = - chrono::NaiveDate::parse_from_str(v, "%Y-%m-%d")?.num_days_from_ce() - DAYS_FROM_CE; + let days = NaiveDate::parse_from_str(v, "%Y-%m-%d")?.num_days_from_ce() - DAYS_FROM_CE; Ok(Value::Date(days)) } @@ -1111,7 +1108,7 @@ impl ValueDecoder { let mut buf = Vec::new(); reader.read_quoted_text(&mut buf, b'\'')?; let v = unsafe { std::str::from_utf8_unchecked(&buf) }; - let ts = chrono::NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S%.6f")? + let ts = NaiveDateTime::parse_from_str(v, "%Y-%m-%d %H:%M:%S%.6f")? .and_utc() .timestamp_micros(); Ok(Value::Timestamp(ts)) From 26ec3ebc1cec6791fe5cee0049b2e1dbe1c99245 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 16:15:01 +0800 Subject: [PATCH 21/52] chore(docs): add type mappings (#394) --- bindings/nodejs/README.md | 32 ++++++++++++++++++++++++++++++++ bindings/python/README.md | 34 ++++++++++++++++++++++++++++++++-- driver/README.md | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) diff --git a/bindings/nodejs/README.md b/bindings/nodejs/README.md index 581530c1..ab23c053 100644 --- a/bindings/nodejs/README.md +++ b/bindings/nodejs/README.md @@ -36,6 +36,38 @@ while (row) { } ``` +## Type Mapping + +[Databend Types](https://docs.databend.com/sql/sql-reference/data-types/) + +### General Data Types + +| Databend | Node.js | +| ----------- | --------------- | +| `BOOLEAN` | `Boolean` | +| `TINYINT` | `Number` | +| `SMALLINT` | `Number` | +| `INT` | `Number` | +| `BIGINT` | `Number` | +| `FLOAT` | `Number` | +| `DOUBLE` | `Number` | +| `DECIMAL` | `String` | +| `DATE` | `Date` | +| `TIMESTAMP` | `Date` | +| `VARCHAR` | `String` | +| `BINARY` | `Array(Number)` | + +### Semi-Structured Data Types + +| Databend | Node.js | +| ---------- | -------- | +| `ARRAY` | `Array` | +| `TUPLE` | `Array` | +| `MAP` | `Object` | +| `VARIANT` | `String` | +| `BITMAP` | `String` | +| `GEOMETRY` | `String` | + ## Development ```shell diff --git a/bindings/python/README.md b/bindings/python/README.md index fedb4764..a736abfc 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -9,7 +9,6 @@ maturin develop ## Usage - ### Blocking ```python @@ -64,6 +63,38 @@ async def main(): asyncio.run(main()) ``` +## Type Mapping + +[Databend Types](https://docs.databend.com/sql/sql-reference/data-types/) + +### General Data Types + +| Databend | Python | +| ----------- | ------------------- | +| `BOOLEAN` | `bool` | +| `TINYINT` | `int` | +| `SMALLINT` | `int` | +| `INT` | `int` | +| `BIGINT` | `int` | +| `FLOAT` | `float` | +| `DOUBLE` | `float` | +| `DECIMAL` | `decimal.Decimal` | +| `DATE` | `datetime.date` | +| `TIMESTAMP` | `datetime.datetime` | +| `VARCHAR` | `str` | +| `BINARY` | `bytes` | + +### Semi-Structured Data Types + +| Databend | Python | +| ---------- | ------- | +| `ARRAY` | `list` | +| `TUPLE` | `tuple` | +| `MAP` | `dict` | +| `VARIANT` | `str` | +| `BITMAP` | `str` | +| `GEOMETRY` | `str` | + ## APIs ### AsyncDatabendClient @@ -106,7 +137,6 @@ class BlockingDatabendConnection: def stream_load(self, sql: str, data: list[list[str]]) -> ServerStats: ... ``` - ### Row ```python diff --git a/driver/README.md b/driver/README.md index d6d3a99d..aef74a2f 100644 --- a/driver/README.md +++ b/driver/README.md @@ -43,3 +43,35 @@ while let Some(row) = rows.next().await { println!("{} {} {}", title, author, date); } ``` + +## Type Mapping + +[Databend Types](https://docs.databend.com/sql/sql-reference/data-types/) + +### General Data Types + +| Databend | Rust | +| ----------- | ----------------------- | +| `BOOLEAN` | `bool` | +| `TINYINT` | `i8`,`u8` | +| `SMALLINT` | `i16`,`u16` | +| `INT` | `i32`,`u32` | +| `BIGINT` | `i64`,`u64` | +| `FLOAT` | `f32` | +| `DOUBLE` | `f64` | +| `DECIMAL` | `String` | +| `DATE` | `chrono::NaiveDate` | +| `TIMESTAMP` | `chrono::NaiveDateTime` | +| `VARCHAR` | `String` | +| `BINARY` | `Vec` | + +### Semi-Structured Data Types + +| Databend | Rust | +| ------------- | --------------- | +| `ARRAY[T]` | `Vec` | +| `TUPLE[T, U]` | `(T, U)` | +| `MAP[K, V]` | `HashMap` | +| `VARIANT` | `String` | +| `BITMAP` | `String` | +| `GEOMETRY` | `String` | From 6efd798481886a0c35d8b28de55f060da522dede Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 16:33:10 +0800 Subject: [PATCH 22/52] chore(core): deprecate dsn arg presigned_url_disabled (#395) --- README.md | 126 +++++++++++++++++++++------------------------ core/src/client.rs | 13 ----- 2 files changed, 59 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index ec5048a8..558465e8 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ Databend Native Client in Rust ## Components -- [**core**](core): Databend RestAPI rust client +- [**core**](core): Databend RestAPI Rust client -- [**driver**](driver): Databend unified SQL client for RestAPI and FlightSQL +- [**driver**](driver): Databend SQL client for both RestAPI and FlightSQL - [**cli**](cli): Databend native CLI - ## Installation for BendSQL ### Cargo: @@ -27,7 +26,6 @@ Or alternatively build from source: cargo install bendsql ``` - ### Homebrew: ```bash @@ -36,13 +34,13 @@ brew install databendcloud/homebrew-tap/bendsql ### Apt: -* Using DEB822-STYLE format on Ubuntu-22.04/Debian-12 and later: +- Using DEB822-STYLE format on Ubuntu-22.04/Debian-12 and later: ```bash sudo curl -L -o /etc/apt/sources.list.d/datafuselabs.sources https://repo.databend.rs/deb/datafuselabs.sources ``` -* Using old format on Ubuntu-20.04/Debian-11 and earlier: +- Using old format on Ubuntu-20.04/Debian-11 and earlier: ```bash sudo curl -L -o /usr/share/keyrings/datafuselabs-keyring.gpg https://repo.databend.rs/deb/datafuselabs.gpg @@ -61,7 +59,6 @@ sudo apt install bendsql Check for latest version on [GitHub Release](https://github.com/datafuselabs/bendsql/releases) - ## Usage ``` @@ -96,9 +93,10 @@ Options: ## Custom configuration By default bendsql will read configuration from `~/.bendsql/config.toml` and `~/.config/bendsql/config.toml` - sequentially if exists. +sequentially if exists. - Example file + ``` ❯ cat ~/.bendsql/config.toml [connection] @@ -113,50 +111,48 @@ prompt = ":) " ``` - - Connection section -| Parameter | Description | -|---|---| -| `host` | Server host to connect. | -| `port` | Server port to connect. | -| `user` | User name. | -| `database` | Which database to connect. | -| `args` | Additional connection args. | - +| Parameter | Description | +| ---------- | --------------------------- | +| `host` | Server host to connect. | +| `port` | Server port to connect. | +| `user` | User name. | +| `database` | Which database to connect. | +| `args` | Additional connection args. | - Settings section -| Parameter | Description | -|---|---| -| `display_pretty_sql` | Whether to display SQL queries in a formatted way. | -| `prompt` | The prompt to display before asking for input. | -| `progress_color` | The color to use for the progress bar. | -| `show_progress` | Whether to show a progress bar when executing queries. | -| `show_stats` | Whether to show statistics after executing queries. | -| `max_display_rows` | The maximum number of rows to display in table output format. | -| `max_width` | Limit display render box max width, 0 means default to the size of the terminal. | -| `max_col_width` | Limit display render each column max width, smaller than 3 means disable the limit. | -| `output_format` | The output format to use. | -| `expand` | Expand table format display, default off, could be on/off/auto. | -| `time` | Whether to show the time elapsed when executing queries. | -| `multi_line` | Whether to allow multi-line input. | -| `replace_newline` | whether replace '\n' with '\\\n'. | - +| Parameter | Description | +| -------------------- | ----------------------------------------------------------------------------------- | +| `display_pretty_sql` | Whether to display SQL queries in a formatted way. | +| `prompt` | The prompt to display before asking for input. | +| `progress_color` | The color to use for the progress bar. | +| `show_progress` | Whether to show a progress bar when executing queries. | +| `show_stats` | Whether to show statistics after executing queries. | +| `max_display_rows` | The maximum number of rows to display in table output format. | +| `max_width` | Limit display render box max width, 0 means default to the size of the terminal. | +| `max_col_width` | Limit display render each column max width, smaller than 3 means disable the limit. | +| `output_format` | The output format to use. | +| `expand` | Expand table format display, default off, could be on/off/auto. | +| `time` | Whether to show the time elapsed when executing queries. | +| `multi_line` | Whether to allow multi-line input. | +| `replace_newline` | whether replace '\n' with '\\\n'. | ## Commands in REPL -| Commands | Description | -|---|---| -| `!exit` | Exit bendsql | -| `!quit` | Exit bendsql | -| `!configs` | Show current settings | -| `!set` | Set settings | +| Commands | Description | +| -------------- | ----------------------- | +| `!exit` | Exit bendsql | +| `!quit` | Exit bendsql | +| `!configs` | Show current settings | +| `!set` | Set settings | | `!source file` | Source file and execute | ## Setting commands in REPL We can use `!set CMD_NAME VAL` to update the `Settings` above in runtime, example: + ``` ❯ bendsql @@ -168,6 +164,7 @@ We can use `!set CMD_NAME VAL` to update the `Settings` above in runtime, exampl ## DSN Format: + ``` databend[+flight]://user:[password]@host[:port]/[database][?sslmode=disable][&arg1=value1] ``` @@ -180,48 +177,43 @@ Examples: - `databend+flight://root:@localhost:8900/database1?connect_timeout=10` - Available Args: Common: -| Arg | Description | -|---|---| -| `tenant` | Tenant ID, Databend Cloud only. | -| `warehouse` | Warehouse name, Databend Cloud only. | -| `sslmode` | Set to `disable` if not using tls. | -| `tls_ca_file` | Custom root CA certificate path. | -| `connect_timeout` | Connect timeout in seconds | - +| Arg | Description | +| ----------------- | ------------------------------------ | +| `tenant` | Tenant ID, Databend Cloud only. | +| `warehouse` | Warehouse name, Databend Cloud only. | +| `sslmode` | Set to `disable` if not using tls. | +| `tls_ca_file` | Custom root CA certificate path. | +| `connect_timeout` | Connect timeout in seconds | RestAPI client: -| Arg | Description | -|---|---| -| `presigned_url_disabled` | Set to `1` to disable presigned upload to object storage, *deprecated*, use `presign` instead | -| `wait_time_secs` | Request wait time for page, default to `1` | -| `max_rows_in_buffer` | Max rows for page buffer | -| `max_rows_per_page` | Max response rows for a single page | -| `page_request_timeout_secs` | Timeout for a single page request, default to `30` | -| `presign` | Whether to enable presign for data loading, available arguments are auto/detect/on/off, default to `auto` which only enable presign for `Databend Cloud` | - +| Arg | Description | +| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `wait_time_secs` | Request wait time for page, default to `1` | +| `max_rows_in_buffer` | Max rows for page buffer | +| `max_rows_per_page` | Max response rows for a single page | +| `page_request_timeout_secs` | Timeout for a single page request, default to `30` | +| `presign` | Whether to enable presign for data loading, available arguments are `auto`/`detect`/`on`/`off`. Default to `auto` which only enable presign for `Databend Cloud` | FlightSQL client: -| Arg | Description | -|---|---| -| `query_timeout` | Query timeout seconds | -| `tcp_nodelay` | Default to `true` | -| `tcp_keepalive` | Tcp keepalive seconds, default to `3600`, set to `0` to disable keepalive | -| `http2_keep_alive_interval` | Keep alive interval in seconds, default to `300` | -| `keep_alive_timeout` | Keep alive timeout in seconds, default to `20` | -| `keep_alive_while_idle` | Default to `true` | +| Arg | Description | +| --------------------------- | ------------------------------------------------------------------------- | +| `query_timeout` | Query timeout seconds | +| `tcp_nodelay` | Default to `true` | +| `tcp_keepalive` | Tcp keepalive seconds, default to `3600`, set to `0` to disable keepalive | +| `http2_keep_alive_interval` | Keep alive interval in seconds, default to `300` | +| `keep_alive_timeout` | Keep alive timeout in seconds, default to `20` | +| `keep_alive_while_idle` | Default to `true` | Query Settings: see: [Databend Query Settings](https://databend.rs/doc/sql-commands/show/show-settings) - ## Development ### Cargo fmt, clippy, deny @@ -238,7 +230,7 @@ make test ### integration tests -*Note: Docker and Docker Compose needed* +_Note: Docker and Docker Compose needed_ ```bash make integration diff --git a/core/src/client.rs b/core/src/client.rs index 05a44c54..5fda2388 100644 --- a/core/src/client.rs +++ b/core/src/client.rs @@ -121,19 +121,6 @@ impl APIClient { Duration::from_secs(secs) }; } - "presigned_url_disabled" => { - warn!("presigned_url_disabled is deprecated, please use presign=auto/detect/on/off in DSN instead"); - client.presign = match v.as_ref() { - "true" | "1" => PresignMode::On, - "false" | "0" => PresignMode::Off, - _ => { - return Err(Error::BadArgument(format!( - "Invalid value for presigned_url_disabled: {}", - v - ))) - } - } - } "presign" => { client.presign = match v.as_ref() { "auto" => PresignMode::Auto, From 7326b3f8290bc11cce817b157a7fc9d32c7531e9 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 16:49:02 +0800 Subject: [PATCH 23/52] fix(bindings/nodejs): map BINARY type to Buffer (#396) --- bindings/nodejs/README.md | 28 ++++++++-------- bindings/nodejs/tests/binding.js | 32 +++++++++++++------ .../python/tests/asyncio/steps/binding.py | 10 +++--- .../python/tests/blocking/steps/binding.py | 10 +++--- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/bindings/nodejs/README.md b/bindings/nodejs/README.md index ab23c053..8ec5d0e6 100644 --- a/bindings/nodejs/README.md +++ b/bindings/nodejs/README.md @@ -42,20 +42,20 @@ while (row) { ### General Data Types -| Databend | Node.js | -| ----------- | --------------- | -| `BOOLEAN` | `Boolean` | -| `TINYINT` | `Number` | -| `SMALLINT` | `Number` | -| `INT` | `Number` | -| `BIGINT` | `Number` | -| `FLOAT` | `Number` | -| `DOUBLE` | `Number` | -| `DECIMAL` | `String` | -| `DATE` | `Date` | -| `TIMESTAMP` | `Date` | -| `VARCHAR` | `String` | -| `BINARY` | `Array(Number)` | +| Databend | Node.js | +| ----------- | --------- | +| `BOOLEAN` | `Boolean` | +| `TINYINT` | `Number` | +| `SMALLINT` | `Number` | +| `INT` | `Number` | +| `BIGINT` | `Number` | +| `FLOAT` | `Number` | +| `DOUBLE` | `Number` | +| `DECIMAL` | `String` | +| `DATE` | `Date` | +| `TIMESTAMP` | `Date` | +| `VARCHAR` | `String` | +| `BINARY` | `Buffer` | ### Semi-Structured Data Types diff --git a/bindings/nodejs/tests/binding.js b/bindings/nodejs/tests/binding.js index bbf68ff9..5d8a46b7 100644 --- a/bindings/nodejs/tests/binding.js +++ b/bindings/nodejs/tests/binding.js @@ -34,21 +34,35 @@ Then("Select string {string} should be equal to {string}", async function (input }); Then("Select types should be expected native types", async function () { - // NumberValue::Decimal - const row1 = await this.conn.queryRow(`SELECT 15.7563::Decimal(8,4), 2.0+3.0`); - assert.deepEqual(row1.values(), ["15.7563", "5.0"]); + // Binary + { + const row = await this.conn.queryRow("select to_binary('xyz')"); + assert.deepEqual(row.values(), [Buffer.from("xyz")]); + } + + // Decimal + { + const row = await this.conn.queryRow(`SELECT 15.7563::Decimal(8,4), 2.0+3.0`); + assert.deepEqual(row.values(), ["15.7563", "5.0"]); + } // Array - const row2 = await this.conn.queryRow(`SELECT [10::Decimal(15,2), 1.1+2.3]`); - assert.deepEqual(row2.values(), [["10.00", "3.40"]]); + { + const row = await this.conn.queryRow(`SELECT [10::Decimal(15,2), 1.1+2.3]`); + assert.deepEqual(row.values(), [["10.00", "3.40"]]); + } // Map - const row3 = await this.conn.queryRow(`SELECT {'xx':to_date('2020-01-01')}`); - assert.deepEqual(row3.values(), [{ xx: new Date("2020-01-01") }]); + { + const row = await this.conn.queryRow(`SELECT {'xx':to_date('2020-01-01')}`); + assert.deepEqual(row.values(), [{ xx: new Date("2020-01-01") }]); + } // Tuple - const row4 = await this.conn.queryRow(`SELECT (10, '20', to_datetime('2024-04-16 12:34:56.789'))`); - assert.deepEqual(row4.values(), [[10, "20", new Date("2024-04-16T12:34:56.789Z")]]); + { + const row = await this.conn.queryRow(`SELECT (10, '20', to_datetime('2024-04-16 12:34:56.789'))`); + assert.deepEqual(row.values(), [[10, "20", new Date("2024-04-16T12:34:56.789Z")]]); + } }); Then("Select numbers should iterate all rows", async function () { diff --git a/bindings/python/tests/asyncio/steps/binding.py b/bindings/python/tests/asyncio/steps/binding.py index 3c2924b6..78fd7bcd 100644 --- a/bindings/python/tests/asyncio/steps/binding.py +++ b/bindings/python/tests/asyncio/steps/binding.py @@ -62,17 +62,17 @@ async def _(context, input, output): @then("Select types should be expected native types") @async_run_until_complete async def _(context): - # NumberValue::Decimal + # Binary + row = await context.conn.query_row("select to_binary('xyz')") + assert row.values() == (b"xyz",), f"Binary: {row.values()}" + + # Decimal row = await context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") assert row.values() == ( Decimal("15.7563"), Decimal("5.0"), ), f"Decimal: {row.values()}" - # Binary - row = await context.conn.query_row("select to_binary('xyz')") - assert row.values() == (b"xyz",), f"Binary: {row.values()}" - # Array row = await context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") assert row.values() == ( diff --git a/bindings/python/tests/blocking/steps/binding.py b/bindings/python/tests/blocking/steps/binding.py index 15c4353e..7e8ed32a 100644 --- a/bindings/python/tests/blocking/steps/binding.py +++ b/bindings/python/tests/blocking/steps/binding.py @@ -57,17 +57,17 @@ def _(context, input, output): @then("Select types should be expected native types") async def _(context): - # NumberValue::Decimal + # Binary + row = context.conn.query_row("select to_binary('xyz')") + assert row.values() == (b"xyz",), f"Binary: {row.values()}" + + # Decimal row = context.conn.query_row("SELECT 15.7563::Decimal(8,4), 2.0+3.0") assert row.values() == ( Decimal("15.7563"), Decimal("5.0"), ), f"Decimal: {row.values()}" - # Binary - row = context.conn.query_row("select to_binary('xyz')") - assert row.values() == (b"xyz",), f"Binary: {row.values()}" - # Array row = context.conn.query_row("select [10::Decimal(15,2), 1.1+2.3]") assert row.values() == ( From 874a48e9c3111b31f2a17c268a491f4248391805 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 17:22:17 +0800 Subject: [PATCH 24/52] fix(cli): support exit & quit for exit (#397) --- cli/src/session.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cli/src/session.rs b/cli/src/session.rs index f2c7ad22..b6ac8e56 100644 --- a/cli/src/session.rs +++ b/cli/src/session.rs @@ -397,8 +397,13 @@ impl Session { ) -> Result> { let query = query.trim_end_matches(';').trim(); - if is_repl && query.starts_with('!') { - return self.handle_commands(query).await; + if is_repl { + if query.starts_with('!') { + return self.handle_commands(query).await; + } + if query == "exit" || query == "quit" { + return Ok(None); + } } let start = Instant::now(); @@ -457,10 +462,6 @@ impl Session { #[async_recursion] pub async fn handle_commands(&mut self, query: &str) -> Result> { - if query == "!exit" || query == "!quit" { - return Ok(None); - } - match query { "!exit" | "!quit" => { return Ok(None); From 5265014d0e08bb636261e2ddad70c0baa26d7007 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 17 Apr 2024 17:51:27 +0800 Subject: [PATCH 25/52] chore: bump version to 0.17.0 (#398) --- Cargo.toml | 10 +++++----- bindings/nodejs/npm/darwin-arm64/package.json | 2 +- bindings/nodejs/npm/darwin-x64/package.json | 2 +- bindings/nodejs/npm/linux-arm64-gnu/package.json | 2 +- bindings/nodejs/npm/linux-x64-gnu/package.json | 2 +- bindings/nodejs/npm/win32-x64-msvc/package.json | 2 +- bindings/nodejs/package.json | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3c808fb1..98c61478 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.16.3" +version = "0.17.0" edition = "2021" license = "Apache-2.0" authors = ["Databend Authors "] @@ -21,10 +21,10 @@ keywords = ["databend", "database"] repository = "https://github.com/datafuselabs/bendsql" [workspace.dependencies] -databend-client = { path = "core", version = "0.16.3" } -databend-driver = { path = "driver", version = "0.16.3" } -databend-driver-macros = { path = "macros", version = "0.16.3" } -databend-sql = { path = "sql", version = "0.16.3" } +databend-client = { path = "core", version = "0.17.0" } +databend-driver = { path = "driver", version = "0.17.0" } +databend-driver-macros = { path = "macros", version = "0.17.0" } +databend-sql = { path = "sql", version = "0.17.0" } [patch.crates-io] pyo3-asyncio = { git = "https://github.com/everpcpc/pyo3-asyncio", rev = "42af887" } diff --git a/bindings/nodejs/npm/darwin-arm64/package.json b/bindings/nodejs/npm/darwin-arm64/package.json index 60deaab5..c74f76c3 100644 --- a/bindings/nodejs/npm/darwin-arm64/package.json +++ b/bindings/nodejs/npm/darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-arm64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.3", + "version": "0.17.0", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/darwin-x64/package.json b/bindings/nodejs/npm/darwin-x64/package.json index a3ff9979..009a2534 100644 --- a/bindings/nodejs/npm/darwin-x64/package.json +++ b/bindings/nodejs/npm/darwin-x64/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-darwin-x64", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.3", + "version": "0.17.0", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/linux-arm64-gnu/package.json b/bindings/nodejs/npm/linux-arm64-gnu/package.json index 3e15a619..d33154df 100644 --- a/bindings/nodejs/npm/linux-arm64-gnu/package.json +++ b/bindings/nodejs/npm/linux-arm64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-arm64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.3", + "version": "0.17.0", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/linux-x64-gnu/package.json b/bindings/nodejs/npm/linux-x64-gnu/package.json index 26bd72e6..d1b6ec99 100644 --- a/bindings/nodejs/npm/linux-x64-gnu/package.json +++ b/bindings/nodejs/npm/linux-x64-gnu/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-linux-x64-gnu", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.3", + "version": "0.17.0", "os": [ "linux" ], diff --git a/bindings/nodejs/npm/win32-x64-msvc/package.json b/bindings/nodejs/npm/win32-x64-msvc/package.json index 2afbcdc1..96c62086 100644 --- a/bindings/nodejs/npm/win32-x64-msvc/package.json +++ b/bindings/nodejs/npm/win32-x64-msvc/package.json @@ -1,7 +1,7 @@ { "name": "@databend-driver/lib-win32-x64-msvc", "repository": "https://github.com/datafuselabs/bendsql.git", - "version": "0.16.3", + "version": "0.17.0", "os": [ "win32" ], diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index 979f80a9..da529314 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "databend-driver", "author": "Databend Authors ", - "version": "0.16.3", + "version": "0.17.0", "license": "Apache-2.0", "main": "index.js", "types": "index.d.ts", From e9824f3919e2c55c23a9128c2d9e7743314ff33e Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 18 Apr 2024 16:31:06 +0800 Subject: [PATCH 26/52] fix(cli): set default missing_field_as to NULL for NDJSON (#401) --- cli/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index c1892aae..4e9f9014 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -74,6 +74,8 @@ impl InputFormat { } InputFormat::NDJSON => { options.insert("type", "NDJSON"); + options.insert("null_field_as", "NULL"); + options.insert("missing_field_as", "NULL"); } InputFormat::Parquet => { options.insert("type", "Parquet"); From 1323bf33f23958a84b362ef28ed47a738148a45b Mon Sep 17 00:00:00 2001 From: everpcpc Date: Mon, 22 Apr 2024 09:59:19 +0800 Subject: [PATCH 27/52] chore(docs): add examples for variant (#402) --- bindings/nodejs/README.md | 16 ++++++++++++++++ bindings/python/README.md | 16 ++++++++++++++++ driver/README.md | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/bindings/nodejs/README.md b/bindings/nodejs/README.md index 8ec5d0e6..ef96018f 100644 --- a/bindings/nodejs/README.md +++ b/bindings/nodejs/README.md @@ -68,6 +68,22 @@ while (row) { | `BITMAP` | `String` | | `GEOMETRY` | `String` | +Note: `VARIANT` is a json encoded string. Example: + +```sql +CREATE TABLE example ( + data VARIANT +); +INSERT INTO example VALUES ('{"a": 1, "b": "hello"}'); +``` + +```javascript +const row = await conn.queryRow("SELECT * FROM example limit 1;"); +const data = row.values()[0]; +const value = JSON.parse(data); +console.log(value); +``` + ## Development ```shell diff --git a/bindings/python/README.md b/bindings/python/README.md index a736abfc..d1b4b946 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -95,6 +95,22 @@ asyncio.run(main()) | `BITMAP` | `str` | | `GEOMETRY` | `str` | +Note: `VARIANT` is a json encoded string. Example: + +```sql +CREATE TABLE example ( + data VARIANT +); +INSERT INTO example VALUES ('{"a": 1, "b": "hello"}'); +``` + +```python +row = await conn.query_row("SELECT * FROM example limit 1;") +data = row.values()[0] +value = json.loads(data) +print(value) +``` + ## APIs ### AsyncDatabendClient diff --git a/driver/README.md b/driver/README.md index aef74a2f..d64192c0 100644 --- a/driver/README.md +++ b/driver/README.md @@ -75,3 +75,19 @@ while let Some(row) = rows.next().await { | `VARIANT` | `String` | | `BITMAP` | `String` | | `GEOMETRY` | `String` | + +Note: `VARIANT` is a json encoded string. Example: + +```sql +CREATE TABLE example ( + data VARIANT +); +INSERT INTO example VALUES ('{"a": 1, "b": "hello"}'); +``` + +```rust +let row = conn.query_row("SELECT * FROM example limit 1;").await.unwrap(); +let (data,): (String,) = row.unwrap().try_into().unwrap(); +let value: serde_json::Value = serde_json::from_str(&data).unwrap(); +println!("{:?}", value); +``` From d138d4fd030e9e68abe025bb64266d4f475a6206 Mon Sep 17 00:00:00 2001 From: Yang Xiufeng Date: Mon, 22 Apr 2024 10:14:04 +0800 Subject: [PATCH 28/52] refactor(core): SessionState hide fields of no interest. (#399) --- core/src/request.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/core/src/request.rs b/core/src/request.rs index 8666daf4..90f12ef2 100644 --- a/core/src/request.rs +++ b/core/src/request.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] @@ -32,11 +32,10 @@ pub struct SessionState { pub secondary_roles: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub txn_state: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_info: Option, - #[serde(default)] - #[serde(skip_serializing_if = "Vec::is_empty")] - pub last_query_ids: Vec, + + // hide fields of no interest (but need to send back to server in next query) + #[serde(flatten)] + additional_fields: HashMap, } impl SessionState { @@ -129,8 +128,7 @@ mod test { role: None, secondary_roles: None, txn_state: None, - last_server_info: None, - last_query_ids: vec![], + additional_fields: Default::default(), })) .with_pagination(Some(PaginationConfig { wait_time_secs: Some(1), From f580aeeb0c824f67d29331cf16fedc45ba04909d Mon Sep 17 00:00:00 2001 From: Yang Xiufeng Date: Mon, 22 Apr 2024 17:44:31 +0800 Subject: [PATCH 29/52] ci: add test for multi-page result. (#403) --- driver/tests/driver/select_iter.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/driver/tests/driver/select_iter.rs b/driver/tests/driver/select_iter.rs index 33c1cfee..4178ba73 100644 --- a/driver/tests/driver/select_iter.rs +++ b/driver/tests/driver/select_iter.rs @@ -198,6 +198,23 @@ async fn select_numbers() { assert_eq!(ret, vec![0, 1, 2, 3, 4]); } +#[tokio::test] +async fn select_multi_page() { + let (conn, _) = prepare("select_multi_page").await; + // default page size is 10000 + let n = 46000; + let sql = format!("select * from NUMBERS({n}) order by number"); + let rows = conn.query_iter(&sql).await.unwrap(); + let ret: Vec = rows + .map(|r| r.unwrap().try_into().unwrap()) + .collect::>() + .await + .into_iter() + .map(|r| r.0) + .collect(); + assert_eq!(ret, (0..n).collect::>()); +} + #[tokio::test] async fn select_sleep() { let (conn, _) = prepare("select_sleep").await; From e64ad8795420b53ae38423aee27793700bc808ce Mon Sep 17 00:00:00 2001 From: everpcpc Date: Tue, 23 Apr 2024 10:33:58 +0800 Subject: [PATCH 30/52] feat: make password & dsn sensitive (#404) * feat: make password & dsn sensitive * chore: tmp disable geometry test --- cli/Cargo.toml | 3 +- cli/src/args.rs | 13 ++-- cli/src/display.rs | 11 ++-- cli/src/main.rs | 73 +++++++++++++---------- cli/tests/00-base.result | 2 +- cli/tests/00-base.sql | 2 +- core/src/auth.rs | 89 +++++++++++++++++++++++++--- core/src/client.rs | 14 ++--- core/src/error.rs | 6 ++ driver/src/flight_sql.rs | 13 ++-- driver/tests/driver/select_iter.rs | 16 ----- driver/tests/driver/select_simple.rs | 34 ++++++++++- sql/src/error.rs | 6 ++ sql/src/value.rs | 4 +- tests/docker-compose.yaml | 2 +- 15 files changed, 196 insertions(+), 92 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 41631b3b..b9c647a7 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -11,9 +11,11 @@ authors = { workspace = true } repository = { workspace = true } [dependencies] +databend-client = { workspace = true } databend-driver = { workspace = true, features = ["rustls", "flight-sql"] } anyhow = "1.0" +async-recursion = "1.1.0" async-trait = "0.1" chrono = { version = "0.4.35", default-features = false, features = ["clock"] } clap = { version = "4.4", features = ["derive", "env"] } @@ -42,7 +44,6 @@ toml = "0.8" tracing-appender = "0.2" unicode-segmentation = "1.10" url = { version = "2.5", default-features = false } -async-recursion = "1.1.0" [build-dependencies] vergen = { version = "8.2", features = ["build", "git", "gix"] } diff --git a/cli/src/args.rs b/cli/src/args.rs index 25643b48..35cf9a4c 100644 --- a/cli/src/args.rs +++ b/cli/src/args.rs @@ -15,13 +15,14 @@ use std::collections::BTreeMap; use anyhow::{anyhow, Result}; +use databend_client::auth::SensitiveString; #[derive(Debug, Clone, PartialEq, Default)] pub struct ConnectionArgs { pub host: String, pub port: Option, pub user: String, - pub password: Option, + pub password: SensitiveString, pub database: Option, pub flight: bool, pub args: BTreeMap, @@ -33,9 +34,7 @@ impl ConnectionArgs { dsn.set_host(Some(&self.host))?; _ = dsn.set_port(self.port); _ = dsn.set_username(&self.user); - if let Some(password) = self.password { - _ = dsn.set_password(Some(&password)) - }; + _ = dsn.set_password(Some(self.password.inner())); if let Some(database) = self.database { dsn.set_path(&database); } @@ -64,7 +63,7 @@ impl ConnectionArgs { let host = u.host_str().ok_or(anyhow!("missing host"))?.to_string(); let port = u.port(); let user = u.username().to_string(); - let password = u.password().map(|s| s.to_string()); + let password = SensitiveString::from(u.password().unwrap_or_default()); let database = u.path().strip_prefix('/').map(|s| s.to_string()); Ok(Self { host, @@ -90,7 +89,7 @@ mod test { host: "app.databend.com".to_string(), port: None, user: "username".to_string(), - password: Some("3a@SC(nYE1k={{R".to_string()), + password: SensitiveString::from("3a@SC(nYE1k={{R"), database: Some("test".to_string()), flight: false, args: { @@ -113,7 +112,7 @@ mod test { host: "app.databend.com".to_string(), port: Some(443), user: "username".to_string(), - password: Some("3a@SC(nYE1k={{R".to_string()), + password: SensitiveString::from("3a@SC(nYE1k={{R"), database: Some("test".to_string()), flight: false, args: { diff --git a/cli/src/display.rs b/cli/src/display.rs index 9283865c..b0f1e798 100644 --- a/cli/src/display.rs +++ b/cli/src/display.rs @@ -14,23 +14,20 @@ use std::collections::HashSet; use std::fmt::Write; -use unicode_segmentation::UnicodeSegmentation; use anyhow::{anyhow, Result}; use comfy_table::{Cell, CellAlignment, Table}; -use terminal_size::{terminal_size, Width}; - use databend_driver::{Row, RowStatsIterator, RowWithStats, SchemaRef, ServerStats}; +use indicatif::{HumanBytes, ProgressBar, ProgressState, ProgressStyle}; use rustyline::highlight::Highlighter; +use terminal_size::{terminal_size, Width}; use tokio::time::Instant; use tokio_stream::StreamExt; +use unicode_segmentation::UnicodeSegmentation; -use indicatif::{HumanBytes, ProgressBar, ProgressState, ProgressStyle}; - -use crate::config::OutputQuoteStyle; use crate::{ ast::format_query, - config::{ExpandMode, OutputFormat, Settings}, + config::{ExpandMode, OutputFormat, OutputQuoteStyle, Settings}, helper::CliHelper, session::QueryKind, }; diff --git a/cli/src/main.rs b/cli/src/main.rs index 4e9f9014..e0ba959a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -27,14 +27,17 @@ use std::{ io::{stdin, IsTerminal}, }; -use crate::args::ConnectionArgs; -use crate::config::OutputQuoteStyle; use anyhow::{anyhow, Result}; use clap::{ArgAction, CommandFactory, Parser, ValueEnum}; -use config::{Config, OutputFormat, Settings, TimeOption}; +use databend_client::auth::SensitiveString; use log::info; use once_cell::sync::Lazy; +use crate::{ + args::ConnectionArgs, + config::{Config, OutputFormat, OutputQuoteStyle, Settings, TimeOption}, +}; + static VERSION: Lazy = Lazy::new(|| { let version = option_env!("CARGO_PKG_VERSION").unwrap_or("unknown"); let sha = option_env!("VERGEN_GIT_SHA").unwrap_or("dev"); @@ -106,32 +109,45 @@ struct Args { #[clap(long, help = "Print help information")] help: bool, - #[clap(long, help = "Using flight sql protocol")] + #[clap(long, help = "Using flight sql protocol, ignored when --dsn is set")] flight: bool, - #[clap(long, help = "Enable TLS")] + #[clap(long, help = "Enable TLS, ignored when --dsn is set")] tls: bool, - #[clap(short = 'h', long, help = "Databend Server host, Default: 127.0.0.1")] + #[clap( + short = 'h', + long, + help = "Databend Server host, Default: 127.0.0.1, ignored when --dsn is set" + )] host: Option, - #[clap(short = 'P', long, help = "Databend Server port, Default: 8000")] + #[clap( + short = 'P', + long, + help = "Databend Server port, Default: 8000, ignored when --dsn is set" + )] port: Option, - #[clap(short = 'u', long, help = "Default: root")] + #[clap(short = 'u', long, help = "Default: root, overrides username in DSN")] user: Option, - #[clap(short = 'p', long, env = "BENDSQL_PASSWORD")] - password: Option, + #[clap( + short = 'p', + long, + env = "BENDSQL_PASSWORD", + help = "Password, overrides password in DSN" + )] + password: Option, - #[clap(short = 'D', long, help = "Database name")] + #[clap(short = 'D', long, help = "Database name, overrides database in DSN")] database: Option, - #[clap(long, value_parser = parse_key_val::, help = "Settings")] + #[clap(long, value_parser = parse_key_val::, help = "Settings, ignored when --dsn is set")] set: Vec<(String, String)>, #[clap(long, env = "BENDSQL_DSN", help = "Data source name")] - dsn: Option, + dsn: Option, #[clap(short = 'n', long, help = "Force non-interactive mode")] non_interactive: bool, @@ -219,15 +235,6 @@ pub async fn main() -> Result<()> { if args.port.is_some() { eprintln!("warning: --port is ignored when --dsn is set"); } - if args.user.is_some() { - eprintln!("warning: --user is ignored when --dsn is set"); - } - if args.password.is_some() { - eprintln!("warning: --password is ignored when --dsn is set"); - } - if args.role.is_some() { - eprintln!("warning: --role is ignored when --dsn is set"); - } if !args.set.is_empty() { eprintln!("warning: --set is ignored when --dsn is set"); } @@ -237,7 +244,7 @@ pub async fn main() -> Result<()> { if args.flight { eprintln!("warning: --flight is ignored when --dsn is set"); } - ConnectionArgs::from_dsn(&dsn)? + ConnectionArgs::from_dsn(dsn.inner())? } None => { if let Some(host) = args.host { @@ -246,9 +253,6 @@ pub async fn main() -> Result<()> { if let Some(port) = args.port { config.connection.port = Some(port); } - if let Some(user) = args.user { - config.connection.user = user; - } for (k, v) in args.set { config.connection.args.insert(k, v); } @@ -258,14 +262,11 @@ pub async fn main() -> Result<()> { .args .insert("sslmode".to_string(), "disable".to_string()); } - if let Some(role) = args.role { - config.connection.args.insert("role".to_string(), role); - } ConnectionArgs { host: config.connection.host.clone(), port: config.connection.port, user: config.connection.user.clone(), - password: args.password, + password: SensitiveString::from(""), database: config.connection.database.clone(), flight: args.flight, args: config.connection.args.clone(), @@ -276,6 +277,18 @@ pub async fn main() -> Result<()> { if args.database.is_some() { conn_args.database = args.database; } + // override user if specified in command line + if let Some(user) = args.user { + config.connection.user = user; + } + // override password if specified in command line + if let Some(password) = args.password { + conn_args.password = password; + } + // override role if specified in command line + if let Some(role) = args.role { + config.connection.args.insert("role".to_string(), role); + } let dsn = conn_args.get_dsn()?; let mut settings = Settings::default(); diff --git a/cli/tests/00-base.result b/cli/tests/00-base.result index 4962e053..ccbe2cb4 100644 --- a/cli/tests/00-base.result +++ b/cli/tests/00-base.result @@ -20,5 +20,5 @@ Asia/Shanghai NULL {'k1':'v1','k2':'v2'} (2,NULL) 1 NULL 1 ab NULL v1 2 NULL -{'k1':'v1','k2':'v2'} [6162,78797A] ('[1,2]','SRID=4326;POINT(1 2)','2024-04-10') +{'k1':'v1','k2':'v2'} [6162,78797A] ('[1,2]','2024-04-10') bye diff --git a/cli/tests/00-base.sql b/cli/tests/00-base.sql index e07c3270..d441fadf 100644 --- a/cli/tests/00-base.sql +++ b/cli/tests/00-base.sql @@ -46,7 +46,7 @@ insert into test_nested values([1,2,3], null, (1, 'ab')), (null, {'k1':'v1', 'k2 select * from test_nested; select a[1], b['k1'], c:x, c:y from test_nested; -select {'k1':'v1','k2':'v2'}, [to_binary('ab'), to_binary('xyz')], (parse_json('[1,2]'), st_geometryfromwkt('SRID=4326;POINT(1.0 2.0)'), to_date('2024-04-10')); +select {'k1':'v1','k2':'v2'}, [to_binary('ab'), to_binary('xyz')], (parse_json('[1,2]'), to_date('2024-04-10')); select 'bye'; drop table test; diff --git a/core/src/auth.rs b/core/src/auth.rs index fe67fc2c..a85c0001 100644 --- a/core/src/auth.rs +++ b/core/src/auth.rs @@ -25,19 +25,22 @@ pub trait Auth: Sync + Send { #[derive(Clone)] pub struct BasicAuth { username: String, - password: Option, + password: SensitiveString, } impl BasicAuth { - pub fn new(username: String, password: Option) -> Self { - Self { username, password } + pub fn new(username: impl ToString, password: impl ToString) -> Self { + Self { + username: username.to_string(), + password: SensitiveString(password.to_string()), + } } } #[async_trait::async_trait] impl Auth for BasicAuth { async fn wrap(&self, builder: RequestBuilder) -> Result { - Ok(builder.basic_auth(&self.username, self.password.as_deref())) + Ok(builder.basic_auth(&self.username, Some(self.password.inner()))) } fn username(&self) -> String { @@ -47,19 +50,21 @@ impl Auth for BasicAuth { #[derive(Clone)] pub struct AccessTokenAuth { - token: String, + token: SensitiveString, } impl AccessTokenAuth { - pub fn new(token: String) -> Self { - Self { token } + pub fn new(token: impl ToString) -> Self { + Self { + token: SensitiveString::from(token.to_string()), + } } } #[async_trait::async_trait] impl Auth for AccessTokenAuth { async fn wrap(&self, builder: RequestBuilder) -> Result { - Ok(builder.bearer_auth(&self.token)) + Ok(builder.bearer_auth(self.token.inner())) } fn username(&self) -> String { @@ -73,7 +78,8 @@ pub struct AccessTokenFileAuth { } impl AccessTokenFileAuth { - pub fn new(token_file: String) -> Self { + pub fn new(token_file: impl ToString) -> Self { + let token_file = token_file.to_string(); Self { token_file } } } @@ -96,3 +102,68 @@ impl Auth for AccessTokenFileAuth { "token".to_string() } } + +#[derive(::serde::Deserialize, ::serde::Serialize)] +#[serde(from = "String", into = "String")] +#[derive(Clone, Default, PartialEq, Eq)] +pub struct SensitiveString(String); + +impl From for SensitiveString { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for SensitiveString { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +impl From for String { + fn from(value: SensitiveString) -> Self { + value.0 + } +} + +impl std::fmt::Display for SensitiveString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "**REDACTED**") + } +} + +impl std::fmt::Debug for SensitiveString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // we keep the double quotes here to keep the String behavior + write!(f, "\"**REDACTED**\"") + } +} + +impl SensitiveString { + #[must_use] + pub fn inner(&self) -> &str { + self.0.as_str() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serialization() { + let json_value = "\"foo\""; + let value: SensitiveString = serde_json::from_str(json_value).unwrap(); + let result: String = serde_json::to_string(&value).unwrap(); + assert_eq!(result, json_value); + } + + #[test] + fn hide_content() { + let value = SensitiveString("hello world".to_string()); + let display = format!("{value}"); + assert_eq!(display, "**REDACTED**"); + let debug = format!("{value:?}"); + assert_eq!(debug, "\"**REDACTED**\""); + } +} diff --git a/core/src/client.rs b/core/src/client.rs index 5fda2388..ca4c3e70 100644 --- a/core/src/client.rs +++ b/core/src/client.rs @@ -90,11 +90,9 @@ impl APIClient { } if u.username() != "" { - client.auth = Arc::new(BasicAuth::new( - u.username().to_string(), - u.password() - .map(|s| percent_decode_str(s).decode_utf8_lossy().to_string()), - )); + let password = u.password().unwrap_or_default(); + let password = percent_decode_str(password).decode_utf8()?; + client.auth = Arc::new(BasicAuth::new(u.username(), password)); } let database = match u.path().trim_start_matches('/') { "" => None, @@ -156,10 +154,10 @@ impl APIClient { client.tls_ca_file = Some(v.to_string()); } "access_token" => { - client.auth = Arc::new(AccessTokenAuth::new(v.to_string())); + client.auth = Arc::new(AccessTokenAuth::new(v)); } "access_token_file" => { - client.auth = Arc::new(AccessTokenFileAuth::new(v.to_string())); + client.auth = Arc::new(AccessTokenFileAuth::new(v)); } _ => { session_settings.insert(k.to_string(), v.to_string()); @@ -581,7 +579,7 @@ impl Default for APIClient { port: 8000, tenant: None, warehouse: Arc::new(Mutex::new(None)), - auth: Arc::new(BasicAuth::new("root".to_string(), None)) as Arc, + auth: Arc::new(BasicAuth::new("root", "")) as Arc, session_state: Arc::new(Mutex::new(SessionState::default())), wait_time_secs: None, max_rows_in_buffer: None, diff --git a/core/src/error.rs b/core/src/error.rs index f4efb8bb..6d535cd8 100644 --- a/core/src/error.rs +++ b/core/src/error.rs @@ -83,3 +83,9 @@ impl From for Error { Error::IO(e.to_string()) } } + +impl From for Error { + fn from(e: std::str::Utf8Error) -> Self { + Error::Parsing(e.to_string()) + } +} diff --git a/driver/src/flight_sql.rs b/driver/src/flight_sql.rs index d80f054c..dd1d2b46 100644 --- a/driver/src/flight_sql.rs +++ b/driver/src/flight_sql.rs @@ -31,6 +31,7 @@ use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; use tonic::Streaming; use url::Url; +use databend_client::auth::SensitiveString; use databend_client::presign::{presign_upload_to_stage, PresignedResponse}; use databend_sql::error::{Error, Result}; use databend_sql::rows::{Row, RowIterator, RowStatsIterator, RowWithStats, Rows, ServerStats}; @@ -166,7 +167,7 @@ impl FlightSQLConnection { } let mut client = self.client.lock().await; let _token = client - .handshake(&self.args.user, &self.args.password) + .handshake(&self.args.user, self.args.password.inner()) .await?; *handshaked = true; Ok(()) @@ -206,7 +207,7 @@ struct Args { host: String, port: u16, user: String, - password: String, + password: SensitiveString, database: Option, tenant: Option, warehouse: Option, @@ -234,7 +235,7 @@ impl Default for Args { tls: true, tls_ca_file: None, user: "root".to_string(), - password: "".to_string(), + password: SensitiveString::from(""), connect_timeout: Duration::from_secs(20), query_timeout: Duration::from_secs(60), tcp_nodelay: true, @@ -308,9 +309,9 @@ impl Args { None => format!("{}://{}:{}", scheme, host, port), }; args.user = u.username().to_string(); - args.password = percent_decode_str(u.password().unwrap_or_default()) - .decode_utf8_lossy() - .to_string(); + let password = u.password().unwrap_or_default(); + let password = percent_decode_str(password).decode_utf8()?; + args.password = SensitiveString::from(password.to_string()); Ok(args) } } diff --git a/driver/tests/driver/select_iter.rs b/driver/tests/driver/select_iter.rs index 4178ba73..9a1f5e50 100644 --- a/driver/tests/driver/select_iter.rs +++ b/driver/tests/driver/select_iter.rs @@ -226,19 +226,3 @@ async fn select_sleep() { } assert_eq!(result, vec![0]); } - -// #[tokio::test] -// async fn select_bitmap_string() { -// let (conn, _) = prepare("select_bitmap_string").await; -// let mut rows = conn -// .query_iter("select build_bitmap([1,2,3,4,5,6]), 11::String") -// .await -// .unwrap(); -// let mut result = vec![]; -// while let Some(row) = rows.next().await { -// let row: (String, String) = row.unwrap().try_into().unwrap(); -// assert!(row.0.contains('\0')); -// result.push(row.1); -// } -// assert_eq!(result, vec!["11".to_string()]); -// } diff --git a/driver/tests/driver/select_simple.rs b/driver/tests/driver/select_simple.rs index f44c72fd..7792e2d6 100644 --- a/driver/tests/driver/select_simple.rs +++ b/driver/tests/driver/select_simple.rs @@ -278,15 +278,15 @@ async fn select_tuple() { assert_eq!(val1, ("[1,2]".to_string(), vec![1, 2], true,)); let row2 = conn - .query_row("select (st_geometryfromwkt('SRID=4126;POINT(3.0 5.0)'), to_timestamp('2024-10-22 10:11:12'))") + .query_row("select (to_binary('xyz'), to_timestamp('2024-10-22 10:11:12'))") .await .unwrap() .unwrap(); - let (val2,): ((String, NaiveDateTime),) = row2.try_into().unwrap(); + let (val2,): ((Vec, NaiveDateTime),) = row2.try_into().unwrap(); assert_eq!( val2, ( - "SRID=4126;POINT(3 5)".to_string(), + vec![120, 121, 122], DateTime::parse_from_rfc3339("2024-10-22T10:11:12Z") .unwrap() .naive_utc() @@ -294,6 +294,34 @@ async fn select_tuple() { ); } +#[tokio::test] +async fn select_variant() { + // TODO: +} + +#[tokio::test] +async fn select_bitmap() { + // TODO: + // let (conn, _) = prepare("select_bitmap_string").await; + // let mut rows = conn + // .query_iter("select build_bitmap([1,2,3,4,5,6]), 11::String") + // .await + // .unwrap(); + // let mut result = vec![]; + // while let Some(row) = rows.next().await { + // let row: (String, String) = row.unwrap().try_into().unwrap(); + // assert!(row.0.contains('\0')); + // result.push(row.1); + // } + // assert_eq!(result, vec!["11".to_string()]); +} + +#[tokio::test] +async fn select_geometry() { + // TODO: response type changed to json after + // https://github.com/datafuselabs/databend/pull/15214 +} + #[tokio::test] async fn select_multiple_columns() { let conn = prepare().await; diff --git a/sql/src/error.rs b/sql/src/error.rs index eda306b3..295e1147 100644 --- a/sql/src/error.rs +++ b/sql/src/error.rs @@ -159,6 +159,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: std::str::Utf8Error) -> Self { + Error::Parsing(e.to_string()) + } +} + impl From for Error { fn from(e: std::string::FromUtf8Error) -> Self { Error::Parsing(e.to_string()) diff --git a/sql/src/value.rs b/sql/src/value.rs index 68d2acc4..be2f6418 100644 --- a/sql/src/value.rs +++ b/sql/src/value.rs @@ -948,8 +948,8 @@ pub fn parse_decimal(text: &str, size: DecimalSize) -> Result { pub fn parse_geometry(raw_data: &[u8]) -> Result { let mut data = std::io::Cursor::new(raw_data); - let wkt = Ewkt::from_wkb(&mut data, WkbDialect::Ewkb); - wkt.map(|g| g.0).map_err(|e| e.into()) + let wkt = Ewkt::from_wkb(&mut data, WkbDialect::Ewkb)?; + Ok(wkt.0) } struct ValueDecoder {} diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 53ec7568..f773bea5 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -7,7 +7,7 @@ services: volumes: - ./data:/data databend: - image: docker.io/datafuselabs/databend:nightly + image: docker.io/datafuselabs/databend environment: - QUERY_STORAGE_TYPE=s3 - AWS_S3_ENDPOINT=http://localhost:9000 From 6d40644fce44419c72397e45098dd3dae0b9fb5d Mon Sep 17 00:00:00 2001 From: everpcpc Date: Tue, 23 Apr 2024 11:04:42 +0800 Subject: [PATCH 31/52] fix(cli): hide env values for cmd arg (#405) --- README.md | 44 ++++++++++++++++++++++++-------------------- cli/src/main.rs | 23 +++++++++++++++-------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 558465e8..1d64934d 100644 --- a/README.md +++ b/README.md @@ -68,26 +68,30 @@ Databend Native Command Line Tool Usage: bendsql [OPTIONS] Options: - --help Print help information - --flight Using flight sql protocol - --tls Enable TLS - -h, --host Databend Server host, Default: 127.0.0.1 - -P, --port Databend Server port, Default: 8000 - -u, --user Default: root - -p, --password [env: BENDSQL_PASSWORD=] - -D, --database Database name - --set Settings - --dsn Data source name [env: BENDSQL_DSN=] - -n, --non-interactive Force non-interactive mode - --query= Query to execute - -d, --data Data to load, @file or @- for stdin - -f, --format Data format to load [default: csv] [possible values: csv, tsv, ndjson, parquet, xml] - --format-opt Data format options - -o, --output Output format [possible values: table, csv, tsv, null] - --progress Show progress for query execution in stderr, only works with output format `table` and `null`. - --stats Show stats after query execution in stderr, only works with non-interactive mode. - --time[=