diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..40b343f1e
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,5 @@
+# global codeowners
+* @ayushdg @charlesbluca @galipremsagar
+
+# rust codeowners
+dask_planner/ @ayushdg @galipremsagar @jdye64
diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
new file mode 100644
index 000000000..7f576cd69
--- /dev/null
+++ b/.github/actions/setup-builder/action.yaml
@@ -0,0 +1,17 @@
+name: Prepare Rust Builder
+description: 'Prepare Rust Build Environment'
+inputs:
+ rust-version:
+ description: 'version of rust to install (e.g. stable)'
+ required: true
+ default: 'stable'
+runs:
+ using: "composite"
+ steps:
+ - name: Setup Rust toolchain
+ shell: bash
+ run: |
+ echo "Installing ${{ inputs.rust-version }}"
+ rustup toolchain install ${{ inputs.rust-version }}
+ rustup default ${{ inputs.rust-version }}
+ rustup component add rustfmt
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
index 54e69b5fc..78047147a 100644
--- a/.github/workflows/conda.yml
+++ b/.github/workflows/conda.yml
@@ -3,7 +3,18 @@ on:
push:
branches:
- main
+ - datafusion-sql-planner
pull_request:
+ paths:
+ - setup.py
+ - dask_planner/Cargo.toml
+ - dask_planner/Cargo.lock
+ - dask_planner/pyproject.toml
+ - dask_planner/rust-toolchain.toml
+ - continuous_integration/recipe/**
+ - .github/workflows/conda.yml
+ schedule:
+ - cron: '0 0 * * 0'
# When this workflow is queued, automatically cancel any previous running
# or pending jobs from the same branch
@@ -49,12 +60,12 @@ jobs:
- name: Upload conda package
if: |
github.event_name == 'push'
- && github.ref == 'refs/heads/main'
&& github.repository == 'dask-contrib/dask-sql'
env:
ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
+ LABEL: ${{ github.ref == 'refs/heads/datafusion-sql-planner' && 'dev_datafusion' || 'dev' }}
run: |
# install anaconda for upload
mamba install anaconda-client
- anaconda upload --label dev noarch/*.tar.bz2
+ anaconda upload --label $LABEL linux-64/*.tar.bz2
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
new file mode 100644
index 000000000..0b8b8536e
--- /dev/null
+++ b/.github/workflows/rust.yml
@@ -0,0 +1,72 @@
+name: Rust
+
+on:
+ # always trigger on PR
+ push:
+ pull_request:
+ # manual trigger
+ # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
+ workflow_dispatch:
+
+jobs:
+ # Check crate compiles
+ linux-build-lib:
+ name: cargo check
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ env:
+ # Disable full debug symbol generation to speed up CI build and keep memory down
+ # "1" means line tables only, which is useful for panic tracebacks.
+ RUSTFLAGS: "-C debuginfo=1"
+ steps:
+ - uses: actions/checkout@v3
+ - name: Cache Cargo
+ uses: actions/cache@v3
+ with:
+ # these represent dependencies downloaded by cargo
+ # and thus do not depend on the OS, arch nor rust version.
+ path: /github/home/.cargo
+ key: cargo-cache-
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Check workspace in debug mode
+ run: |
+ cd dask_planner
+ cargo check
+ - name: Check workspace in release mode
+ run: |
+ cd dask_planner
+ cargo check --release
+
+ # test the crate
+ linux-test:
+ name: cargo test (amd64)
+ needs: [linux-build-lib]
+ runs-on: ubuntu-latest
+ container:
+ image: amd64/rust
+ env:
+ # Disable full debug symbol generation to speed up CI build and keep memory down
+ # "1" means line tables only, which is useful for panic tracebacks.
+ RUSTFLAGS: "-C debuginfo=1"
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: true
+ - name: Cache Cargo
+ uses: actions/cache@v3
+ with:
+ path: /github/home/.cargo
+ # this key equals the ones on `linux-build-lib` for re-use
+ key: cargo-cache-
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Run tests
+ run: |
+ cd dask_planner
+ cargo test
diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml
index 7f41a4d6c..fd0cae327 100644
--- a/.github/workflows/test-upstream.yml
+++ b/.github/workflows/test-upstream.yml
@@ -10,58 +10,24 @@ defaults:
shell: bash -l {0}
jobs:
- build:
- # This build step should be similar to the deploy build, to make sure we actually test
- # the future deployable
- name: Build the jar on ubuntu
- runs-on: ubuntu-latest
- if: github.repository == 'dask-contrib/dask-sql'
- steps:
- - uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
- - name: Set up Python
- uses: conda-incubator/setup-miniconda@v2
- with:
- miniforge-variant: Mambaforge
- use-mamba: true
- python-version: "3.8"
- channel-priority: strict
- activate-environment: dask-sql
- environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
- - name: Install dependencies and build the jar
- run: |
- python setup.py build_ext
- - name: Upload the jar
- uses: actions/upload-artifact@v1
- with:
- name: jar
- path: dask_sql/jar/DaskSQL.jar
-
test-dev:
- name: "Test upstream dev (${{ matrix.os }}, java: ${{ matrix.java }}, python: ${{ matrix.python }})"
- needs: build
+ name: "Test upstream dev (${{ matrix.os }}, python: ${{ matrix.python }})"
runs-on: ${{ matrix.os }}
+ if: github.repository == 'dask-contrib/dask-sql'
env:
- CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-jdk${{ matrix.java }}-dev.yaml
+ CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+ defaults:
+ run:
+ shell: bash -l {0}
strategy:
fail-fast: false
matrix:
- java: [8, 11]
os: [ubuntu-latest, windows-latest]
python: ["3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0 # Fetch all history for all branches and tags.
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk${{ matrix.java }}-${{ hashFiles('**/pom.xml') }}
- name: Set up Python
uses: conda-incubator/setup-miniconda@v2
with:
@@ -72,21 +38,21 @@ jobs:
channels: dask/label/dev,conda-forge,nodefaults
activate-environment: dask-sql
environment-file: ${{ env.CONDA_FILE }}
- - name: Download the pre-build jar
- uses: actions/download-artifact@v1
+ - name: Setup Rust Toolchain
+ uses: actions-rs/toolchain@v1
+ id: rust-toolchain
with:
- name: jar
- path: dask_sql/jar/
+ toolchain: stable
+ override: true
+ - name: Build the Rust DataFusion bindings
+ run: |
+ python setup.py build install
- name: Install hive testing dependencies for Linux
if: matrix.os == 'ubuntu-latest'
run: |
mamba install -c conda-forge sasl>=0.3.1
docker pull bde2020/hive:2.3.2-postgresql-metastore
docker pull bde2020/hive-metastore-postgresql:2.3.0
- - name: Set proper JAVA_HOME for Windows
- if: matrix.os == 'windows-latest'
- run: |
- echo "JAVA_HOME=${{ env.CONDA }}\envs\dask-sql\Library" >> $GITHUB_ENV
- name: Install upstream dev Dask / dask-ml
run: |
mamba update dask
@@ -101,11 +67,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
- name: Set up Python
uses: conda-incubator/setup-miniconda@v2
with:
@@ -115,12 +76,16 @@ jobs:
channel-priority: strict
channels: dask/label/dev,conda-forge,nodefaults
activate-environment: dask-sql
- environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml
- - name: Download the pre-build jar
- uses: actions/download-artifact@v1
+ environment-file: continuous_integration/environment-3.9-dev.yaml
+ - name: Setup Rust Toolchain
+ uses: actions-rs/toolchain@v1
+ id: rust-toolchain
with:
- name: jar
- path: dask_sql/jar/
+ toolchain: stable
+ override: true
+ - name: Build the Rust DataFusion bindings
+ run: |
+ python setup.py build install
- name: Install cluster dependencies
run: |
mamba install python-blosc lz4 -c conda-forge
@@ -151,11 +116,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
- name: Set up Python
uses: conda-incubator/setup-miniconda@v2
with:
@@ -163,11 +123,15 @@ jobs:
mamba-version: "*"
channels: dask/label/dev,conda-forge,nodefaults
channel-priority: strict
- - name: Download the pre-build jar
- uses: actions/download-artifact@v1
+ - name: Setup Rust Toolchain
+ uses: actions-rs/toolchain@v1
+ id: rust-toolchain
with:
- name: jar
- path: dask_sql/jar/
+ toolchain: stable
+ override: true
+ - name: Build the Rust DataFusion bindings
+ run: |
+ python setup.py build install
- name: Install upstream dev Dask / dask-ml
if: needs.detect-ci-trigger.outputs.triggered == 'true'
run: |
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c0319afd1..ed9c25a74 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,7 +1,7 @@
---
# Test the main branch and every pull request by
-# 1. building the jar on ubuntu
-# 2. testing code (using the build jar) on ubuntu and windows, with different java versions
+# 1. build dask_planner (Arrow DataFusion Rust bindings) on ubuntu
+# 2. testing code (using the build DataFusion bindings) on ubuntu and windows
name: Test Python package
on:
push:
@@ -36,55 +36,20 @@ jobs:
with:
keyword: "[test-upstream]"
- build:
- # This build step should be similar to the deploy build, to make sure we actually test
- # the future deployable
- name: Build the jar on ubuntu
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
- - name: Set up Python
- uses: conda-incubator/setup-miniconda@v2
- with:
- miniforge-variant: Mambaforge
- use-mamba: true
- python-version: "3.8"
- channel-priority: strict
- activate-environment: dask-sql
- environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
- - name: Build the jar
- run: |
- python setup.py build_ext
- - name: Upload the jar
- uses: actions/upload-artifact@v1
- with:
- name: jar
- path: dask_sql/jar/DaskSQL.jar
-
test:
- name: "Test (${{ matrix.os }}, java: ${{ matrix.java }}, python: ${{ matrix.python }})"
- needs: [detect-ci-trigger, build]
+ name: "Build & Test (${{ matrix.os }}, python: ${{ matrix.python }}, Rust: ${{ matrix.toolchain }})"
+ needs: [detect-ci-trigger]
runs-on: ${{ matrix.os }}
env:
- CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-jdk${{ matrix.java }}-dev.yaml
+ CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
strategy:
fail-fast: false
matrix:
- java: [8, 11]
os: [ubuntu-latest, windows-latest]
python: ["3.8", "3.9", "3.10"]
+ toolchain: [stable]
steps:
- uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk${{ matrix.java }}-${{ hashFiles('**/pom.xml') }}
- name: Set up Python
uses: conda-incubator/setup-miniconda@v2
with:
@@ -95,21 +60,21 @@ jobs:
channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
activate-environment: dask-sql
environment-file: ${{ env.CONDA_FILE }}
- - name: Download the pre-build jar
- uses: actions/download-artifact@v1
+ - name: Setup Rust Toolchain
+ uses: actions-rs/toolchain@v1
+ id: rust-toolchain
with:
- name: jar
- path: dask_sql/jar/
+ toolchain: stable
+ override: true
+ - name: Build the Rust DataFusion bindings
+ run: |
+ python setup.py build install
- name: Install hive testing dependencies for Linux
if: matrix.os == 'ubuntu-latest'
run: |
mamba install -c conda-forge sasl>=0.3.1
docker pull bde2020/hive:2.3.2-postgresql-metastore
docker pull bde2020/hive-metastore-postgresql:2.3.0
- - name: Set proper JAVA_HOME for Windows
- if: matrix.os == 'windows-latest'
- run: |
- echo "JAVA_HOME=${{ env.CONDA }}\envs\dask-sql\Library" >> $GITHUB_ENV
- name: Optionally install upstream dev Dask / dask-ml
if: needs.detect-ci-trigger.outputs.triggered == 'true'
run: |
@@ -130,15 +95,10 @@ jobs:
cluster:
name: "Test in a dask cluster"
- needs: [detect-ci-trigger, build]
+ needs: [detect-ci-trigger]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
- name: Set up Python
uses: conda-incubator/setup-miniconda@v2
with:
@@ -148,13 +108,17 @@ jobs:
channel-priority: strict
channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
activate-environment: dask-sql
- environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml
- - name: Download the pre-build jar
- uses: actions/download-artifact@v1
+ environment-file: continuous_integration/environment-3.9-dev.yaml
+ - name: Setup Rust Toolchain
+ uses: actions-rs/toolchain@v1
+ id: rust-toolchain
with:
- name: jar
- path: dask_sql/jar/
- - name: Install cluster dependencies
+ toolchain: stable
+ override: true
+ - name: Build the Rust DataFusion bindings
+ run: |
+ python setup.py build install
+ - name: Install dependencies
run: |
mamba install python-blosc lz4 -c conda-forge
@@ -187,15 +151,10 @@ jobs:
import:
name: "Test importing with bare requirements"
- needs: [detect-ci-trigger, build]
+ needs: [detect-ci-trigger]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
- name: Set up Python
uses: conda-incubator/setup-miniconda@v2
with:
@@ -203,18 +162,9 @@ jobs:
mamba-version: "*"
channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
channel-priority: strict
- - name: Download the pre-build jar
- uses: actions/download-artifact@v1
- with:
- name: jar
- path: dask_sql/jar/
- - name: Optionally install upstream dev Dask / dask-ml
- if: needs.detect-ci-trigger.outputs.triggered == 'true'
- run: |
- mamba update dask
- python -m pip install --no-deps git+https://github.com/dask/dask-ml
- name: Install dependencies and nothing else
run: |
+ conda install setuptools-rust
pip install -e .
which python
diff --git a/.gitignore b/.gitignore
index 947f81393..245817fc1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,7 @@ share/python-wheels/
.installed.cfg
*.egg
MANIFEST
+*.so
# Unit test / coverage reports
htmlcov/
@@ -32,6 +33,7 @@ coverage.xml
*.cover
.pytest_cache/
.hypothesis/
+.pytest-html
# Jupyter Notebook
.ipynb_checkpoints
@@ -59,3 +61,9 @@ dask_sql/jar
dask-worker-space/
node_modules/
docs/source/_build/
+tests/unit/queries
+tests/unit/data
+
+# Ignore development specific local testing files
+dev_tests
+dev-tests
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 50af0bbb4..f1feccf47 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -16,8 +16,17 @@ repos:
args:
- "--profile"
- "black"
+ - repo: https://github.com/doublify/pre-commit-rust
+ rev: v1.0
+ hooks:
+ - id: fmt
+ args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--']
+ - id: cargo-check
+ args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--']
+ - id: clippy
+ args: ['--manifest-path', './dask_planner/Cargo.toml', '--verbose', '--', '-D', 'warnings']
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v3.2.0
+ rev: v4.2.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 3b3682543..831ec6d50 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -3,15 +3,15 @@ version: 2
build:
os: ubuntu-20.04
tools:
- python: "3.8"
- apt_packages:
- - maven
+ python: "mambaforge-4.10"
sphinx:
configuration: docs/source/conf.py
+conda:
+ environment: docs/environment.yml
+
python:
install:
- - requirements: docs/requirements-docs.txt
- method: pip
path: .
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000..b7fdf1de7
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,127 @@
+# Contributing to Dask-SQL
+
+## Environment Setup
+
+Conda is used both by CI and the development team. Therefore Conda is the fully supported and preferred method for using and developing Dask-SQL.
+
+Installing Conda is outside the scope of this document. However a nice guide for installing on Linux can be found [here](https://docs.conda.io/projects/conda/en/latest/user-guide/install/linux.html)
+
+Setting up your Conda environment for development is straightforward. First, clone the repository locally
+```
+DASK_SQL_HOME=${pwd}/dask-sql
+git clone https://github.com/dask-contrib/dask-sql.git
+cd $DASK_SQL_HOME
+```
+Then, run:
+```
+conda env create -f {DASK_SQL_HOME}/continuous_integration/environment-3.10-dev.yaml
+```
+
+The Conda process will take awhile to complete, once finished you will have a resulting environment named `dask-sql` which can be activated and used by running `conda activate dask-sql`
+
+## Rust Developers Guide
+
+Dask-SQL utilizes [Apache Arrow Datafusion](https://github.com/apache/arrow-datafusion) for parsing, planning, and optimizing SQL queries. DataFusion is written in Rust and therefore requires some Rust experience to be productive. Luckily, there are tons of great Rust learning resources on the internet. We have listed some of our favorite ones [here](#rust-learning-resources)
+
+### Apache Arrow DataFusion
+The Dask-SQL Rust codebase makes heavy use [Apache Arrow DataFusion](https://github.com/apache/arrow-datafusion). Contributors should familiarize themselves with the [codebase](https://github.com/apache/arrow-datafusion) and [documentation](https://docs.rs/datafusion/latest/datafusion/).
+
+#### Purpose
+DataFusion provides Dask-SQL with key functionality.
+- Parsing SQL query strings into a `LogicalPlan` datastructure
+- Future integration points with [substrait.io](https://substrait.io/)
+- An optimization framework used as the baseline for creating custom highly efficient `LogicalPlan`s specific to Dask.
+
+### Building
+Building the Dask-SQL Rust codebase is a straightforward process. If you create and activate the Dask-SQL Conda environment the Rust compiler and all necessary components will be installed for you during that process and therefore requires no further manual setup.
+
+`setuptools-rust` is used by Dask-SQL for building and bundling the resulting Rust binaries. This helps make building and installing the Rust binaries feel much more like a native Python workflow.
+
+More details about the building setup can be found at [setup.py](setup.py) and searching for `rust_extensions` which is the hook for the Rust code build and inclusion.
+
+Note that while `setuptools-rust` is used by CI and should be used during your development cycle, if the need arises to do something more specific that is not yet supported by `setuptools-rust` you can opt to use `cargo` directly from the command line.
+
+#### Building with Python
+Building Dask-SQL is straightforward with Python. To build run ```python setup.py install```. This will build both the Rust and Python codebase and install it into your locally activated conda environment. While not required, if you have updated dependencies for Rust you might prefer a clean build. To clean your setup run ```python setup.py clean``` and then run ```python setup.py install```
+
+#### DataFusion Modules
+DataFusion is broken down into a few modules. We consume those modules in our [Cargo.toml](dask_planner/Cargo.toml). The modules that we use currently are
+
+- `datafusion-common` - Datastructures and core logic
+- `datafusion-expr` - Expression based logic and operators
+- `datafusion-sql` - SQL components such as parsing and planning
+- `datafusion-optimizer` - Optimization logic and datastructures for modifying current plans into more efficient ones.
+
+#### Retrieving Upstream Dependencies
+During development you might find yourself needing some upstream DataFusion changes not present in the projects current version. Luckily this can easily be achieved by updating [Cargo.toml](dask_planner/Cargo.toml) and changing the `rev` to the SHA of the version you need. Note that the same SHA should be used for all DataFusion modules.
+
+After updating the `Cargo.toml` file the codebase can be re-built to reflect those changes by running `python setup.py install`
+
+#### Local Documentation
+Sometimes when building against the latest Github commits for DataFusion you may find that the features you are consuming do not have their documentation public yet. In this case it can be helpful to build the DataFusion documentation locally so that it can be referenced to assist with development. Here is a rough outline for building that documentation locally.
+
+- clone https://github.com/apache/arrow-datafusion
+- change into the `arrow-datafusion` directory
+- run `cargo doc`
+- navigate to `target/doc/datafusion/all.html` and open in your desired browser
+
+### Datastructures
+While working in the Rust codebase there are a few datastructures that you should make yourself familiar with. This section does not aim to verbosely list out all of the datastructure with in the project but rather just the key datastructures that you are likely to encounter while working on almost any feature/issue. The aim is to give you a better overview of the codebase without having to manually dig through the all the source code.
+
+- [`PyLogicalPlan`](dask_planner/src/sql/logical.rs) -> [DataFusion LogicalPlan](https://docs.rs/datafusion/latest/datafusion/logical_plan/enum.LogicalPlan.html)
+ - Often encountered in Python code with variable name `rel`
+ - Python serializable umbrella representation of the entire LogicalPlan that was generated by DataFusion
+ - Provides access to `DaskTable` instances and type information for each table
+ - Access to individual nodes in the logical plan tree. Ex: `TableScan`
+- [`DaskSQLContext`](dask_planner/src/sql.rs)
+ - Analogous to Python `Context`
+ - Contains metadata about the tables, schemas, functions, operators, and configurations that are persent within the current execution context
+ - When adding custom functions/UDFs this is the location that you would register them
+ - Entry point for parsing SQL strings to sql node trees. This is the location Python will begin its interactions with Rust
+- [`PyExpr`](dask_planner/src/expression.rs) -> [DataFusion Expr](https://docs.rs/datafusion/latest/datafusion/prelude/enum.Expr.html)
+ - Arguably where most of your time will be spent
+ - Represents a single node in sql tree. Ex: `avg(age)` from `SELECT avg(age) FROM people`
+ - Is associate with a single `RexType`
+ - Can contain literal values or represent function calls, `avg()` for example
+ - The expressions "index" in the tree can be retrieved by calling `PyExpr.index()` on an instance. This is useful when mapping frontend column names in Dask code to backend Dataframe columns
+ - Certain `PyExpr`s contain operands. Ex: `2 + 2` would contain 3 operands. 1) A literal `PyExpr` instance with value 2 2) Another literal `PyExpr` instance with a value of 2. 3) A `+` `PyExpr` representing the addition of the 2 literals.
+- [`DaskSqlOptimizer`](dask_planner/src/sql/optimizer.rs)
+ - Registering location for all Dask-SQL specific logical plan optimizations
+ - Optimizations that are written either custom or use from another source, DataFusion, are registered here in the order they are wished to be executed
+ - Represents functions that modify/convert an original `PyLogicalPlan` into another `PyLogicalPlan` that would be more efficient when running in the underlying Dask framework
+- [`RelDataType`](dask_planner/src/sql/types/rel_data_type.rs)
+ - Not a fan of this name, was chosen to match existing Calcite logic
+ - Represents a "row" in a table
+ - Contains a list of "columns" that are present in that row
+ - [RelDataTypeField](dask_planner/src/sql/types/rel_data_type_field.rs)
+- [RelDataTypeField](dask_planner/src/sql/types/rel_data_type_field.rs)
+ - Represents an individual column in a table
+ - Contains:
+ - `qualifier` - schema the field belongs to
+ - `name` - name of the column/field
+ - `data_type` - `DaskTypeMap` instance containing information about the SQL type and underlying Arrow DataType
+ - `index` - location of the field in the LogicalPlan
+- [DaskTypeMap](dask_planner/src/sql/types.rs)
+ - Maps a conventional SQL type to an underlying Arrow DataType
+
+
+### Rust Learning Resources
+- ["The Book"](https://doc.rust-lang.org/book/)
+- [Lets Get Rusty "LGR" YouTube series](https://www.youtube.com/c/LetsGetRusty)
+
+## Documentation TODO
+- [ ] SQL Parsing overview diagram
+- [ ] Architecture diagram
+- [x] Setup dev environment
+- [x] Version of Rust and specs
+- [x] Updating version of datafusion
+- [x] Building
+- [x] Rust learning resources
+- [x] Rust Datastructures local to Dask-SQL
+- [x] Build DataFusion documentation locally
+- [ ] Python & Rust with PyO3
+- [ ] Types mapping, Arrow datatypes
+- [ ] RexTypes explaination, show simple query and show it broken down into its parts in a diagram
+- [ ] Registering tables with DaskSqlContext, also functions
+- [ ] Creating your own optimizer
+- [ ] Simple diagram of PyExpr, showing something like 2+2 but broken down into a tree looking diagram
diff --git a/MANIFEST.in b/MANIFEST.in
index 4b4310eee..d0108fedd 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
recursive-include dask_sql *.yaml
-recursive-include planner *
+recursive-include dask_planner *
include versioneer.py
include dask_sql/_version.py
diff --git a/README.md b/README.md
index 7651c8196..b97640bf7 100644
--- a/README.md
+++ b/README.md
@@ -89,17 +89,7 @@ Install the package from the `conda-forge` channel:
### With `pip`
-`dask-sql` needs Java for the parsing of the SQL queries.
-Make sure you have a running java installation with version >= 8.
-
-To test if you have Java properly installed and set up, run
-
- $ java -version
- openjdk version "1.8.0_152-release"
- OpenJDK Runtime Environment (build 1.8.0_152-release-1056-b12)
- OpenJDK 64-Bit Server VM (build 25.152-b12, mixed mode)
-
-After installing Java, you can install the package with
+You can install the package with
pip install dask-sql
@@ -111,19 +101,18 @@ If you want to have the newest (unreleased) `dask-sql` version or if you plan to
Create a new conda environment and install the development environment:
- conda env create -f continuous_integration/environment-3.9-jdk11-dev.yaml
+ conda env create -f continuous_integration/environment-3.9-dev.yaml
It is not recommended to use `pip` instead of `conda` for the environment setup.
-If you however need to, make sure to have Java (jdk >= 8) and maven installed and correctly setup before continuing.
-Have a look into `environment-3.9-jdk11-dev.yaml` for the rest of the development environment.
After that, you can install the package in development mode
pip install -e ".[dev]"
-To recompile the Java classes after changes have been made to the source contained in `planner/`, run
+The Rust DataFusion bindings are built as part of the `pip install`.
+If changes are made to the Rust source in `dask_planner/`, another build/install must be run to recompile the bindings:
- python setup.py build_ext
+ python setup.py build install
This repository uses [pre-commit](https://pre-commit.com/) hooks. To install them, call
@@ -195,5 +184,5 @@ At the core, `dask-sql` does two things:
- translate the SQL query using [Apache Calcite](https://calcite.apache.org/) into a relational algebra, which is specified as a tree of java objects - similar to many other SQL engines (Hive, Flink, ...)
- convert this description of the query from java objects into dask API calls (and execute them) - returning a dask dataframe.
-For the first step, Apache Calcite needs to know about the columns and types of the dask dataframes, therefore some java classes to store this information for dask dataframes are defined in `planner`.
-After the translation to a relational algebra is done (using `RelationalAlgebraGenerator.getRelationalAlgebra`), the python methods defined in `dask_sql.physical` turn this into a physical dask execution plan by converting each piece of the relational algebra one-by-one.
+For the first step, Arrow DataFusion needs to know about the columns and types of the dask dataframes, therefore some Rust code to store this information for dask dataframes are defined in `dask_planner`.
+After the translation to a relational algebra is done (using `DaskSQLContext.logical_relational_algebra`), the python methods defined in `dask_sql.physical` turn this into a physical dask execution plan by converting each piece of the relational algebra one-by-one.
diff --git a/conftest.py b/conftest.py
index 6f38951e1..594208456 100644
--- a/conftest.py
+++ b/conftest.py
@@ -5,8 +5,11 @@
def pytest_addoption(parser):
parser.addoption("--rungpu", action="store_true", help="run tests meant for GPU")
+ parser.addoption("--runqueries", action="store_true", help="run test queries")
def pytest_runtest_setup(item):
if "gpu" in item.keywords and not item.config.getoption("--rungpu"):
pytest.skip("need --rungpu option to run")
+ if "queries" in item.keywords and not item.config.getoption("--runqueries"):
+ pytest.skip("need --runqueries option to run")
diff --git a/continuous_integration/environment-3.10-jdk11-dev.yaml b/continuous_integration/environment-3.10-dev.yaml
similarity index 84%
rename from continuous_integration/environment-3.10-jdk11-dev.yaml
rename to continuous_integration/environment-3.10-dev.yaml
index 5dd8b19d6..47e7a7d2d 100644
--- a/continuous_integration/environment-3.10-jdk11-dev.yaml
+++ b/continuous_integration/environment-3.10-dev.yaml
@@ -8,15 +8,13 @@ dependencies:
- fastapi>=0.69.0
- fugue>=0.7.0
- intake>=0.6.0
-- jpype1>=1.0.2
- jsonschema
- lightgbm
-- maven
+- maturin>=0.12.8
- mlflow
- mock
- nest-asyncio
-- openjdk=11
-- pandas>=1.1.2
+- pandas>=1.4.0
- pre-commit
- prompt_toolkit
- psycopg2
@@ -27,7 +25,9 @@ dependencies:
- pytest-xdist
- pytest
- python=3.10
+- rust=1.62.1
- scikit-learn>=1.0.0
+- setuptools-rust>=1.4.1
- sphinx
- tpot
- tzlocal>=2.1
diff --git a/continuous_integration/environment-3.10-jdk8-dev.yaml b/continuous_integration/environment-3.10-jdk8-dev.yaml
deleted file mode 100644
index f93609bd6..000000000
--- a/continuous_integration/environment-3.10-jdk8-dev.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: dask-sql
-channels:
-- conda-forge
-- nodefaults
-dependencies:
-- dask-ml>=2022.1.22
-- dask>=2022.3.0
-- fastapi>=0.69.0
-- fugue>=0.7.0
-- intake>=0.6.0
-- jpype1>=1.0.2
-- jsonschema
-- lightgbm
-- maven
-- mlflow
-- mock
-- nest-asyncio
-- openjdk=8
-- pandas>=1.1.2
-- pre-commit
-- prompt_toolkit
-- psycopg2
-- pyarrow>=6.0.1
-- pygments
-- pyhive
-- pytest-cov
-- pytest-xdist
-- pytest
-- python=3.10
-- scikit-learn>=1.0.0
-- sphinx
-- tpot
-- tzlocal>=2.1
-- uvicorn>=0.11.3
diff --git a/continuous_integration/environment-3.8-jdk11-dev.yaml b/continuous_integration/environment-3.8-dev.yaml
similarity index 85%
rename from continuous_integration/environment-3.8-jdk11-dev.yaml
rename to continuous_integration/environment-3.8-dev.yaml
index 638bf743f..1c3185f11 100644
--- a/continuous_integration/environment-3.8-jdk11-dev.yaml
+++ b/continuous_integration/environment-3.8-dev.yaml
@@ -8,15 +8,13 @@ dependencies:
- fastapi=0.69.0
- fugue=0.7.0
- intake=0.6.0
-- jpype1=1.0.2
- jsonschema
- lightgbm
-- maven
+- maturin=0.12.8
- mlflow
- mock
- nest-asyncio
-- openjdk=11
-- pandas=1.1.2
+- pandas=1.4.0
- pre-commit
- prompt_toolkit
- psycopg2
@@ -27,7 +25,9 @@ dependencies:
- pytest-xdist
- pytest
- python=3.8
+- rust=1.62.1
- scikit-learn=1.0.0
+- setuptools-rust=1.4.1
- sphinx
- tpot
- tzlocal=2.1
diff --git a/continuous_integration/environment-3.8-jdk8-dev.yaml b/continuous_integration/environment-3.8-jdk8-dev.yaml
deleted file mode 100644
index 1b6fc69a4..000000000
--- a/continuous_integration/environment-3.8-jdk8-dev.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: dask-sql
-channels:
-- conda-forge
-- nodefaults
-dependencies:
-- dask-ml=2022.1.22
-- dask=2022.3.0
-- fastapi=0.69.0
-- fugue=0.7.0
-- intake=0.6.0
-- jpype1=1.0.2
-- jsonschema
-- lightgbm
-- maven
-- mlflow
-- mock
-- nest-asyncio
-- openjdk=8
-- pandas=1.1.2
-- pre-commit
-- prompt_toolkit
-- psycopg2
-- pyarrow=6.0.1
-- pygments
-- pyhive
-- pytest-cov
-- pytest-xdist
-- pytest
-- python=3.8
-- scikit-learn=1.0.0
-- sphinx
-- tpot
-- tzlocal=2.1
-- uvicorn=0.11.3
diff --git a/continuous_integration/environment-3.9-jdk11-dev.yaml b/continuous_integration/environment-3.9-dev.yaml
similarity index 84%
rename from continuous_integration/environment-3.9-jdk11-dev.yaml
rename to continuous_integration/environment-3.9-dev.yaml
index 1751f685b..60335b7e2 100644
--- a/continuous_integration/environment-3.9-jdk11-dev.yaml
+++ b/continuous_integration/environment-3.9-dev.yaml
@@ -8,15 +8,13 @@ dependencies:
- fastapi>=0.69.0
- fugue>=0.7.0
- intake>=0.6.0
-- jpype1>=1.0.2
- jsonschema
- lightgbm
-- maven
+- maturin>=0.12.8
- mlflow
- mock
- nest-asyncio
-- openjdk=11
-- pandas>=1.1.2
+- pandas>=1.4.0
- pre-commit
- prompt_toolkit
- psycopg2
@@ -27,7 +25,9 @@ dependencies:
- pytest-xdist
- pytest
- python=3.9
+- rust=1.62.1
- scikit-learn>=1.0.0
+- setuptools-rust>=1.4.1
- sphinx
- tpot
- tzlocal>=2.1
diff --git a/continuous_integration/environment-3.9-jdk8-dev.yaml b/continuous_integration/environment-3.9-jdk8-dev.yaml
deleted file mode 100644
index 5d42c22a7..000000000
--- a/continuous_integration/environment-3.9-jdk8-dev.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: dask-sql
-channels:
-- conda-forge
-- nodefaults
-dependencies:
-- dask-ml>=2022.1.22
-- dask>=2022.3.0
-- fastapi>=0.69.0
-- fugue>=0.7.0
-- intake>=0.6.0
-- jpype1>=1.0.2
-- jsonschema
-- lightgbm
-- maven
-- mlflow
-- mock
-- nest-asyncio
-- openjdk=8
-- pandas>=1.1.2
-- pre-commit
-- prompt_toolkit
-- psycopg2
-- pyarrow>=6.0.1
-- pygments
-- pyhive
-- pytest-cov
-- pytest-xdist
-- pytest
-- python=3.9
-- scikit-learn>=1.0.0
-- sphinx
-- tpot
-- tzlocal>=2.1
-- uvicorn>=0.11.3
diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh
index 38cb8c00a..cdcb4a6e9 100644
--- a/continuous_integration/gpuci/build.sh
+++ b/continuous_integration/gpuci/build.sh
@@ -17,9 +17,6 @@ export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
# Set home to the job's workspace
export HOME="$WORKSPACE"
-# specify maven options
-export MAVEN_OPTS="-Dmaven.repo.local=${WORKSPACE}/.m2/repository"
-
# Switch to project root; also root of repo checkout
cd "$WORKSPACE"
@@ -40,6 +37,15 @@ gpuci_logger "Activate conda env"
. /opt/conda/etc/profile.d/conda.sh
conda activate dask_sql
+gpuci_logger "Install awscli"
+gpuci_mamba_retry install -y -c conda-forge awscli
+
+gpuci_logger "Download parquet dataset"
+gpuci_retry aws s3 cp --only-show-errors "${DASK_SQL_BUCKET_NAME}parquet_2gb/" tests/unit/data/ --recursive
+
+gpuci_logger "Download query files"
+gpuci_retry aws s3 cp --only-show-errors "${DASK_SQL_BUCKET_NAME}queries/" tests/unit/queries/ --recursive
+
gpuci_logger "Install dask"
python -m pip install git+https://github.com/dask/dask
@@ -58,4 +64,4 @@ conda config --show-sources
conda list --show-channel-urls
gpuci_logger "Python py.test for dask-sql"
-py.test $WORKSPACE -n 4 -v -m gpu --rungpu --junitxml="$WORKSPACE/junit-dask-sql.xml" --cov-config="$WORKSPACE/.coveragerc" --cov=dask_sql --cov-report=xml:"$WORKSPACE/dask-sql-coverage.xml" --cov-report term
+py.test $WORKSPACE -n 4 -v -m gpu --runqueries --rungpu --junitxml="$WORKSPACE/junit-dask-sql.xml" --cov-config="$WORKSPACE/.coveragerc" --cov=dask_sql --cov-report=xml:"$WORKSPACE/dask-sql-coverage.xml" --cov-report term
diff --git a/continuous_integration/gpuci/environment.yaml b/continuous_integration/gpuci/environment.yaml
index 2e4eff82d..efbc41122 100644
--- a/continuous_integration/gpuci/environment.yaml
+++ b/continuous_integration/gpuci/environment.yaml
@@ -11,15 +11,13 @@ dependencies:
- fastapi>=0.69.0
- fugue>=0.7.0
- intake>=0.6.0
-- jpype1>=1.0.2
- jsonschema
- lightgbm
-- maven
+- maturin>=0.12.8
- mlflow
- mock
- nest-asyncio
-- openjdk=11
-- pandas>=1.1.2
+- pandas>=1.4.0
- pre-commit
- prompt_toolkit
- psycopg2
@@ -30,7 +28,9 @@ dependencies:
- pytest-xdist
- pytest
- python=3.9
+- rust=1.62.1
- scikit-learn>=1.0.0
+- setuptools-rust>=1.4.1
- sphinx
- tpot
- tzlocal>=2.1
diff --git a/continuous_integration/recipe/conda_build_config.yaml b/continuous_integration/recipe/conda_build_config.yaml
new file mode 100644
index 000000000..dcffe42d2
--- /dev/null
+++ b/continuous_integration/recipe/conda_build_config.yaml
@@ -0,0 +1,6 @@
+rust_compiler_version:
+ - 1.62.1
+python:
+ - 3.8
+ - 3.9
+ - 3.10
diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml
index cd5abd580..51d4c5ac0 100644
--- a/continuous_integration/recipe/meta.yaml
+++ b/continuous_integration/recipe/meta.yaml
@@ -13,25 +13,25 @@ source:
build:
number: {{ GIT_DESCRIBE_NUMBER }}
- noarch: python
+ skip: true # [py2k]
entry_points:
- dask-sql-server = dask_sql.server.app:main
- dask-sql = dask_sql.cmd:main
- string: py_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+ string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
script: {{ PYTHON }} -m pip install . --no-deps -vv
requirements:
build:
- - maven >=3.6.0
+ - {{ compiler('rust') }}
+ - setuptools-rust >=1.4.1
host:
- pip
- - python >=3.8
+ - python
+ - setuptools-rust >=1.4.1
run:
- python
- dask >=2022.3.0
- - pandas >=1.1.2
- - jpype1 >=1.0.2
- - openjdk >=8
+ - pandas >=1.4.0
- fastapi >=0.69.0
- uvicorn >=0.11.3
- tzlocal >=2.1
diff --git a/continuous_integration/recipe/run_test.py b/continuous_integration/recipe/run_test.py
index 0ca97261b..01616d1db 100644
--- a/continuous_integration/recipe/run_test.py
+++ b/continuous_integration/recipe/run_test.py
@@ -13,19 +13,21 @@
df = pd.DataFrame({"name": ["Alice", "Bob", "Chris"] * 100, "x": list(range(300))})
ddf = dd.from_pandas(df, npartitions=10)
-c.create_table("my_data", ddf)
-got = c.sql(
- """
- SELECT
- my_data.name,
- SUM(my_data.x) AS "S"
- FROM
- my_data
- GROUP BY
- my_data.name
-"""
-)
-expect = pd.DataFrame({"name": ["Alice", "Bob", "Chris"], "S": [14850, 14950, 15050]})
+# This needs to be temprarily disabled since this query requires features that are not yet implemented
+# c.create_table("my_data", ddf)
+
+# got = c.sql(
+# """
+# SELECT
+# my_data.name,
+# SUM(my_data.x) AS "S"
+# FROM
+# my_data
+# GROUP BY
+# my_data.name
+# """
+# )
+# expect = pd.DataFrame({"name": ["Alice", "Bob", "Chris"], "S": [14850, 14950, 15050]})
-dd.assert_eq(got, expect)
+# dd.assert_eq(got, expect)
diff --git a/dask_planner/.classpath b/dask_planner/.classpath
new file mode 100644
index 000000000..b14b13a76
--- /dev/null
+++ b/dask_planner/.classpath
@@ -0,0 +1,55 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dask_planner/.gitignore b/dask_planner/.gitignore
new file mode 100644
index 000000000..c8f044299
--- /dev/null
+++ b/dask_planner/.gitignore
@@ -0,0 +1,72 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version
diff --git a/dask_planner/.settings/org.eclipse.core.resources.prefs b/dask_planner/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 000000000..92920805e
--- /dev/null
+++ b/dask_planner/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//target/generated-sources/annotations=UTF-8
+encoding/=UTF-8
diff --git a/dask_planner/.settings/org.eclipse.jdt.apt.core.prefs b/dask_planner/.settings/org.eclipse.jdt.apt.core.prefs
new file mode 100644
index 000000000..d4313d4b2
--- /dev/null
+++ b/dask_planner/.settings/org.eclipse.jdt.apt.core.prefs
@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.apt.aptEnabled=false
diff --git a/dask_planner/.settings/org.eclipse.jdt.core.prefs b/dask_planner/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 000000000..1b6e1ef22
--- /dev/null
+++ b/dask_planner/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,9 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
+org.eclipse.jdt.core.compiler.compliance=1.8
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.processAnnotations=disabled
+org.eclipse.jdt.core.compiler.release=disabled
+org.eclipse.jdt.core.compiler.source=1.8
diff --git a/dask_planner/.settings/org.eclipse.m2e.core.prefs b/dask_planner/.settings/org.eclipse.m2e.core.prefs
new file mode 100644
index 000000000..f897a7f1c
--- /dev/null
+++ b/dask_planner/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
diff --git a/dask_planner/Cargo.lock b/dask_planner/Cargo.lock
new file mode 100644
index 000000000..72482d972
--- /dev/null
+++ b/dask_planner/Cargo.lock
@@ -0,0 +1,1419 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.7",
+ "once_cell",
+ "version_check",
+]
+
+[[package]]
+name = "ahash"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57e6e951cfbb2db8de1828d49073a113a29fd7117b1596caa781a258c7e38d72"
+dependencies = [
+ "cfg-if",
+ "const-random",
+ "getrandom 0.2.7",
+ "once_cell",
+ "version_check",
+]
+
+[[package]]
+name = "aho-corasick"
+version = "0.7.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "arrayref"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
+
+[[package]]
+name = "arrow"
+version = "23.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fedc767fbaa36ea50f086215f54f1a007d22046fc4754b0448c657bcbe9f8413"
+dependencies = [
+ "ahash 0.8.0",
+ "arrow-buffer",
+ "bitflags",
+ "chrono",
+ "comfy-table",
+ "csv",
+ "flatbuffers",
+ "half",
+ "hashbrown",
+ "indexmap",
+ "lazy_static",
+ "lexical-core",
+ "multiversion",
+ "num",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "arrow-buffer"
+version = "23.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d290050c6e12a81a24ad08525cef2203c4156a6350f75508d49885d677e88ea9"
+dependencies = [
+ "half",
+ "num",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "blake2"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "blake3"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+ "digest",
+]
+
+[[package]]
+name = "block-buffer"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "lazy_static",
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "chrono"
+version = "0.4.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
+dependencies = [
+ "iana-time-zone",
+ "num-integer",
+ "num-traits",
+ "winapi",
+]
+
+[[package]]
+name = "comfy-table"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121d8a5b0346092c18a4b2fd6f620d7a06f0eb7ac0a45860939a0884bc579c56"
+dependencies = [
+ "strum",
+ "strum_macros",
+ "unicode-width",
+]
+
+[[package]]
+name = "const-random"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f590d95d011aa80b063ffe3253422ed5aa462af4e9867d43ce8337562bac77c4"
+dependencies = [
+ "const-random-macro",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "615f6e27d000a2bffbc7f2f6a8669179378fa27ee4d0a509e985dfc0a7defb40"
+dependencies = [
+ "getrandom 0.2.7",
+ "lazy_static",
+ "proc-macro-hack",
+ "tiny-keccak",
+]
+
+[[package]]
+name = "constant_time_eq"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc948ebb96241bb40ab73effeb80d9f93afaad49359d159a5e61be51619fe813"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crunchy"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+
+[[package]]
+name = "crypto-common"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "csv"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
+dependencies = [
+ "bstr",
+ "csv-core",
+ "itoa 0.4.8",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "dask_planner"
+version = "0.1.0"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-optimizer",
+ "datafusion-sql",
+ "env_logger",
+ "log",
+ "mimalloc",
+ "parking_lot",
+ "pyo3",
+ "rand 0.7.3",
+ "tokio",
+ "uuid",
+]
+
+[[package]]
+name = "datafusion-common"
+version = "12.0.0"
+source = "git+https://github.com/apache/arrow-datafusion/?rev=1261741af2a5e142fa0c7916e759859cc18ea59a#1261741af2a5e142fa0c7916e759859cc18ea59a"
+dependencies = [
+ "arrow",
+ "ordered-float",
+ "sqlparser",
+]
+
+[[package]]
+name = "datafusion-expr"
+version = "12.0.0"
+source = "git+https://github.com/apache/arrow-datafusion/?rev=1261741af2a5e142fa0c7916e759859cc18ea59a#1261741af2a5e142fa0c7916e759859cc18ea59a"
+dependencies = [
+ "ahash 0.8.0",
+ "arrow",
+ "datafusion-common",
+ "sqlparser",
+]
+
+[[package]]
+name = "datafusion-optimizer"
+version = "12.0.0"
+source = "git+https://github.com/apache/arrow-datafusion/?rev=1261741af2a5e142fa0c7916e759859cc18ea59a#1261741af2a5e142fa0c7916e759859cc18ea59a"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "chrono",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "hashbrown",
+ "log",
+]
+
+[[package]]
+name = "datafusion-physical-expr"
+version = "12.0.0"
+source = "git+https://github.com/apache/arrow-datafusion/?rev=1261741af2a5e142fa0c7916e759859cc18ea59a#1261741af2a5e142fa0c7916e759859cc18ea59a"
+dependencies = [
+ "ahash 0.8.0",
+ "arrow",
+ "blake2",
+ "blake3",
+ "chrono",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-row",
+ "hashbrown",
+ "lazy_static",
+ "md-5",
+ "ordered-float",
+ "paste",
+ "rand 0.8.5",
+ "regex",
+ "sha2",
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "datafusion-row"
+version = "12.0.0"
+source = "git+https://github.com/apache/arrow-datafusion/?rev=1261741af2a5e142fa0c7916e759859cc18ea59a#1261741af2a5e142fa0c7916e759859cc18ea59a"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "paste",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "datafusion-sql"
+version = "12.0.0"
+source = "git+https://github.com/apache/arrow-datafusion/?rev=1261741af2a5e142fa0c7916e759859cc18ea59a#1261741af2a5e142fa0c7916e759859cc18ea59a"
+dependencies = [
+ "ahash 0.8.0",
+ "arrow",
+ "datafusion-common",
+ "datafusion-expr",
+ "hashbrown",
+ "sqlparser",
+ "tokio",
+]
+
+[[package]]
+name = "digest"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+ "subtle",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272"
+dependencies = [
+ "atty",
+ "humantime",
+ "log",
+ "regex",
+ "termcolor",
+]
+
+[[package]]
+name = "flatbuffers"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86b428b715fdbdd1c364b84573b5fdc0f84f8e423661b9f398735278bc7f2b6a"
+dependencies = [
+ "bitflags",
+ "smallvec",
+ "thiserror",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "half"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad6a9459c9c30b177b925162351f97e7d967c7ea8bab3b8352805327daf45554"
+dependencies = [
+ "crunchy",
+ "num-traits",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash 0.7.6",
+]
+
+[[package]]
+name = "heck"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
+[[package]]
+name = "iana-time-zone"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "js-sys",
+ "wasm-bindgen",
+ "winapi",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "indoc"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3"
+
+[[package]]
+name = "itoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
+
+[[package]]
+name = "itoa"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
+
+[[package]]
+name = "js-sys"
+version = "0.3.59"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "lexical-core"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46"
+dependencies = [
+ "lexical-parse-float",
+ "lexical-parse-integer",
+ "lexical-util",
+ "lexical-write-float",
+ "lexical-write-integer",
+]
+
+[[package]]
+name = "lexical-parse-float"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f"
+dependencies = [
+ "lexical-parse-integer",
+ "lexical-util",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-parse-integer"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9"
+dependencies = [
+ "lexical-util",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-util"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc"
+dependencies = [
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-write-float"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862"
+dependencies = [
+ "lexical-util",
+ "lexical-write-integer",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-write-integer"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446"
+dependencies = [
+ "lexical-util",
+ "static_assertions",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.132"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
+
+[[package]]
+name = "libm"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565"
+
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11ca136052550448f55df7898c6dbe651c6b574fe38a0d9ea687a9f8088a2e2c"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "lock_api"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "md-5"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "memoffset"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "mimalloc"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f64ad83c969af2e732e907564deb0d0ed393cec4af80776f77dd77a1a427698"
+dependencies = [
+ "libmimalloc-sys",
+]
+
+[[package]]
+name = "multiversion"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373"
+dependencies = [
+ "multiversion-macros",
+]
+
+[[package]]
+name = "multiversion-macros"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "num"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606"
+dependencies = [
+ "num-bigint",
+ "num-complex",
+ "num-integer",
+ "num-iter",
+ "num-rational",
+ "num-traits",
+]
+
+[[package]]
+name = "num-bigint"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
+dependencies = [
+ "autocfg",
+ "num-traits",
+]
+
+[[package]]
+name = "num-iter"
+version = "0.1.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
+dependencies = [
+ "autocfg",
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+ "libm",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"
+
+[[package]]
+name = "ordered-float"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96bcbab4bfea7a59c2c0fe47211a1ac4e3e96bea6eb446d704f310bc5c732ae2"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-sys",
+]
+
+[[package]]
+name = "paste"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9423e2b32f7a043629287a536f21951e8c6a82482d0acb1eeebfc90bc2225b22"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+
+[[package]]
+name = "proc-macro-hack"
+version = "0.5.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12f72538a0230791398a0986a6518ebd88abc3fded89007b506ed072acc831e1"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "parking_lot",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc4cf18c20f4f09995f3554e6bcf9b09bd5e4d6b67c562fdfaafa644526ba479"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a41877f28d8ebd600b6aa21a17b40c3b0fc4dfe73a27b6e81ab3d895e401b0e9"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e81c8d4bcc2f216dc1b665412df35e46d12ee8d3d046b381aad05f1fcf30547"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85752a767ee19399a78272cc2ab625cd7d373b2e112b4b13db28de71fa892784"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom 0.2.7",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "regex"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
+
+[[package]]
+name = "rustversion"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
+
+[[package]]
+name = "ryu"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
+name = "serde"
+version = "1.0.144"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.144"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.85"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
+dependencies = [
+ "itoa 1.0.3",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
+
+[[package]]
+name = "sqlparser"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25"
+dependencies = [
+ "log",
+]
+
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
+[[package]]
+name = "strum"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
+
+[[package]]
+name = "strum_macros"
+version = "0.24.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
+[[package]]
+name = "subtle"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+
+[[package]]
+name = "syn"
+version = "1.0.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1"
+
+[[package]]
+name = "termcolor"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
+name = "tokio"
+version = "1.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581"
+dependencies = [
+ "autocfg",
+ "num_cpus",
+ "once_cell",
+ "parking_lot",
+ "pin-project-lite",
+ "tokio-macros",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "typenum"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+
+[[package]]
+name = "unindent"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58ee9362deb4a96cef4d437d1ad49cffc9b9e92d202b6995674e928ce684f112"
+
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+dependencies = [
+ "getrandom 0.2.7",
+]
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+dependencies = [
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
diff --git a/dask_planner/Cargo.toml b/dask_planner/Cargo.toml
new file mode 100644
index 000000000..51826ab96
--- /dev/null
+++ b/dask_planner/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "dask_planner"
+repository = "https://github.com/dask-contrib/dask-sql"
+version = "0.1.0"
+description = "Bindings for DataFusion used by Dask-SQL"
+readme = "README.md"
+license = "Apache-2.0"
+edition = "2021"
+rust-version = "1.62"
+
+[dependencies]
+arrow = { version = "23.0.0", features = ["prettyprint"] }
+async-trait = "0.1.41"
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion/", rev = "1261741af2a5e142fa0c7916e759859cc18ea59a" }
+env_logger = "0.9"
+log = "^0.4"
+mimalloc = { version = "*", default-features = false }
+parking_lot = "0.12"
+pyo3 = { version = "0.17.1", features = ["extension-module", "abi3", "abi3-py38"] }
+rand = "0.7"
+tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] }
+uuid = { version = "0.8", features = ["v4"] }
+
+[lib]
+crate-type = ["cdylib"]
diff --git a/dask_planner/MANIFEST.in b/dask_planner/MANIFEST.in
new file mode 100644
index 000000000..7c68298bd
--- /dev/null
+++ b/dask_planner/MANIFEST.in
@@ -0,0 +1,2 @@
+include Cargo.toml
+recursive-include src *
diff --git a/dask_planner/README.md b/dask_planner/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/dask_planner/pyproject.toml b/dask_planner/pyproject.toml
new file mode 100644
index 000000000..f153e3f5a
--- /dev/null
+++ b/dask_planner/pyproject.toml
@@ -0,0 +1,11 @@
+[build-system]
+requires = ["setuptools", "wheel", "setuptools-rust"]
+
+[project]
+name = "datafusion_planner"
+requires-python = ">=3.8"
+classifiers = [
+ "Programming Language :: Rust",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy",
+]
diff --git a/dask_planner/rust-toolchain.toml b/dask_planner/rust-toolchain.toml
new file mode 100644
index 000000000..5d56faf9a
--- /dev/null
+++ b/dask_planner/rust-toolchain.toml
@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly"
diff --git a/dask_planner/src/dialect.rs b/dask_planner/src/dialect.rs
new file mode 100644
index 000000000..492f4aca3
--- /dev/null
+++ b/dask_planner/src/dialect.rs
@@ -0,0 +1,40 @@
+use core::iter::Peekable;
+use core::str::Chars;
+use datafusion_sql::sqlparser::dialect::Dialect;
+
+#[derive(Debug)]
+pub struct DaskDialect {}
+
+impl Dialect for DaskDialect {
+ fn is_identifier_start(&self, ch: char) -> bool {
+ // See https://www.postgresql.org/docs/11/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
+ // We don't yet support identifiers beginning with "letters with
+ // diacritical marks and non-Latin letters"
+ ('a'..='z').contains(&ch) || ('A'..='Z').contains(&ch) || ch == '_'
+ }
+
+ fn is_identifier_part(&self, ch: char) -> bool {
+ ('a'..='z').contains(&ch)
+ || ('A'..='Z').contains(&ch)
+ || ('0'..='9').contains(&ch)
+ || ch == '$'
+ || ch == '_'
+ }
+
+ /// Determine if a character starts a quoted identifier. The default
+ /// implementation, accepting "double quoted" ids is both ANSI-compliant
+ /// and appropriate for most dialects (with the notable exception of
+ /// MySQL, MS SQL, and sqlite). You can accept one of characters listed
+ /// in `Word::matching_end_quote` here
+ fn is_delimited_identifier_start(&self, ch: char) -> bool {
+ ch == '"'
+ }
+ /// Determine if quoted characters are proper for identifier
+ fn is_proper_identifier_inside_quotes(&self, mut _chars: Peekable>) -> bool {
+ true
+ }
+ /// Determine if FILTER (WHERE ...) filters are allowed during aggregations
+ fn supports_filter_during_aggregation(&self) -> bool {
+ true
+ }
+}
diff --git a/dask_planner/src/expression.rs b/dask_planner/src/expression.rs
new file mode 100644
index 000000000..9c85b5324
--- /dev/null
+++ b/dask_planner/src/expression.rs
@@ -0,0 +1,844 @@
+use crate::sql::exceptions::{py_runtime_err, py_type_err};
+use crate::sql::logical;
+use crate::sql::types::RexType;
+use arrow::datatypes::DataType;
+use datafusion_common::{Column, DFField, DFSchema, Result, ScalarValue};
+use datafusion_expr::Operator;
+use datafusion_expr::{lit, utils::exprlist_to_fields, BuiltinScalarFunction, Expr, LogicalPlan};
+use pyo3::prelude::*;
+use std::convert::From;
+use std::sync::Arc;
+
+/// An PyExpr that can be used on a DataFrame
+#[pyclass(name = "Expression", module = "datafusion", subclass)]
+#[derive(Debug, Clone)]
+pub struct PyExpr {
+ pub expr: Expr,
+ // Why a Vec here? Because BinaryExpr on Join might have multiple LogicalPlans
+ pub input_plan: Option>>,
+}
+
+impl From for Expr {
+ fn from(expr: PyExpr) -> Expr {
+ expr.expr
+ }
+}
+
+#[pyclass(name = "ScalarValue", module = "datafusion", subclass)]
+#[derive(Debug, Clone)]
+pub struct PyScalarValue {
+ pub scalar_value: ScalarValue,
+}
+
+impl From for ScalarValue {
+ fn from(pyscalar: PyScalarValue) -> ScalarValue {
+ pyscalar.scalar_value
+ }
+}
+
+impl From for PyScalarValue {
+ fn from(scalar_value: ScalarValue) -> PyScalarValue {
+ PyScalarValue { scalar_value }
+ }
+}
+
+/// Convert a list of DataFusion Expr to PyExpr
+pub fn py_expr_list(input: &Arc, expr: &[Expr]) -> PyResult> {
+ Ok(expr
+ .iter()
+ .map(|e| PyExpr::from(e.clone(), Some(vec![input.clone()])))
+ .collect())
+}
+
+impl PyExpr {
+ /// Generally we would implement the `From` trait offered by Rust
+ /// However in this case Expr does not contain the contextual
+ /// `LogicalPlan` instance that we need so we need to make a instance
+ /// function to take and create the PyExpr.
+ pub fn from(expr: Expr, input: Option>>) -> PyExpr {
+ PyExpr {
+ input_plan: input,
+ expr,
+ }
+ }
+
+ /// Determines the name of the `Expr` instance by examining the LogicalPlan
+ pub fn _column_name(&self, plan: &LogicalPlan) -> Result {
+ let field = expr_to_field(&self.expr, plan)?;
+ Ok(field.qualified_column().flat_name())
+ }
+
+ fn _rex_type(&self, expr: &Expr) -> RexType {
+ match expr {
+ Expr::Alias(..)
+ | Expr::Column(..)
+ | Expr::QualifiedWildcard { .. }
+ | Expr::GetIndexedField { .. } => RexType::Reference,
+ Expr::ScalarVariable(..) | Expr::Literal(..) => RexType::Literal,
+ Expr::BinaryExpr { .. }
+ | Expr::Not(..)
+ | Expr::IsNotNull(..)
+ | Expr::Negative(..)
+ | Expr::IsNull(..)
+ | Expr::Like { .. }
+ | Expr::ILike { .. }
+ | Expr::SimilarTo { .. }
+ | Expr::Between { .. }
+ | Expr::Case { .. }
+ | Expr::Cast { .. }
+ | Expr::TryCast { .. }
+ | Expr::Sort { .. }
+ | Expr::ScalarFunction { .. }
+ | Expr::AggregateFunction { .. }
+ | Expr::WindowFunction { .. }
+ | Expr::AggregateUDF { .. }
+ | Expr::InList { .. }
+ | Expr::Wildcard
+ | Expr::ScalarUDF { .. }
+ | Expr::Exists { .. }
+ | Expr::InSubquery { .. }
+ | Expr::GroupingSet(..)
+ | Expr::IsTrue(..)
+ | Expr::IsFalse(..)
+ | Expr::IsUnknown(_)
+ | Expr::IsNotTrue(..)
+ | Expr::IsNotFalse(..)
+ | Expr::IsNotUnknown(_) => RexType::Call,
+ Expr::ScalarSubquery(..) => RexType::SubqueryAlias,
+ }
+ }
+}
+
+#[pymethods]
+impl PyExpr {
+ #[staticmethod]
+ pub fn literal(value: PyScalarValue) -> PyExpr {
+ PyExpr::from(lit(value.scalar_value), None)
+ }
+
+ /// Extracts the LogicalPlan from a Subquery, or supported Subquery sub-type, from
+ /// the expression instance
+ #[pyo3(name = "getSubqueryLogicalPlan")]
+ pub fn subquery_plan(&self) -> PyResult {
+ match &self.expr {
+ Expr::ScalarSubquery(subquery) => Ok(subquery.subquery.as_ref().clone().into()),
+ _ => Err(py_type_err(format!(
+ "Attempted to extract a LogicalPlan instance from invalid Expr {:?}.
+ Only Subquery and related variants are supported for this operation.",
+ &self.expr
+ ))),
+ }
+ }
+
+ /// If this Expression instances references an existing
+ /// Column in the SQL parse tree or not
+ #[pyo3(name = "isInputReference")]
+ pub fn is_input_reference(&self) -> PyResult {
+ Ok(matches!(&self.expr, Expr::Column(_col)))
+ }
+
+ #[pyo3(name = "toString")]
+ pub fn to_string(&self) -> PyResult {
+ Ok(format!("{}", &self.expr))
+ }
+
+ /// Gets the positional index of the Expr instance from the LogicalPlan DFSchema
+ #[pyo3(name = "getIndex")]
+ pub fn index(&self) -> PyResult {
+ let input: &Option>> = &self.input_plan;
+ match input {
+ Some(input_plans) if !input_plans.is_empty() => {
+ let mut schema: DFSchema = (**input_plans[0].schema()).clone();
+ for plan in input_plans.iter().skip(1) {
+ schema.merge(plan.schema().as_ref());
+ }
+ get_expr_name(&self.expr)
+ .and_then(|fq_name| {
+ schema.index_of_column(&Column::from_qualified_name(&fq_name))
+ })
+ .map_err(py_runtime_err)
+ }
+ _ => Err(py_runtime_err(
+ "We need a valid LogicalPlan instance to get the Expr's index in the schema",
+ )),
+ }
+ }
+
+ /// Examine the current/"self" PyExpr and return its "type"
+ /// In this context a "type" is what Dask-SQL Python
+ /// RexConverter plugin instance should be invoked to handle
+ /// the Rex conversion
+ #[pyo3(name = "getExprType")]
+ pub fn get_expr_type(&self) -> PyResult {
+ Ok(String::from(match &self.expr {
+ Expr::Alias(..)
+ | Expr::Column(..)
+ | Expr::Literal(..)
+ | Expr::BinaryExpr { .. }
+ | Expr::Between { .. }
+ | Expr::Cast { .. }
+ | Expr::Sort { .. }
+ | Expr::ScalarFunction { .. }
+ | Expr::AggregateFunction { .. }
+ | Expr::InList { .. }
+ | Expr::InSubquery { .. }
+ | Expr::ScalarUDF { .. }
+ | Expr::AggregateUDF { .. }
+ | Expr::Exists { .. }
+ | Expr::ScalarSubquery(..)
+ | Expr::QualifiedWildcard { .. }
+ | Expr::Not(..)
+ | Expr::GroupingSet(..) => self.expr.variant_name(),
+ Expr::ScalarVariable(..)
+ | Expr::IsNotNull(..)
+ | Expr::Negative(..)
+ | Expr::GetIndexedField { .. }
+ | Expr::IsNull(..)
+ | Expr::IsTrue(_)
+ | Expr::IsFalse(_)
+ | Expr::IsUnknown(_)
+ | Expr::IsNotTrue(_)
+ | Expr::IsNotFalse(_)
+ | Expr::Like { .. }
+ | Expr::ILike { .. }
+ | Expr::SimilarTo { .. }
+ | Expr::IsNotUnknown(_)
+ | Expr::Case { .. }
+ | Expr::TryCast { .. }
+ | Expr::WindowFunction { .. }
+ | Expr::Wildcard => {
+ return Err(py_type_err(format!(
+ "Encountered unsupported expression type: {}",
+ &self.expr.variant_name()
+ )))
+ }
+ }))
+ }
+
+ /// Determines the type of this Expr based on its variant
+ #[pyo3(name = "getRexType")]
+ pub fn rex_type(&self) -> PyResult {
+ Ok(self._rex_type(&self.expr))
+ }
+
+ /// Python friendly shim code to get the name of a column referenced by an expression
+ pub fn column_name(&self, mut plan: logical::PyLogicalPlan) -> PyResult {
+ self._column_name(&plan.current_node())
+ .map_err(py_runtime_err)
+ }
+
+ /// Row expressions, Rex(s), operate on the concept of operands. This maps to expressions that are used in
+ /// the "call" logic of the Dask-SQL python codebase. Different variants of Expressions, Expr(s),
+ /// store those operands in different datastructures. This function examines the Expr variant and returns
+ /// the operands to the calling logic as a Vec of PyExpr instances.
+ #[pyo3(name = "getOperands")]
+ pub fn get_operands(&self) -> PyResult> {
+ match &self.expr {
+ // Expr variants that are themselves the operand to return
+ Expr::Column(..) | Expr::ScalarVariable(..) | Expr::Literal(..) => {
+ Ok(vec![PyExpr::from(
+ self.expr.clone(),
+ self.input_plan.clone(),
+ )])
+ }
+
+ // Expr(s) that house the Expr instance to return in their bounded params
+ Expr::Alias(expr, ..)
+ | Expr::Not(expr)
+ | Expr::IsNull(expr)
+ | Expr::IsNotNull(expr)
+ | Expr::IsTrue(expr)
+ | Expr::IsFalse(expr)
+ | Expr::IsUnknown(expr)
+ | Expr::IsNotTrue(expr)
+ | Expr::IsNotFalse(expr)
+ | Expr::IsNotUnknown(expr)
+ | Expr::Negative(expr)
+ | Expr::GetIndexedField { expr, .. }
+ | Expr::Cast { expr, .. }
+ | Expr::TryCast { expr, .. }
+ | Expr::Sort { expr, .. }
+ | Expr::InSubquery { expr, .. } => {
+ Ok(vec![PyExpr::from(*expr.clone(), self.input_plan.clone())])
+ }
+
+ // Expr variants containing a collection of Expr(s) for operands
+ Expr::AggregateFunction { args, .. }
+ | Expr::AggregateUDF { args, .. }
+ | Expr::ScalarFunction { args, .. }
+ | Expr::ScalarUDF { args, .. }
+ | Expr::WindowFunction { args, .. } => Ok(args
+ .iter()
+ .map(|arg| PyExpr::from(arg.clone(), self.input_plan.clone()))
+ .collect()),
+
+ // Expr(s) that require more specific processing
+ Expr::Case {
+ expr,
+ when_then_expr,
+ else_expr,
+ } => {
+ let mut operands: Vec = Vec::new();
+
+ if let Some(e) = expr {
+ operands.push(PyExpr::from(*e.clone(), self.input_plan.clone()));
+ };
+
+ for (when, then) in when_then_expr {
+ operands.push(PyExpr::from(*when.clone(), self.input_plan.clone()));
+ operands.push(PyExpr::from(*then.clone(), self.input_plan.clone()));
+ }
+
+ if let Some(e) = else_expr {
+ operands.push(PyExpr::from(*e.clone(), self.input_plan.clone()));
+ };
+
+ Ok(operands)
+ }
+ Expr::InList { expr, list, .. } => {
+ let mut operands: Vec =
+ vec![PyExpr::from(*expr.clone(), self.input_plan.clone())];
+ for list_elem in list {
+ operands.push(PyExpr::from(list_elem.clone(), self.input_plan.clone()));
+ }
+
+ Ok(operands)
+ }
+ Expr::BinaryExpr { left, right, .. } => Ok(vec![
+ PyExpr::from(*left.clone(), self.input_plan.clone()),
+ PyExpr::from(*right.clone(), self.input_plan.clone()),
+ ]),
+ Expr::Like { expr, pattern, .. } => Ok(vec![
+ PyExpr::from(*expr.clone(), self.input_plan.clone()),
+ PyExpr::from(*pattern.clone(), self.input_plan.clone()),
+ ]),
+ Expr::ILike { expr, pattern, .. } => Ok(vec![
+ PyExpr::from(*expr.clone(), self.input_plan.clone()),
+ PyExpr::from(*pattern.clone(), self.input_plan.clone()),
+ ]),
+ Expr::SimilarTo { expr, pattern, .. } => Ok(vec![
+ PyExpr::from(*expr.clone(), self.input_plan.clone()),
+ PyExpr::from(*pattern.clone(), self.input_plan.clone()),
+ ]),
+ Expr::Between {
+ expr,
+ negated: _,
+ low,
+ high,
+ } => Ok(vec![
+ PyExpr::from(*expr.clone(), self.input_plan.clone()),
+ PyExpr::from(*low.clone(), self.input_plan.clone()),
+ PyExpr::from(*high.clone(), self.input_plan.clone()),
+ ]),
+
+ // Currently un-support/implemented Expr types for Rex Call operations
+ Expr::GroupingSet(..)
+ | Expr::Wildcard
+ | Expr::QualifiedWildcard { .. }
+ | Expr::ScalarSubquery(..)
+ | Expr::Exists { .. } => Err(py_runtime_err(format!(
+ "Unimplemented Expr type: {}",
+ self.expr
+ ))),
+ }
+ }
+
+ #[pyo3(name = "getOperatorName")]
+ pub fn get_operator_name(&self) -> PyResult {
+ Ok(match &self.expr {
+ Expr::BinaryExpr {
+ left: _,
+ op,
+ right: _,
+ } => format!("{}", op),
+ Expr::ScalarFunction { fun, args: _ } => format!("{}", fun),
+ Expr::ScalarUDF { fun, .. } => fun.name.clone(),
+ Expr::Cast { .. } => "cast".to_string(),
+ Expr::Between { .. } => "between".to_string(),
+ Expr::Case { .. } => "case".to_string(),
+ Expr::IsNull(..) => "is null".to_string(),
+ Expr::IsNotNull(..) => "is not null".to_string(),
+ Expr::IsTrue(_) => "is true".to_string(),
+ Expr::IsFalse(_) => "is false".to_string(),
+ Expr::IsUnknown(_) => "is unknown".to_string(),
+ Expr::IsNotTrue(_) => "is not true".to_string(),
+ Expr::IsNotFalse(_) => "is not false".to_string(),
+ Expr::IsNotUnknown(_) => "is not unknown".to_string(),
+ Expr::InList { .. } => "in list".to_string(),
+ Expr::Negative(..) => "negative".to_string(),
+ Expr::Not(..) => "not".to_string(),
+ Expr::Like { negated, .. } => {
+ if *negated {
+ "not like".to_string()
+ } else {
+ "like".to_string()
+ }
+ }
+ Expr::ILike { negated, .. } => {
+ if *negated {
+ "not ilike".to_string()
+ } else {
+ "ilike".to_string()
+ }
+ }
+ Expr::SimilarTo { negated, .. } => {
+ if *negated {
+ "not similar to".to_string()
+ } else {
+ "similar to".to_string()
+ }
+ }
+ _ => {
+ return Err(py_type_err(format!(
+ "Catch all triggered in get_operator_name: {:?}",
+ &self.expr
+ )))
+ }
+ })
+ }
+
+ /// Gets the ScalarValue represented by the Expression
+ #[pyo3(name = "getType")]
+ pub fn get_type(&self) -> PyResult {
+ Ok(String::from(match &self.expr {
+ Expr::BinaryExpr {
+ left: _,
+ op,
+ right: _,
+ } => match op {
+ Operator::Eq
+ | Operator::NotEq
+ | Operator::Lt
+ | Operator::LtEq
+ | Operator::Gt
+ | Operator::GtEq
+ | Operator::And
+ | Operator::Or
+ | Operator::Like
+ | Operator::NotLike
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
+ | Operator::RegexMatch
+ | Operator::RegexIMatch
+ | Operator::RegexNotMatch
+ | Operator::RegexNotIMatch => "BOOLEAN",
+ Operator::Plus | Operator::Minus | Operator::Multiply | Operator::Modulo => {
+ "BIGINT"
+ }
+ Operator::Divide => "FLOAT",
+ Operator::StringConcat => "VARCHAR",
+ Operator::BitwiseShiftLeft
+ | Operator::BitwiseShiftRight
+ | Operator::BitwiseXor
+ | Operator::BitwiseAnd
+ | Operator::BitwiseOr => {
+ // the type here should be the same as the type of the left expression
+ // but we can only compute that if we have the schema available
+ return Err(py_type_err(
+ "Bitwise operators unsupported in get_type".to_string(),
+ ));
+ }
+ },
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Boolean(_value) => "Boolean",
+ ScalarValue::Float32(_value) => "Float32",
+ ScalarValue::Float64(_value) => "Float64",
+ ScalarValue::Decimal128(_value, ..) => "Decimal128",
+ ScalarValue::Dictionary(..) => "Dictionary",
+ ScalarValue::Int8(_value) => "Int8",
+ ScalarValue::Int16(_value) => "Int16",
+ ScalarValue::Int32(_value) => "Int32",
+ ScalarValue::Int64(_value) => "Int64",
+ ScalarValue::UInt8(_value) => "UInt8",
+ ScalarValue::UInt16(_value) => "UInt16",
+ ScalarValue::UInt32(_value) => "UInt32",
+ ScalarValue::UInt64(_value) => "UInt64",
+ ScalarValue::Utf8(_value) => "Utf8",
+ ScalarValue::LargeUtf8(_value) => "LargeUtf8",
+ ScalarValue::Binary(_value) => "Binary",
+ ScalarValue::LargeBinary(_value) => "LargeBinary",
+ ScalarValue::Date32(_value) => "Date32",
+ ScalarValue::Date64(_value) => "Date64",
+ ScalarValue::Time64(_value) => "Time64",
+ ScalarValue::Null => "Null",
+ ScalarValue::TimestampSecond(..) => "TimestampSecond",
+ ScalarValue::TimestampMillisecond(..) => "TimestampMillisecond",
+ ScalarValue::TimestampMicrosecond(..) => "TimestampMicrosecond",
+ ScalarValue::TimestampNanosecond(..) => "TimestampNanosecond",
+ ScalarValue::IntervalYearMonth(..) => "IntervalYearMonth",
+ ScalarValue::IntervalDayTime(..) => "IntervalDayTime",
+ ScalarValue::IntervalMonthDayNano(..) => "IntervalMonthDayNano",
+ ScalarValue::List(..) => "List",
+ ScalarValue::Struct(..) => "Struct",
+ },
+ Expr::ScalarFunction { fun, args: _ } => match fun {
+ BuiltinScalarFunction::Abs => "Abs",
+ BuiltinScalarFunction::DatePart => "DatePart",
+ _ => {
+ return Err(py_type_err(format!(
+ "Catch all triggered for ScalarFunction in get_type; {:?}",
+ fun
+ )))
+ }
+ },
+ Expr::Cast { expr: _, data_type } => match data_type {
+ DataType::Null => "NULL",
+ DataType::Boolean => "BOOLEAN",
+ DataType::Int8 | DataType::UInt8 => "TINYINT",
+ DataType::Int16 | DataType::UInt16 => "SMALLINT",
+ DataType::Int32 | DataType::UInt32 => "INTEGER",
+ DataType::Int64 | DataType::UInt64 => "BIGINT",
+ DataType::Float32 => "FLOAT",
+ DataType::Float64 => "DOUBLE",
+ DataType::Timestamp { .. } => "TIMESTAMP",
+ DataType::Date32 | DataType::Date64 => "DATE",
+ DataType::Time32(..) => "TIME32",
+ DataType::Time64(..) => "TIME64",
+ DataType::Duration(..) => "DURATION",
+ DataType::Interval(..) => "INTERVAL",
+ DataType::Binary => "BINARY",
+ DataType::FixedSizeBinary(..) => "FIXEDSIZEBINARY",
+ DataType::LargeBinary => "LARGEBINARY",
+ DataType::Utf8 => "VARCHAR",
+ DataType::LargeUtf8 => "BIGVARCHAR",
+ DataType::List(..) => "LIST",
+ DataType::FixedSizeList(..) => "FIXEDSIZELIST",
+ DataType::LargeList(..) => "LARGELIST",
+ DataType::Struct(..) => "STRUCT",
+ DataType::Union(..) => "UNION",
+ DataType::Dictionary(..) => "DICTIONARY",
+ DataType::Decimal128(..) => "DECIMAL",
+ DataType::Decimal256(..) => "DECIMAL",
+ DataType::Map(..) => "MAP",
+ _ => {
+ return Err(py_type_err(format!(
+ "Catch all triggered for Cast in get_type; {:?}",
+ data_type
+ )))
+ }
+ },
+ _ => {
+ return Err(py_type_err(format!(
+ "Catch all triggered in get_type; {:?}",
+ &self.expr
+ )))
+ }
+ }))
+ }
+
+ #[pyo3(name = "getFilterExpr")]
+ pub fn get_filter_expr(&self) -> PyResult> {
+ // TODO refactor to avoid duplication
+ match &self.expr {
+ Expr::Alias(expr, _) => match expr.as_ref() {
+ Expr::AggregateFunction { filter, .. } | Expr::AggregateUDF { filter, .. } => {
+ match filter {
+ Some(filter) => {
+ Ok(Some(PyExpr::from(*filter.clone(), self.input_plan.clone())))
+ }
+ None => Ok(None),
+ }
+ }
+ _ => Err(py_type_err(
+ "getFilterExpr() - Non-aggregate expression encountered",
+ )),
+ },
+ Expr::AggregateFunction { filter, .. } | Expr::AggregateUDF { filter, .. } => {
+ match filter {
+ Some(filter) => {
+ Ok(Some(PyExpr::from(*filter.clone(), self.input_plan.clone())))
+ }
+ None => Ok(None),
+ }
+ }
+ _ => Err(py_type_err(
+ "getFilterExpr() - Non-aggregate expression encountered",
+ )),
+ }
+ }
+
+ /// TODO: I can't express how much I dislike explicity listing all of these methods out
+ /// but PyO3 makes it necessary since its annotations cannot be used in trait impl blocks
+ #[pyo3(name = "getFloat32Value")]
+ pub fn float_32_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Float32(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getFloat64Value")]
+ pub fn float_64_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Float64(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getDecimal128Value")]
+ pub fn decimal_128_value(&mut self) -> PyResult<(Option, u8, u8)> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Decimal128(value, precision, scale) => {
+ Ok((*value, *precision, *scale))
+ }
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getInt8Value")]
+ pub fn int_8_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Int8(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getInt16Value")]
+ pub fn int_16_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Int16(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getInt32Value")]
+ pub fn int_32_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Int32(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getInt64Value")]
+ pub fn int_64_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Int64(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getUInt8Value")]
+ pub fn uint_8_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::UInt8(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getUInt16Value")]
+ pub fn uint_16_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::UInt16(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getUInt32Value")]
+ pub fn uint_32_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::UInt32(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getUInt64Value")]
+ pub fn uint_64_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::UInt64(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getDate32Value")]
+ pub fn date_32_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Date32(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getDate64Value")]
+ pub fn date_64_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Date64(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getTime64Value")]
+ pub fn time_64_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Time64(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getTimestampValue")]
+ pub fn timestamp_value(&mut self) -> PyResult<(Option, Option)> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::TimestampNanosecond(iv, tz)
+ | ScalarValue::TimestampMicrosecond(iv, tz)
+ | ScalarValue::TimestampMillisecond(iv, tz)
+ | ScalarValue::TimestampSecond(iv, tz) => Ok((*iv, tz.clone())),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getBoolValue")]
+ pub fn bool_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Boolean(iv) => Ok(*iv),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getStringValue")]
+ pub fn string_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::Utf8(iv) => Ok(iv.clone()),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "getIntervalDayTimeValue")]
+ pub fn interval_day_time_value(&mut self) -> PyResult> {
+ match &self.expr {
+ Expr::Literal(scalar_value) => match scalar_value {
+ ScalarValue::IntervalDayTime(Some(iv)) => {
+ let interval = *iv as u64;
+ let days = (interval >> 32) as i32;
+ let ms = interval as i32;
+ Ok(Some((days, ms)))
+ }
+ ScalarValue::IntervalDayTime(None) => Ok(None),
+ _ => Err(py_type_err("getValue() - Unexpected value")),
+ },
+ _ => Err(py_type_err("getValue() - Non literal value encountered")),
+ }
+ }
+
+ #[pyo3(name = "isNegated")]
+ pub fn is_negated(&self) -> PyResult {
+ match &self.expr {
+ Expr::Between { negated, .. }
+ | Expr::Exists { negated, .. }
+ | Expr::InList { negated, .. }
+ | Expr::InSubquery { negated, .. } => Ok(*negated),
+ _ => Err(py_type_err(format!(
+ "unknown Expr type {:?} encountered",
+ &self.expr
+ ))),
+ }
+ }
+
+ /// Returns if a sort expressions is an ascending sort
+ #[pyo3(name = "isSortAscending")]
+ pub fn is_sort_ascending(&self) -> PyResult {
+ match &self.expr {
+ Expr::Sort { asc, .. } => Ok(*asc),
+ _ => Err(py_type_err(format!(
+ "Provided Expr {:?} is not a sort type",
+ &self.expr
+ ))),
+ }
+ }
+
+ /// Returns if nulls should be placed first in a sort expression
+ #[pyo3(name = "isSortNullsFirst")]
+ pub fn is_sort_nulls_first(&self) -> PyResult {
+ match &self.expr {
+ Expr::Sort { nulls_first, .. } => Ok(*nulls_first),
+ _ => Err(py_type_err(format!(
+ "Provided Expr {:?} is not a sort type",
+ &self.expr
+ ))),
+ }
+ }
+
+ /// Returns the escape char for like/ilike/similar to expr variants
+ #[pyo3(name = "getEscapeChar")]
+ pub fn get_escape_char(&self) -> PyResult> {
+ match &self.expr {
+ Expr::Like { escape_char, .. }
+ | Expr::ILike { escape_char, .. }
+ | Expr::SimilarTo { escape_char, .. } => Ok(*escape_char),
+ _ => Err(py_type_err(format!(
+ "Provided Expr {:?} not one of Like/ILike/SimilarTo",
+ &self.expr
+ ))),
+ }
+ }
+}
+
+fn get_expr_name(expr: &Expr) -> Result {
+ match expr {
+ Expr::Alias(expr, _) => get_expr_name(expr),
+ _ => expr.name(),
+ }
+}
+
+/// Create a [DFField] representing an [Expr], given an input [LogicalPlan] to resolve against
+pub fn expr_to_field(expr: &Expr, input_plan: &LogicalPlan) -> Result {
+ match expr {
+ Expr::Sort { expr, .. } => {
+ // DataFusion does not support create_name for sort expressions (since they never
+ // appear in projections) so we just delegate to the contained expression instead
+ expr_to_field(expr, input_plan)
+ }
+ _ => {
+ let fields = exprlist_to_fields(&[expr.clone()], input_plan)?;
+ Ok(fields[0].clone())
+ }
+ }
+}
diff --git a/dask_planner/src/lib.rs b/dask_planner/src/lib.rs
new file mode 100644
index 000000000..2011ff221
--- /dev/null
+++ b/dask_planner/src/lib.rs
@@ -0,0 +1,44 @@
+use mimalloc::MiMalloc;
+use pyo3::prelude::*;
+
+mod dialect;
+mod expression;
+mod parser;
+mod sql;
+
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;
+
+/// Low-level DataFusion internal package.
+///
+/// The higher-level public API is defined in pure python files under the
+/// dask_planner directory.
+#[pymodule]
+#[pyo3(name = "rust")]
+fn rust(py: Python, m: &PyModule) -> PyResult<()> {
+ // Register the python classes
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+ m.add_class::()?;
+
+ // Exceptions
+ m.add(
+ "DFParsingException",
+ py.get_type::(),
+ )?;
+ m.add(
+ "DFOptimizationException",
+ py.get_type::(),
+ )?;
+
+ Ok(())
+}
diff --git a/dask_planner/src/parser.rs b/dask_planner/src/parser.rs
new file mode 100644
index 000000000..26ea8b49e
--- /dev/null
+++ b/dask_planner/src/parser.rs
@@ -0,0 +1,959 @@
+//! SQL Parser
+//!
+//! Declares a SQL parser based on sqlparser that handles custom formats that we need.
+
+use crate::dialect::DaskDialect;
+use crate::sql::parser_utils::DaskParserUtils;
+use datafusion_sql::sqlparser::{
+ ast::{Expr, SelectItem, Statement as SQLStatement},
+ dialect::{keywords::Keyword, Dialect},
+ parser::{Parser, ParserError},
+ tokenizer::{Token, Tokenizer},
+};
+use std::collections::VecDeque;
+
+macro_rules! parser_err {
+ ($MSG:expr) => {
+ Err(ParserError::ParserError($MSG.to_string()))
+ };
+}
+
+/// Dask-SQL extension DDL for `CREATE MODEL`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreateModel {
+ /// model name
+ pub name: String,
+ /// input Query
+ pub select: DaskStatement,
+ /// IF NOT EXISTS
+ pub if_not_exists: bool,
+ /// To replace the model or not
+ pub or_replace: bool,
+ /// with options
+ pub with_options: Vec,
+}
+
+/// Dask-SQL extension DDL for `PREDICT`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PredictModel {
+ /// model schema
+ pub schema_name: String,
+ /// model name
+ pub name: String,
+ /// input Query
+ pub select: DaskStatement,
+}
+
+/// Dask-SQL extension DDL for `CREATE SCHEMA`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreateCatalogSchema {
+ /// schema_name
+ pub schema_name: String,
+ /// if not exists
+ pub if_not_exists: bool,
+ /// or replace
+ pub or_replace: bool,
+}
+
+/// Dask-SQL extension DDL for `CREATE TABLE ... WITH`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreateTable {
+ /// table schema, "something" in "something.table_name"
+ pub table_schema: String,
+ /// table name
+ pub name: String,
+ /// if not exists
+ pub if_not_exists: bool,
+ /// or replace
+ pub or_replace: bool,
+ /// with options
+ pub with_options: Vec,
+}
+
+/// Dask-SQL extension DDL for `CREATE VIEW`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CreateView {
+ /// view schema, "something" in "something.view_name"
+ pub view_schema: String,
+ /// view name
+ pub name: String,
+ /// if not exists
+ pub if_not_exists: bool,
+ /// or replace
+ pub or_replace: bool,
+}
+
+/// Dask-SQL extension DDL for `DROP MODEL`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DropModel {
+ /// model name
+ pub name: String,
+ /// if exists
+ pub if_exists: bool,
+}
+
+/// Dask-SQL extension DDL for `EXPORT MODEL`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ExportModel {
+ /// model name
+ pub name: String,
+ /// with options
+ pub with_options: Vec,
+}
+
+/// Dask-SQL extension DDL for `DESCRIBE MODEL`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DescribeModel {
+ /// model name
+ pub name: String,
+}
+
+/// Dask-SQL extension DDL for `SHOW SCHEMAS`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ShowSchemas {
+ /// like
+ pub like: Option,
+}
+
+/// Dask-SQL extension DDL for `SHOW TABLES FROM`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ShowTables {
+ /// schema name
+ pub schema_name: Option,
+}
+
+/// Dask-SQL extension DDL for `SHOW COLUMNS FROM`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ShowColumns {
+ /// Table name
+ pub table_name: String,
+ pub schema_name: Option,
+}
+
+/// Dask-SQL extension DDL for `SHOW MODELS`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ShowModels;
+
+/// Dask-SQL extension DDL for `USE SCHEMA`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DropSchema {
+ /// schema name
+ pub schema_name: String,
+ /// if exists
+ pub if_exists: bool,
+}
+
+/// Dask-SQL extension DDL for `USE SCHEMA`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct UseSchema {
+ /// schema name
+ pub schema_name: String,
+}
+
+/// Dask-SQL extension DDL for `ANALYZE TABLE`
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AnalyzeTable {
+ pub table_name: String,
+ pub schema_name: Option,
+ pub columns: Vec,
+}
+
+/// Dask-SQL Statement representations.
+///
+/// Tokens parsed by `DaskParser` are converted into these values.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum DaskStatement {
+ /// ANSI SQL AST node
+ Statement(Box),
+ /// Extension: `CREATE MODEL`
+ CreateModel(Box),
+ /// Extension: `CREATE SCHEMA`
+ CreateCatalogSchema(Box),
+ /// Extension: `CREATE TABLE`
+ CreateTable(Box),
+ /// Extension: `CREATE VIEW`
+ CreateView(Box),
+ /// Extension: `DROP MODEL`
+ DropModel(Box),
+ /// Extension: `EXPORT MODEL`
+ ExportModel(Box),
+ /// Extension: `DESCRIBE MODEL`
+ DescribeModel(Box),
+ /// Extension: `PREDICT`
+ PredictModel(Box),
+ // Extension: `SHOW SCHEMAS`
+ ShowSchemas(Box),
+ // Extension: `SHOW TABLES FROM`
+ ShowTables(Box),
+ // Extension: `SHOW COLUMNS FROM`
+ ShowColumns(Box),
+ // Extension: `SHOW COLUMNS FROM`
+ ShowModels(Box),
+ // Exntension: `DROP SCHEMA`
+ DropSchema(Box),
+ // Extension: `USE SCHEMA`
+ UseSchema(Box),
+ // Extension: `ANALYZE TABLE`
+ AnalyzeTable(Box),
+}
+
+/// SQL Parser
+pub struct DaskParser<'a> {
+ parser: Parser<'a>,
+}
+
+impl<'a> DaskParser<'a> {
+ #[allow(dead_code)]
+ /// Parse the specified tokens
+ pub fn new(sql: &str) -> Result {
+ let dialect = &DaskDialect {};
+ DaskParser::new_with_dialect(sql, dialect)
+ }
+
+ /// Parse the specified tokens with dialect
+ pub fn new_with_dialect(sql: &str, dialect: &'a dyn Dialect) -> Result {
+ let mut tokenizer = Tokenizer::new(dialect, sql);
+ let tokens = tokenizer.tokenize()?;
+
+ Ok(DaskParser {
+ parser: Parser::new(tokens, dialect),
+ })
+ }
+
+ #[allow(dead_code)]
+ /// Parse a SQL statement and produce a set of statements with dialect
+ pub fn parse_sql(sql: &str) -> Result, ParserError> {
+ let dialect = &DaskDialect {};
+ DaskParser::parse_sql_with_dialect(sql, dialect)
+ }
+
+ /// Parse a SQL statement and produce a set of statements
+ pub fn parse_sql_with_dialect(
+ sql: &str,
+ dialect: &dyn Dialect,
+ ) -> Result, ParserError> {
+ let mut parser = DaskParser::new_with_dialect(sql, dialect)?;
+ let mut stmts = VecDeque::new();
+ let mut expecting_statement_delimiter = false;
+ loop {
+ // ignore empty statements (between successive statement delimiters)
+ while parser.parser.consume_token(&Token::SemiColon) {
+ expecting_statement_delimiter = false;
+ }
+
+ if parser.parser.peek_token() == Token::EOF {
+ break;
+ }
+ if expecting_statement_delimiter {
+ return parser.expected("end of statement", parser.parser.peek_token());
+ }
+
+ let statement = parser.parse_statement()?;
+ stmts.push_back(statement);
+ expecting_statement_delimiter = true;
+ }
+ Ok(stmts)
+ }
+
+ /// Report unexpected token
+ fn expected(&self, expected: &str, found: Token) -> Result {
+ parser_err!(format!("Expected {}, found: {}", expected, found))
+ }
+
+ /// Parse a new expression
+ pub fn parse_statement(&mut self) -> Result {
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.keyword {
+ Keyword::CREATE => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_create()
+ }
+ Keyword::DROP => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_drop()
+ }
+ Keyword::SELECT => {
+ // Check for PREDICT token in statement
+ let mut cnt = 1;
+ loop {
+ match self.parser.next_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "predict" => {
+ return self.parse_predict_model();
+ }
+ _ => {
+ // Keep looking for PREDICT
+ cnt += 1;
+ continue;
+ }
+ }
+ }
+ Token::EOF => {
+ break;
+ }
+ _ => {
+ // Keep looking for PREDICT
+ cnt += 1;
+ continue;
+ }
+ }
+ }
+
+ // Reset the parser back to where we started
+ for _ in 0..cnt {
+ self.parser.prev_token();
+ }
+
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_statement()?,
+ )))
+ }
+ Keyword::SHOW => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_show()
+ }
+ Keyword::DESCRIBE => {
+ // move one token forwrd
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_describe()
+ }
+ Keyword::USE => {
+ // move one token forwrd
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_use()
+ }
+ Keyword::ANALYZE => {
+ // move one token foward
+ self.parser.next_token();
+ self.parse_analyze()
+ }
+ _ => {
+ match w.value.to_lowercase().as_str() {
+ "export" => {
+ // move one token forwrd
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_export_model()
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_statement()?,
+ )))
+ }
+ }
+ }
+ }
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_statement()?,
+ )))
+ }
+ }
+ }
+
+ /// Parse a SQL CREATE statement
+ pub fn parse_create(&mut self) -> Result {
+ let or_replace = self.parser.parse_keywords(&[Keyword::OR, Keyword::REPLACE]);
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "model" => {
+ // move one token forward
+ self.parser.next_token();
+
+ let if_not_exists = self.parser.parse_keywords(&[
+ Keyword::IF,
+ Keyword::NOT,
+ Keyword::EXISTS,
+ ]);
+
+ // use custom parsing
+ self.parse_create_model(if_not_exists, or_replace)
+ }
+ "schema" => {
+ // move one token forward
+ self.parser.next_token();
+
+ let if_not_exists = self.parser.parse_keywords(&[
+ Keyword::IF,
+ Keyword::NOT,
+ Keyword::EXISTS,
+ ]);
+
+ // use custom parsing
+ self.parse_create_schema(if_not_exists, or_replace)
+ }
+ "table" => {
+ // move one token forward
+ self.parser.next_token();
+
+ // use custom parsing
+ self.parse_create_table(true, or_replace)
+ }
+ "view" => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_create_table(false, or_replace)
+ }
+ _ => {
+ if or_replace {
+ // Go back two tokens if OR REPLACE was consumed
+ self.parser.prev_token();
+ self.parser.prev_token();
+ }
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_create()?,
+ )))
+ }
+ }
+ }
+ _ => {
+ if or_replace {
+ // Go back two tokens if OR REPLACE was consumed
+ self.parser.prev_token();
+ self.parser.prev_token();
+ }
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_create()?,
+ )))
+ }
+ }
+ }
+
+ /// Parse a SQL DROP statement
+ pub fn parse_drop(&mut self) -> Result {
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "model" => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_drop_model()
+ }
+ "schema" => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+
+ let if_exists = self.parser.parse_keywords(&[Keyword::IF, Keyword::EXISTS]);
+
+ let schema_name = self.parser.parse_identifier()?;
+
+ let drop_schema = DropSchema {
+ schema_name: schema_name.value,
+ if_exists,
+ };
+ Ok(DaskStatement::DropSchema(Box::new(drop_schema)))
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_drop()?,
+ )))
+ }
+ }
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_drop()?,
+ )))
+ }
+ }
+ }
+
+ /// Parse a SQL SHOW statement
+ pub fn parse_show(&mut self) -> Result {
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "schemas" => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_show_schemas()
+ }
+ "tables" => {
+ // move one token forward
+ self.parser.next_token();
+
+ // If non ansi ... `FROM {schema_name}` is present custom parse
+ // otherwise use sqlparser-rs
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "from" => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_show_tables()
+ }
+ _ => {
+ self.parser.prev_token();
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_show()?,
+ )))
+ }
+ }
+ }
+ _ => self.parse_show_tables(),
+ }
+ }
+ "columns" => {
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_show_columns()
+ }
+ "models" => {
+ self.parser.next_token();
+ Ok(DaskStatement::ShowModels(Box::new(ShowModels)))
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_show()?,
+ )))
+ }
+ }
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_show()?,
+ )))
+ }
+ }
+ }
+
+ /// Parse a SQL DESCRIBE statement
+ pub fn parse_describe(&mut self) -> Result {
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "model" => {
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_describe_model()
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_show()?,
+ )))
+ }
+ }
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_show()?,
+ )))
+ }
+ }
+ }
+
+ /// Parse a SQL USE SCHEMA statement
+ pub fn parse_use(&mut self) -> Result {
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "schema" => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ let schema_name = self.parser.parse_identifier()?;
+
+ let use_schema = UseSchema {
+ schema_name: schema_name.value,
+ };
+ Ok(DaskStatement::UseSchema(Box::new(use_schema)))
+ }
+ _ => Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_show()?,
+ ))),
+ }
+ }
+ _ => Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_show()?,
+ ))),
+ }
+ }
+
+ /// Parse a SQL ANALYZE statement
+ pub fn parse_analyze(&mut self) -> Result {
+ match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "table" => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ self.parse_analyze_table()
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_analyze()?,
+ )))
+ }
+ }
+ }
+ _ => {
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_analyze()?,
+ )))
+ }
+ }
+ }
+
+ /// Parse a SQL PREDICT statement
+ pub fn parse_predict_model(&mut self) -> Result {
+ // PREDICT(
+ // MODEL model_name,
+ // SQLStatement
+ // )
+ self.parser.expect_token(&Token::LParen)?;
+
+ let is_model = match self.parser.next_token() {
+ Token::Word(w) => matches!(w.value.to_lowercase().as_str(), "model"),
+ _ => false,
+ };
+ if !is_model {
+ return Err(ParserError::ParserError(
+ "parse_predict_model: Expected `MODEL`".to_string(),
+ ));
+ }
+
+ let (mdl_schema, mdl_name) =
+ DaskParserUtils::elements_from_tablefactor(&self.parser.parse_table_factor()?)?;
+ self.parser.expect_token(&Token::Comma)?;
+
+ // Limit our input to ANALYZE, DESCRIBE, SELECT, SHOW statements
+ // TODO: find a more sophisticated way to allow any statement that would return a table
+ self.parser.expect_one_of_keywords(&[
+ Keyword::SELECT,
+ Keyword::DESCRIBE,
+ Keyword::SHOW,
+ Keyword::ANALYZE,
+ ])?;
+ self.parser.prev_token();
+
+ let sql_statement = self.parse_statement()?;
+ self.parser.expect_token(&Token::RParen)?;
+
+ let predict = PredictModel {
+ schema_name: mdl_schema,
+ name: mdl_name,
+ select: sql_statement,
+ };
+ Ok(DaskStatement::PredictModel(Box::new(predict)))
+ }
+
+ /// Parse Dask-SQL CREATE MODEL statement
+ fn parse_create_model(
+ &mut self,
+ if_not_exists: bool,
+ or_replace: bool,
+ ) -> Result {
+ let model_name = self.parser.parse_object_name()?;
+ self.parser.expect_keyword(Keyword::WITH)?;
+
+ // `table_name` has been parsed at this point but is needed in `parse_table_factor`, reset consumption
+ self.parser.prev_token();
+ self.parser.prev_token();
+
+ let table_factor = self.parser.parse_table_factor()?;
+ let with_options = DaskParserUtils::options_from_tablefactor(&table_factor);
+
+ // Parse the nested query statement
+ self.parser.expect_keyword(Keyword::AS)?;
+ self.parser.expect_token(&Token::LParen)?;
+
+ // Limit our input to ANALYZE, DESCRIBE, SELECT, SHOW statements
+ // TODO: find a more sophisticated way to allow any statement that would return a table
+ self.parser.expect_one_of_keywords(&[
+ Keyword::SELECT,
+ Keyword::DESCRIBE,
+ Keyword::SHOW,
+ Keyword::ANALYZE,
+ ])?;
+ self.parser.prev_token();
+
+ let select = self.parse_statement()?;
+
+ self.parser.expect_token(&Token::RParen)?;
+
+ let create = CreateModel {
+ name: model_name.to_string(),
+ select,
+ if_not_exists,
+ or_replace,
+ with_options,
+ };
+ Ok(DaskStatement::CreateModel(Box::new(create)))
+ }
+
+ /// Parse Dask-SQL CREATE {IF NOT EXISTS | OR REPLACE} SCHEMA ... statement
+ fn parse_create_schema(
+ &mut self,
+ if_not_exists: bool,
+ or_replace: bool,
+ ) -> Result {
+ let schema_name = self.parser.parse_identifier()?.value;
+
+ let create = CreateCatalogSchema {
+ schema_name,
+ if_not_exists,
+ or_replace,
+ };
+ Ok(DaskStatement::CreateCatalogSchema(Box::new(create)))
+ }
+
+ /// Parse Dask-SQL CREATE [OR REPLACE] TABLE ... statement
+ ///
+ /// # Arguments
+ ///
+ /// * `is_table` - Whether the "table" is a "TABLE" or "VIEW", True if "TABLE" and False otherwise.
+ /// * `or_replace` - True if the "TABLE" or "VIEW" should be replaced and False otherwise
+ fn parse_create_table(
+ &mut self,
+ is_table: bool,
+ or_replace: bool,
+ ) -> Result {
+ // parse [IF NOT EXISTS] `table_name` AS|WITH
+ let if_not_exists =
+ self.parser
+ .parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
+
+ let _table_name = self.parser.parse_identifier();
+ let after_name_token = self.parser.peek_token();
+
+ match after_name_token {
+ Token::Word(w) => {
+ match w.value.to_lowercase().as_str() {
+ "as" => {
+ self.parser.prev_token();
+ if if_not_exists {
+ // Go back three tokens if IF NOT EXISTS was consumed, native parser consumes these tokens as well
+ self.parser.prev_token();
+ self.parser.prev_token();
+ self.parser.prev_token();
+ }
+
+ // True if TABLE and False if VIEW
+ if is_table {
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_create_table(or_replace, false, None)?,
+ )))
+ } else {
+ self.parser.prev_token();
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_create_view(or_replace)?,
+ )))
+ }
+ }
+ "with" => {
+ // `table_name` has been parsed at this point but is needed in `parse_table_factor`, reset consumption
+ self.parser.prev_token();
+
+ let table_factor = self.parser.parse_table_factor()?;
+ let (tbl_schema, tbl_name) =
+ DaskParserUtils::elements_from_tablefactor(&table_factor)?;
+ let with_options = DaskParserUtils::options_from_tablefactor(&table_factor);
+
+ let create = CreateTable {
+ table_schema: tbl_schema,
+ name: tbl_name,
+ if_not_exists,
+ or_replace,
+ with_options,
+ };
+ Ok(DaskStatement::CreateTable(Box::new(create)))
+ }
+ _ => self.expected("'as' or 'with'", self.parser.peek_token()),
+ }
+ }
+ _ => {
+ self.parser.prev_token();
+ if if_not_exists {
+ // Go back three tokens if IF NOT EXISTS was consumed
+ self.parser.prev_token();
+ self.parser.prev_token();
+ self.parser.prev_token();
+ }
+ // use the native parser
+ Ok(DaskStatement::Statement(Box::from(
+ self.parser.parse_create_table(or_replace, false, None)?,
+ )))
+ }
+ }
+ }
+
+ /// Parse Dask-SQL EXPORT MODEL statement
+ fn parse_export_model(&mut self) -> Result {
+ let is_model = match self.parser.next_token() {
+ Token::Word(w) => matches!(w.value.to_lowercase().as_str(), "model"),
+ _ => false,
+ };
+ if !is_model {
+ return Err(ParserError::ParserError(
+ "parse_export_model: Expected `MODEL`".to_string(),
+ ));
+ }
+
+ let model_name = self.parser.parse_object_name()?;
+ self.parser.expect_keyword(Keyword::WITH)?;
+
+ // `table_name` has been parsed at this point but is needed in `parse_table_factor`, reset consumption
+ self.parser.prev_token();
+ self.parser.prev_token();
+
+ let table_factor = self.parser.parse_table_factor()?;
+ let with_options = DaskParserUtils::options_from_tablefactor(&table_factor);
+
+ let export = ExportModel {
+ name: model_name.to_string(),
+ with_options,
+ };
+ Ok(DaskStatement::ExportModel(Box::new(export)))
+ }
+
+ /// Parse Dask-SQL DROP MODEL statement
+ fn parse_drop_model(&mut self) -> Result {
+ let if_exists = self.parser.parse_keywords(&[Keyword::IF, Keyword::EXISTS]);
+ let model_name = self.parser.parse_object_name()?;
+
+ let drop = DropModel {
+ name: model_name.to_string(),
+ if_exists,
+ };
+ Ok(DaskStatement::DropModel(Box::new(drop)))
+ }
+
+ /// Parse Dask-SQL DESRIBE MODEL statement
+ fn parse_describe_model(&mut self) -> Result {
+ let model_name = self.parser.parse_object_name()?;
+
+ let describe = DescribeModel {
+ name: model_name.to_string(),
+ };
+ Ok(DaskStatement::DescribeModel(Box::new(describe)))
+ }
+
+ /// Parse Dask-SQL SHOW SCHEMAS statement
+ fn parse_show_schemas(&mut self) -> Result {
+ // Check for existence of `LIKE` clause
+ let like_val = match self.parser.peek_token() {
+ Token::Word(w) => {
+ match w.keyword {
+ Keyword::LIKE => {
+ // move one token forward
+ self.parser.next_token();
+ // use custom parsing
+ Some(self.parser.parse_identifier()?.value)
+ }
+ _ => None,
+ }
+ }
+ _ => None,
+ };
+
+ Ok(DaskStatement::ShowSchemas(Box::new(ShowSchemas {
+ like: like_val,
+ })))
+ }
+
+ /// Parse Dask-SQL SHOW TABLES [FROM] statement
+ fn parse_show_tables(&mut self) -> Result {
+ let mut schema_name = None;
+ if !self.parser.consume_token(&Token::EOF) {
+ schema_name = Some(self.parser.parse_identifier()?.value);
+ }
+ Ok(DaskStatement::ShowTables(Box::new(ShowTables {
+ schema_name,
+ })))
+ }
+
+ /// Parse Dask-SQL SHOW COLUMNS FROM
+ fn parse_show_columns(&mut self) -> Result {
+ self.parser.expect_keyword(Keyword::FROM)?;
+ let table_factor = self.parser.parse_table_factor()?;
+ let (tbl_schema, tbl_name) = DaskParserUtils::elements_from_tablefactor(&table_factor)?;
+ Ok(DaskStatement::ShowColumns(Box::new(ShowColumns {
+ table_name: tbl_name,
+ schema_name: match tbl_schema.as_str() {
+ "" => None,
+ _ => Some(tbl_schema),
+ },
+ })))
+ }
+
+ /// Parse Dask-SQL ANALYZE TABLE
+ fn parse_analyze_table(&mut self) -> Result {
+ let table_factor = self.parser.parse_table_factor()?;
+ // parse_table_factor parses the following keyword as an alias, so we need to go back a token
+ // TODO: open an issue in sqlparser around this when possible
+ self.parser.prev_token();
+ self.parser
+ .expect_keywords(&[Keyword::COMPUTE, Keyword::STATISTICS, Keyword::FOR])?;
+ let (tbl_schema, tbl_name) = DaskParserUtils::elements_from_tablefactor(&table_factor)?;
+ let columns = match self
+ .parser
+ .parse_keywords(&[Keyword::ALL, Keyword::COLUMNS])
+ {
+ true => vec![],
+ false => {
+ self.parser.expect_keyword(Keyword::COLUMNS)?;
+ let mut values = vec![];
+ for select in self.parser.parse_projection()? {
+ match select {
+ SelectItem::UnnamedExpr(expr) => match expr {
+ Expr::Identifier(ident) => values.push(ident.value),
+ unexpected => {
+ return parser_err!(format!(
+ "Expected Identifier, found: {}",
+ unexpected
+ ))
+ }
+ },
+ unexpected => {
+ return parser_err!(format!(
+ "Expected UnnamedExpr, found: {}",
+ unexpected
+ ))
+ }
+ }
+ }
+ values
+ }
+ };
+ Ok(DaskStatement::AnalyzeTable(Box::new(AnalyzeTable {
+ table_name: tbl_name,
+ schema_name: match tbl_schema.as_str() {
+ "" => None,
+ _ => Some(tbl_schema),
+ },
+ columns,
+ })))
+ }
+}
diff --git a/dask_planner/src/sql.rs b/dask_planner/src/sql.rs
new file mode 100644
index 000000000..6ae956867
--- /dev/null
+++ b/dask_planner/src/sql.rs
@@ -0,0 +1,552 @@
+pub mod column;
+pub mod exceptions;
+pub mod function;
+pub mod logical;
+pub mod optimizer;
+pub mod parser_utils;
+pub mod schema;
+pub mod statement;
+pub mod table;
+pub mod types;
+
+use crate::sql::exceptions::{py_optimization_exp, py_parsing_exp, py_runtime_err};
+
+use arrow::datatypes::{DataType, Field, Schema};
+use datafusion_common::{DFSchema, DataFusionError};
+use datafusion_expr::logical_plan::Extension;
+use datafusion_expr::{
+ AccumulatorFunctionImplementation, AggregateUDF, LogicalPlan, PlanVisitor, ReturnTypeFunction,
+ ScalarFunctionImplementation, ScalarUDF, Signature, StateTypeFunction, TableSource,
+ TypeSignature, Volatility,
+};
+use datafusion_sql::{
+ parser::Statement as DFStatement,
+ planner::{ContextProvider, SqlToRel},
+ ResolvedTableReference, TableReference,
+};
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::dialect::DaskDialect;
+use crate::parser::{DaskParser, DaskStatement};
+use crate::sql::logical::analyze_table::AnalyzeTablePlanNode;
+use crate::sql::logical::create_model::CreateModelPlanNode;
+use crate::sql::logical::create_table::CreateTablePlanNode;
+use crate::sql::logical::create_view::CreateViewPlanNode;
+use crate::sql::logical::describe_model::DescribeModelPlanNode;
+use crate::sql::logical::drop_model::DropModelPlanNode;
+use crate::sql::logical::export_model::ExportModelPlanNode;
+use crate::sql::logical::predict_model::PredictModelPlanNode;
+use crate::sql::logical::show_columns::ShowColumnsPlanNode;
+use crate::sql::logical::show_models::ShowModelsPlanNode;
+use crate::sql::logical::show_schema::ShowSchemasPlanNode;
+use crate::sql::logical::show_tables::ShowTablesPlanNode;
+
+use crate::sql::logical::PyLogicalPlan;
+use pyo3::prelude::*;
+
+use self::logical::create_catalog_schema::CreateCatalogSchemaPlanNode;
+use self::logical::drop_schema::DropSchemaPlanNode;
+use self::logical::use_schema::UseSchemaPlanNode;
+
+/// DaskSQLContext is main interface used for interacting with DataFusion to
+/// parse SQL queries, build logical plans, and optimize logical plans.
+///
+/// The following example demonstrates how to generate an optimized LogicalPlan
+/// from SQL using DaskSQLContext.
+///
+/// ```
+/// use datafusion::prelude::*;
+///
+/// # use datafusion_common::Result;
+/// # #[tokio::main]
+/// # async fn main() -> Result<()> {
+/// let mut ctx = DaskSQLContext::new();
+/// let parsed_sql = ctx.parse_sql("SELECT COUNT(*) FROM test_table");
+/// let nonOptimizedRelAlgebra = ctx.logical_relational_algebra(parsed_sql);
+/// let optmizedRelAlg = ctx.optimizeRelationalAlgebra(nonOptimizedRelAlgebra);
+/// # Ok(())
+/// # }
+/// ```
+#[pyclass(name = "DaskSQLContext", module = "dask_planner", subclass)]
+#[derive(Debug, Clone)]
+pub struct DaskSQLContext {
+ current_catalog: String,
+ current_schema: String,
+ schemas: HashMap,
+}
+
+impl ContextProvider for DaskSQLContext {
+ fn get_table_provider(
+ &self,
+ name: TableReference,
+ ) -> Result, DataFusionError> {
+ let reference: ResolvedTableReference =
+ name.resolve(&self.current_catalog, &self.current_schema);
+ if reference.catalog != self.current_catalog {
+ // there is a single catalog in Dask SQL
+ return Err(DataFusionError::Plan(format!(
+ "Cannot resolve catalog '{}'",
+ reference.catalog
+ )));
+ }
+ match self.schemas.get(reference.schema) {
+ Some(schema) => {
+ let mut resp = None;
+ for table in schema.tables.values() {
+ if table.name.eq(&name.table()) {
+ // Build the Schema here
+ let mut fields: Vec = Vec::new();
+ // Iterate through the DaskTable instance and create a Schema instance
+ for (column_name, column_type) in &table.columns {
+ fields.push(Field::new(
+ column_name,
+ DataType::from(column_type.data_type()),
+ true,
+ ));
+ }
+
+ resp = Some(Schema::new(fields));
+ }
+ }
+
+ // If the Table is not found return None. DataFusion will handle the error propagation
+ match resp {
+ Some(e) => Ok(Arc::new(table::DaskTableSource::new(Arc::new(e)))),
+ None => Err(DataFusionError::Plan(format!(
+ "Table '{}.{}.{}' not found",
+ reference.catalog, reference.schema, reference.table
+ ))),
+ }
+ }
+ None => Err(DataFusionError::Plan(format!(
+ "Unable to locate Schema: '{}.{}'",
+ reference.catalog, reference.schema
+ ))),
+ }
+ }
+
+ fn get_function_meta(&self, name: &str) -> Option> {
+ let fun: ScalarFunctionImplementation =
+ Arc::new(|_| Err(DataFusionError::NotImplemented("".to_string())));
+
+ match name {
+ "year" => {
+ let sig = Signature::variadic(vec![DataType::Int64], Volatility::Immutable);
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Int64)));
+ return Some(Arc::new(ScalarUDF::new(name, &sig, &rtf, &fun)));
+ }
+ "atan2" | "mod" => {
+ let sig = Signature::variadic(
+ vec![DataType::Float64, DataType::Float64],
+ Volatility::Immutable,
+ );
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Float64)));
+ return Some(Arc::new(ScalarUDF::new(name, &sig, &rtf, &fun)));
+ }
+ "cbrt" | "cot" | "degrees" | "radians" | "sign" | "truncate" => {
+ let sig = Signature::variadic(vec![DataType::Float64], Volatility::Immutable);
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Float64)));
+ return Some(Arc::new(ScalarUDF::new(name, &sig, &rtf, &fun)));
+ }
+ "rand" => {
+ let sig = Signature::variadic(vec![DataType::Int64], Volatility::Volatile);
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Float64)));
+ return Some(Arc::new(ScalarUDF::new(name, &sig, &rtf, &fun)));
+ }
+ "rand_integer" => {
+ let sig = Signature::variadic(
+ vec![DataType::Int64, DataType::Int64],
+ Volatility::Volatile,
+ );
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Int64)));
+ return Some(Arc::new(ScalarUDF::new(name, &sig, &rtf, &fun)));
+ }
+ _ => (),
+ }
+
+ // Loop through all of the user defined functions
+ for schema in self.schemas.values() {
+ for (fun_name, func_mutex) in &schema.functions {
+ if fun_name.eq(name) {
+ let function = func_mutex.lock().unwrap();
+ if function.aggregation.eq(&true) {
+ return None;
+ }
+ let sig = {
+ Signature::one_of(
+ function
+ .return_types
+ .keys()
+ .map(|v| TypeSignature::Exact(v.to_vec()))
+ .collect(),
+ Volatility::Immutable,
+ )
+ };
+ let function = function.clone();
+ let rtf: ReturnTypeFunction = Arc::new(move |input_types| {
+ match function.return_types.get(&input_types.to_vec()) {
+ Some(return_type) => Ok(Arc::new(return_type.clone())),
+ None => Err(DataFusionError::Plan(format!(
+ "UDF signature not found for input types {:?}",
+ input_types
+ ))),
+ }
+ });
+ return Some(Arc::new(ScalarUDF::new(
+ fun_name.as_str(),
+ &sig,
+ &rtf,
+ &fun,
+ )));
+ }
+ }
+ }
+
+ None
+ }
+
+ fn get_aggregate_meta(&self, name: &str) -> Option> {
+ let acc: AccumulatorFunctionImplementation =
+ Arc::new(|_return_type| Err(DataFusionError::NotImplemented("".to_string())));
+
+ let st: StateTypeFunction =
+ Arc::new(|_| Err(DataFusionError::NotImplemented("".to_string())));
+
+ match name {
+ "every" => {
+ let sig = Signature::variadic(vec![DataType::Int64], Volatility::Immutable);
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Boolean)));
+ return Some(Arc::new(AggregateUDF::new(name, &sig, &rtf, &acc, &st)));
+ }
+ "bit_and" | "bit_or" => {
+ let sig = Signature::variadic(vec![DataType::Int64], Volatility::Immutable);
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Int64)));
+ return Some(Arc::new(AggregateUDF::new(name, &sig, &rtf, &acc, &st)));
+ }
+ "single_value" => {
+ let sig = Signature::variadic(vec![DataType::Int64], Volatility::Immutable);
+ let rtf: ReturnTypeFunction =
+ Arc::new(|input_types| Ok(Arc::new(input_types[0].clone())));
+ return Some(Arc::new(AggregateUDF::new(name, &sig, &rtf, &acc, &st)));
+ }
+ "regr_count" => {
+ let sig = Signature::variadic(
+ vec![DataType::Float64, DataType::Float64],
+ Volatility::Immutable,
+ );
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Int64)));
+ return Some(Arc::new(AggregateUDF::new(name, &sig, &rtf, &acc, &st)));
+ }
+ "regr_syy" | "regr_sxx" => {
+ let sig = Signature::variadic(
+ vec![DataType::Float64, DataType::Float64],
+ Volatility::Immutable,
+ );
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Float64)));
+ return Some(Arc::new(AggregateUDF::new(name, &sig, &rtf, &acc, &st)));
+ }
+ "var_pop" => {
+ let sig = Signature::variadic(vec![DataType::Float64], Volatility::Immutable);
+ let rtf: ReturnTypeFunction = Arc::new(|_| Ok(Arc::new(DataType::Float64)));
+ return Some(Arc::new(AggregateUDF::new(name, &sig, &rtf, &acc, &st)));
+ }
+ _ => (),
+ }
+
+ // Loop through all of the user defined functions
+ for schema in self.schemas.values() {
+ for (fun_name, func_mutex) in &schema.functions {
+ if fun_name.eq(name) {
+ let function = func_mutex.lock().unwrap();
+ if function.aggregation.eq(&false) {
+ return None;
+ }
+ let sig = {
+ Signature::one_of(
+ function
+ .return_types
+ .keys()
+ .map(|v| TypeSignature::Exact(v.to_vec()))
+ .collect(),
+ Volatility::Immutable,
+ )
+ };
+ let function = function.clone();
+ let rtf: ReturnTypeFunction = Arc::new(move |input_types| {
+ match function.return_types.get(&input_types.to_vec()) {
+ Some(return_type) => Ok(Arc::new(return_type.clone())),
+ None => Err(DataFusionError::Plan(format!(
+ "UDAF signature not found for input types {:?}",
+ input_types
+ ))),
+ }
+ });
+ return Some(Arc::new(AggregateUDF::new(fun_name, &sig, &rtf, &acc, &st)));
+ }
+ }
+ }
+
+ None
+ }
+
+ fn get_variable_type(&self, _: &[String]) -> Option {
+ unimplemented!("RUST: get_variable_type is not yet implemented for DaskSQLContext")
+ }
+}
+
+#[pymethods]
+impl DaskSQLContext {
+ #[new]
+ pub fn new(default_catalog_name: &str, default_schema_name: &str) -> Self {
+ Self {
+ current_catalog: default_catalog_name.to_owned(),
+ current_schema: default_schema_name.to_owned(),
+ schemas: HashMap::new(),
+ }
+ }
+
+ /// Change the current schema
+ pub fn use_schema(&mut self, schema_name: &str) -> PyResult<()> {
+ if self.schemas.contains_key(schema_name) {
+ self.current_schema = schema_name.to_owned();
+ Ok(())
+ } else {
+ Err(py_runtime_err(format!(
+ "Schema: {} not found in DaskSQLContext",
+ schema_name
+ )))
+ }
+ }
+
+ /// Register a Schema with the current DaskSQLContext
+ pub fn register_schema(
+ &mut self,
+ schema_name: String,
+ schema: schema::DaskSchema,
+ ) -> PyResult {
+ self.schemas.insert(schema_name, schema);
+ Ok(true)
+ }
+
+ /// Register a DaskTable instance under the specified schema in the current DaskSQLContext
+ pub fn register_table(
+ &mut self,
+ schema_name: String,
+ table: table::DaskTable,
+ ) -> PyResult {
+ match self.schemas.get_mut(&schema_name) {
+ Some(schema) => {
+ schema.add_table(table);
+ Ok(true)
+ }
+ None => Err(py_runtime_err(format!(
+ "Schema: {} not found in DaskSQLContext",
+ schema_name
+ ))),
+ }
+ }
+
+ /// Parses a SQL string into an AST presented as a Vec of Statements
+ pub fn parse_sql(&self, sql: &str) -> PyResult> {
+ let dd: DaskDialect = DaskDialect {};
+ match DaskParser::parse_sql_with_dialect(sql, &dd) {
+ Ok(k) => {
+ let mut statements: Vec = Vec::new();
+ for statement in k {
+ statements.push(statement.into());
+ }
+ Ok(statements)
+ }
+ Err(e) => Err(py_parsing_exp(e)),
+ }
+ }
+
+ /// Creates a non-optimized Relational Algebra LogicalPlan from an AST Statement
+ pub fn logical_relational_algebra(
+ &self,
+ statement: statement::PyStatement,
+ ) -> PyResult {
+ self._logical_relational_algebra(statement.statement)
+ .map(|e| PyLogicalPlan {
+ original_plan: e,
+ current_node: None,
+ })
+ .map_err(py_parsing_exp)
+ }
+
+ /// Accepts an existing relational plan, `LogicalPlan`, and optimizes it
+ /// by applying a set of `optimizer` trait implementations against the
+ /// `LogicalPlan`
+ pub fn optimize_relational_algebra(
+ &self,
+ existing_plan: logical::PyLogicalPlan,
+ ) -> PyResult {
+ // Certain queries cannot be optimized. Ex: `EXPLAIN SELECT * FROM test` simply return those plans as is
+ let mut visitor = OptimizablePlanVisitor {};
+
+ match existing_plan.original_plan.accept(&mut visitor) {
+ Ok(valid) => {
+ if valid {
+ optimizer::DaskSqlOptimizer::new()
+ .run_optimizations(existing_plan.original_plan)
+ .map(|k| PyLogicalPlan {
+ original_plan: k,
+ current_node: None,
+ })
+ .map_err(py_optimization_exp)
+ } else {
+ // This LogicalPlan does not support Optimization. Return original
+ Ok(existing_plan)
+ }
+ }
+ Err(e) => Err(py_optimization_exp(e)),
+ }
+ }
+}
+
+/// non-Python methods
+impl DaskSQLContext {
+ /// Creates a non-optimized Relational Algebra LogicalPlan from an AST Statement
+ pub fn _logical_relational_algebra(
+ &self,
+ dask_statement: DaskStatement,
+ ) -> Result {
+ match dask_statement {
+ DaskStatement::Statement(statement) => {
+ let planner = SqlToRel::new(self);
+ planner.statement_to_plan(DFStatement::Statement(statement))
+ }
+ DaskStatement::CreateModel(create_model) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(CreateModelPlanNode {
+ model_name: create_model.name,
+ input: self._logical_relational_algebra(create_model.select)?,
+ if_not_exists: create_model.if_not_exists,
+ or_replace: create_model.or_replace,
+ with_options: create_model.with_options,
+ }),
+ })),
+ DaskStatement::PredictModel(predict_model) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(PredictModelPlanNode {
+ model_schema: predict_model.schema_name,
+ model_name: predict_model.name,
+ input: self._logical_relational_algebra(predict_model.select)?,
+ }),
+ })),
+ DaskStatement::DescribeModel(describe_model) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(DescribeModelPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ model_name: describe_model.name,
+ }),
+ })),
+ DaskStatement::CreateCatalogSchema(create_schema) => {
+ Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(CreateCatalogSchemaPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ schema_name: create_schema.schema_name,
+ if_not_exists: create_schema.if_not_exists,
+ or_replace: create_schema.or_replace,
+ }),
+ }))
+ }
+ DaskStatement::CreateTable(create_table) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(CreateTablePlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ table_schema: create_table.table_schema,
+ table_name: create_table.name,
+ if_not_exists: create_table.if_not_exists,
+ or_replace: create_table.or_replace,
+ with_options: create_table.with_options,
+ }),
+ })),
+ DaskStatement::ExportModel(export_model) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(ExportModelPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ model_name: export_model.name,
+ with_options: export_model.with_options,
+ }),
+ })),
+ DaskStatement::CreateView(create_view) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(CreateViewPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ view_schema: create_view.view_schema,
+ view_name: create_view.name,
+ if_not_exists: create_view.if_not_exists,
+ or_replace: create_view.or_replace,
+ }),
+ })),
+ DaskStatement::DropModel(drop_model) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(DropModelPlanNode {
+ model_name: drop_model.name,
+ if_exists: drop_model.if_exists,
+ schema: Arc::new(DFSchema::empty()),
+ }),
+ })),
+ DaskStatement::ShowSchemas(show_schemas) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(ShowSchemasPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ like: show_schemas.like,
+ }),
+ })),
+ DaskStatement::ShowTables(show_tables) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(ShowTablesPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ schema_name: show_tables.schema_name,
+ }),
+ })),
+ DaskStatement::ShowColumns(show_columns) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(ShowColumnsPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ table_name: show_columns.table_name,
+ schema_name: show_columns.schema_name,
+ }),
+ })),
+ DaskStatement::ShowModels(_show_models) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(ShowModelsPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ }),
+ })),
+ DaskStatement::DropSchema(drop_schema) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(DropSchemaPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ schema_name: drop_schema.schema_name,
+ if_exists: drop_schema.if_exists,
+ }),
+ })),
+ DaskStatement::UseSchema(use_schema) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(UseSchemaPlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ schema_name: use_schema.schema_name,
+ }),
+ })),
+ DaskStatement::AnalyzeTable(analyze_table) => Ok(LogicalPlan::Extension(Extension {
+ node: Arc::new(AnalyzeTablePlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ table_name: analyze_table.table_name,
+ schema_name: analyze_table.schema_name,
+ columns: analyze_table.columns,
+ }),
+ })),
+ }
+ }
+}
+
+/// Visits each AST node to determine if the plan is valid for optimization or not
+pub struct OptimizablePlanVisitor;
+
+impl PlanVisitor for OptimizablePlanVisitor {
+ type Error = DataFusionError;
+
+ fn pre_visit(&mut self, plan: &LogicalPlan) -> std::result::Result {
+ // If the plan contains an unsupported Node type we flag the plan as un-optimizable here
+ match plan {
+ LogicalPlan::Explain(..) => Ok(false),
+ _ => Ok(true),
+ }
+ }
+
+ fn post_visit(&mut self, _plan: &LogicalPlan) -> std::result::Result {
+ Ok(true)
+ }
+}
diff --git a/dask_planner/src/sql/column.rs b/dask_planner/src/sql/column.rs
new file mode 100644
index 000000000..a3a9b8954
--- /dev/null
+++ b/dask_planner/src/sql/column.rs
@@ -0,0 +1,35 @@
+use datafusion_common::Column;
+
+use pyo3::prelude::*;
+
+#[pyclass(name = "Column", module = "dask_planner", subclass)]
+#[derive(Debug, Clone)]
+pub struct PyColumn {
+ /// Original Column instance
+ pub(crate) column: Column,
+}
+
+impl From for Column {
+ fn from(column: PyColumn) -> Column {
+ column.column
+ }
+}
+
+impl From for PyColumn {
+ fn from(column: Column) -> PyColumn {
+ PyColumn { column }
+ }
+}
+
+#[pymethods]
+impl PyColumn {
+ #[pyo3(name = "getRelation")]
+ pub fn relation(&self) -> String {
+ self.column.relation.clone().unwrap()
+ }
+
+ #[pyo3(name = "getName")]
+ pub fn name(&self) -> String {
+ self.column.name.clone()
+ }
+}
diff --git a/dask_planner/src/sql/exceptions.rs b/dask_planner/src/sql/exceptions.rs
new file mode 100644
index 000000000..5627bbb3f
--- /dev/null
+++ b/dask_planner/src/sql/exceptions.rs
@@ -0,0 +1,24 @@
+use pyo3::{create_exception, PyErr};
+use std::fmt::Debug;
+
+// Identifies expections that occur while attempting to generate a `LogicalPlan` from a SQL string
+create_exception!(rust, ParsingException, pyo3::exceptions::PyException);
+
+// Identifies exceptions that occur during attempts to optimization an existing `LogicalPlan`
+create_exception!(rust, OptimizationException, pyo3::exceptions::PyException);
+
+pub fn py_type_err(e: impl Debug) -> PyErr {
+ PyErr::new::(format!("{:?}", e))
+}
+
+pub fn py_runtime_err(e: impl Debug) -> PyErr {
+ PyErr::new::(format!("{:?}", e))
+}
+
+pub fn py_parsing_exp(e: impl Debug) -> PyErr {
+ PyErr::new::(format!("{:?}", e))
+}
+
+pub fn py_optimization_exp(e: impl Debug) -> PyErr {
+ PyErr::new::(format!("{:?}", e))
+}
diff --git a/dask_planner/src/sql/function.rs b/dask_planner/src/sql/function.rs
new file mode 100644
index 000000000..ca41220aa
--- /dev/null
+++ b/dask_planner/src/sql/function.rs
@@ -0,0 +1,38 @@
+use super::types::PyDataType;
+use pyo3::prelude::*;
+
+use arrow::datatypes::DataType;
+use std::collections::HashMap;
+
+#[pyclass(name = "DaskFunction", module = "dask_planner", subclass)]
+#[derive(Debug, Clone)]
+pub struct DaskFunction {
+ #[pyo3(get, set)]
+ pub(crate) name: String,
+ pub(crate) return_types: HashMap, DataType>,
+ pub(crate) aggregation: bool,
+}
+
+impl DaskFunction {
+ pub fn new(
+ function_name: String,
+ input_types: Vec,
+ return_type: PyDataType,
+ aggregation_bool: bool,
+ ) -> Self {
+ let mut func = Self {
+ name: function_name,
+ return_types: HashMap::new(),
+ aggregation: aggregation_bool,
+ };
+ func.add_type_mapping(input_types, return_type);
+ func
+ }
+
+ pub fn add_type_mapping(&mut self, input_types: Vec, return_type: PyDataType) {
+ self.return_types.insert(
+ input_types.iter().map(|t| t.clone().into()).collect(),
+ return_type.into(),
+ );
+ }
+}
diff --git a/dask_planner/src/sql/logical.rs b/dask_planner/src/sql/logical.rs
new file mode 100644
index 000000000..1a7dc865b
--- /dev/null
+++ b/dask_planner/src/sql/logical.rs
@@ -0,0 +1,406 @@
+use crate::sql::table;
+use crate::sql::types::rel_data_type::RelDataType;
+use crate::sql::types::rel_data_type_field::RelDataTypeField;
+
+pub mod aggregate;
+pub mod analyze_table;
+pub mod create_catalog_schema;
+pub mod create_memory_table;
+pub mod create_model;
+pub mod create_table;
+pub mod create_view;
+pub mod describe_model;
+pub mod drop_model;
+pub mod drop_schema;
+pub mod drop_table;
+pub mod empty_relation;
+pub mod explain;
+pub mod export_model;
+pub mod filter;
+pub mod join;
+pub mod limit;
+pub mod predict_model;
+pub mod projection;
+pub mod repartition_by;
+pub mod show_columns;
+pub mod show_models;
+pub mod show_schema;
+pub mod show_tables;
+pub mod sort;
+pub mod table_scan;
+pub mod use_schema;
+pub mod window;
+
+use datafusion_common::{DFSchemaRef, DataFusionError, Result};
+use datafusion_expr::LogicalPlan;
+
+use crate::sql::exceptions::py_type_err;
+use pyo3::prelude::*;
+
+use self::analyze_table::AnalyzeTablePlanNode;
+use self::create_catalog_schema::CreateCatalogSchemaPlanNode;
+use self::create_model::CreateModelPlanNode;
+use self::create_table::CreateTablePlanNode;
+use self::create_view::CreateViewPlanNode;
+use self::describe_model::DescribeModelPlanNode;
+use self::drop_model::DropModelPlanNode;
+use self::drop_schema::DropSchemaPlanNode;
+use self::export_model::ExportModelPlanNode;
+use self::predict_model::PredictModelPlanNode;
+use self::show_columns::ShowColumnsPlanNode;
+use self::show_models::ShowModelsPlanNode;
+use self::show_schema::ShowSchemasPlanNode;
+use self::show_tables::ShowTablesPlanNode;
+use self::use_schema::UseSchemaPlanNode;
+
+#[pyclass(name = "LogicalPlan", module = "dask_planner", subclass)]
+#[derive(Debug, Clone)]
+pub struct PyLogicalPlan {
+ /// The original LogicalPlan that was parsed by DataFusion from the input SQL
+ pub(crate) original_plan: LogicalPlan,
+ /// The original_plan is traversed. current_node stores the current node of this traversal
+ pub(crate) current_node: Option,
+}
+
+/// Unfortunately PyO3 forces us to do this as placing these methods in the #[pymethods] version
+/// of `impl PyLogicalPlan` causes issues with types not properly being mapped to Python from Rust
+impl PyLogicalPlan {
+ /// Getter method for the LogicalPlan, if current_node is None return original_plan.
+ pub(crate) fn current_node(&mut self) -> LogicalPlan {
+ match &self.current_node {
+ Some(current) => current.clone(),
+ None => {
+ self.current_node = Some(self.original_plan.clone());
+ self.current_node.clone().unwrap()
+ }
+ }
+ }
+}
+
+/// Convert a LogicalPlan to a Python equivalent type
+fn to_py_plan>(
+ current_node: Option<&LogicalPlan>,
+) -> PyResult {
+ match current_node {
+ Some(plan) => plan.clone().try_into(),
+ _ => Err(py_type_err("current_node was None")),
+ }
+}
+
+#[pymethods]
+impl PyLogicalPlan {
+ /// LogicalPlan::Aggregate as PyAggregate
+ pub fn aggregate(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::EmptyRelation as PyEmptyRelation
+ pub fn empty_relation(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Explain as PyExplain
+ pub fn explain(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Filter as PyFilter
+ pub fn filter(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Join as PyJoin
+ pub fn join(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Limit as PyLimit
+ pub fn limit(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Projection as PyProjection
+ pub fn projection(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Sort as PySort
+ pub fn sort(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Window as PyWindow
+ pub fn window(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::TableScan as PyTableScan
+ pub fn table_scan(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::CreateMemoryTable as PyCreateMemoryTable
+ pub fn create_memory_table(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::CreateModel as PyCreateModel
+ pub fn create_model(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::DropTable as DropTable
+ pub fn drop_table(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::DropModel as DropModel
+ pub fn drop_model(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::ShowSchemas as PyShowSchemas
+ pub fn show_schemas(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Repartition as PyRepartitionBy
+ pub fn repartition_by(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::ShowTables as PyShowTables
+ pub fn show_tables(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::CreateTable as PyCreateTable
+ pub fn create_table(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::PredictModel as PyPredictModel
+ pub fn predict_model(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::DescribeModel as PyDescribeModel
+ pub fn describe_model(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::ExportModel as PyExportModel
+ pub fn export_model(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::ShowColumns as PyShowColumns
+ pub fn show_columns(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+ /// LogicalPlan::Extension::ShowColumns as PyShowColumns
+ pub fn analyze_table(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::CreateCatalogSchema as PyCreateCatalogSchema
+ pub fn create_catalog_schema(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::DropSchema as PyDropSchema
+ pub fn drop_schema(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// LogicalPlan::Extension::UseSchema as PyUseSchema
+ pub fn use_schema(&self) -> PyResult {
+ to_py_plan(self.current_node.as_ref())
+ }
+
+ /// Gets the "input" for the current LogicalPlan
+ pub fn get_inputs(&mut self) -> PyResult> {
+ let mut py_inputs: Vec = Vec::new();
+ for input in self.current_node().inputs() {
+ py_inputs.push(input.clone().into());
+ }
+ Ok(py_inputs)
+ }
+
+ /// If the LogicalPlan represents access to a Table that instance is returned
+ /// otherwise None is returned
+ #[pyo3(name = "getTable")]
+ pub fn table(&mut self) -> PyResult {
+ match table::table_from_logical_plan(&self.current_node()) {
+ Some(table) => Ok(table),
+ None => Err(py_type_err(
+ "Unable to compute DaskTable from DataFusion LogicalPlan",
+ )),
+ }
+ }
+
+ #[pyo3(name = "getCurrentNodeSchemaName")]
+ pub fn get_current_node_schema_name(&self) -> PyResult<&str> {
+ match &self.current_node {
+ Some(e) => {
+ let _sch: &DFSchemaRef = e.schema();
+ //TODO: Where can I actually get this in the context of the running query?
+ Ok("root")
+ }
+ None => Err(py_type_err(DataFusionError::Plan(format!(
+ "Current schema not found. Defaulting to {:?}",
+ "root"
+ )))),
+ }
+ }
+
+ #[pyo3(name = "getCurrentNodeTableName")]
+ pub fn get_current_node_table_name(&mut self) -> PyResult {
+ match self.table() {
+ Ok(dask_table) => Ok(dask_table.name),
+ Err(_e) => Err(py_type_err("Unable to determine current node table name")),
+ }
+ }
+
+ /// Gets the Relation "type" of the current node. Ex: Projection, TableScan, etc
+ pub fn get_current_node_type(&mut self) -> PyResult<&str> {
+ Ok(match self.current_node() {
+ LogicalPlan::Distinct(_) => "Distinct",
+ LogicalPlan::Projection(_projection) => "Projection",
+ LogicalPlan::Filter(_filter) => "Filter",
+ LogicalPlan::Window(_window) => "Window",
+ LogicalPlan::Aggregate(_aggregate) => "Aggregate",
+ LogicalPlan::Sort(_sort) => "Sort",
+ LogicalPlan::Join(_join) => "Join",
+ LogicalPlan::CrossJoin(_cross_join) => "CrossJoin",
+ LogicalPlan::Repartition(_repartition) => "Repartition",
+ LogicalPlan::Union(_union) => "Union",
+ LogicalPlan::TableScan(_table_scan) => "TableScan",
+ LogicalPlan::EmptyRelation(_empty_relation) => "EmptyRelation",
+ LogicalPlan::Limit(_limit) => "Limit",
+ LogicalPlan::CreateExternalTable(_create_external_table) => "CreateExternalTable",
+ LogicalPlan::CreateMemoryTable(_create_memory_table) => "CreateMemoryTable",
+ LogicalPlan::DropTable(_drop_table) => "DropTable",
+ LogicalPlan::DropView(_drop_view) => "DropView",
+ LogicalPlan::Values(_values) => "Values",
+ LogicalPlan::Explain(_explain) => "Explain",
+ LogicalPlan::Analyze(_analyze) => "Analyze",
+ LogicalPlan::Subquery(_sub_query) => "Subquery",
+ LogicalPlan::SubqueryAlias(_sqalias) => "SubqueryAlias",
+ LogicalPlan::CreateCatalogSchema(_create) => "CreateCatalogSchema",
+ LogicalPlan::CreateCatalog(_create_catalog) => "CreateCatalog",
+ LogicalPlan::CreateView(_create_view) => "CreateView",
+ // Further examine and return the name that is a possible Dask-SQL Extension type
+ LogicalPlan::Extension(extension) => {
+ let node = extension.node.as_any();
+ if node.downcast_ref::().is_some() {
+ "CreateModel"
+ } else if node.downcast_ref::().is_some() {
+ "CreateCatalogSchema"
+ } else if node.downcast_ref::().is_some() {
+ "CreateTable"
+ } else if node.downcast_ref::().is_some() {
+ "CreateView"
+ } else if node.downcast_ref::().is_some() {
+ "DropModel"
+ } else if node.downcast_ref::().is_some() {
+ "PredictModel"
+ } else if node.downcast_ref::().is_some() {
+ "ExportModel"
+ } else if node.downcast_ref::().is_some() {
+ "ShowModelParams"
+ } else if node.downcast_ref::().is_some() {
+ "ShowSchemas"
+ } else if node.downcast_ref::().is_some() {
+ "ShowTables"
+ } else if node.downcast_ref::().is_some() {
+ "ShowColumns"
+ } else if node.downcast_ref::().is_some() {
+ "ShowModels"
+ } else if node.downcast_ref::().is_some() {
+ "DropSchema"
+ } else if node.downcast_ref::().is_some() {
+ "UseSchema"
+ } else if node.downcast_ref::().is_some() {
+ "AnalyzeTable"
+ } else {
+ // Default to generic `Extension`
+ "Extension"
+ }
+ }
+ })
+ }
+
+ /// Explain plan for the full and original LogicalPlan
+ pub fn explain_original(&self) -> PyResult {
+ Ok(format!("{}", self.original_plan.display_indent()))
+ }
+
+ /// Explain plan from the current node onward
+ pub fn explain_current(&mut self) -> PyResult {
+ Ok(format!("{}", self.current_node().display_indent()))
+ }
+
+ #[pyo3(name = "getRowType")]
+ pub fn row_type(&self) -> PyResult {
+ match &self.original_plan {
+ LogicalPlan::Join(join) => {
+ let mut lhs_fields: Vec = join
+ .left
+ .schema()
+ .fields()
+ .iter()
+ .map(|f| RelDataTypeField::from(f, join.left.schema().as_ref()))
+ .collect::>>()
+ .map_err(py_type_err)?;
+
+ let mut rhs_fields: Vec = join
+ .right
+ .schema()
+ .fields()
+ .iter()
+ .map(|f| RelDataTypeField::from(f, join.right.schema().as_ref()))
+ .collect::>>()
+ .map_err(py_type_err)?;
+
+ lhs_fields.append(&mut rhs_fields);
+ Ok(RelDataType::new(false, lhs_fields))
+ }
+ LogicalPlan::Distinct(distinct) => {
+ let schema = distinct.input.schema();
+ let rel_fields: Vec = schema
+ .fields()
+ .iter()
+ .map(|f| RelDataTypeField::from(f, schema.as_ref()))
+ .collect::>>()
+ .map_err(py_type_err)?;
+ Ok(RelDataType::new(false, rel_fields))
+ }
+ _ => {
+ let schema = self.original_plan.schema();
+ let rel_fields: Vec = schema
+ .fields()
+ .iter()
+ .map(|f| RelDataTypeField::from(f, schema.as_ref()))
+ .collect::>>()
+ .map_err(py_type_err)?;
+ Ok(RelDataType::new(false, rel_fields))
+ }
+ }
+ }
+}
+
+impl From for LogicalPlan {
+ fn from(logical_plan: PyLogicalPlan) -> LogicalPlan {
+ logical_plan.original_plan
+ }
+}
+
+impl From for PyLogicalPlan {
+ fn from(logical_plan: LogicalPlan) -> PyLogicalPlan {
+ PyLogicalPlan {
+ original_plan: logical_plan,
+ current_node: None,
+ }
+ }
+}
diff --git a/dask_planner/src/sql/logical/aggregate.rs b/dask_planner/src/sql/logical/aggregate.rs
new file mode 100644
index 000000000..b181c158e
--- /dev/null
+++ b/dask_planner/src/sql/logical/aggregate.rs
@@ -0,0 +1,124 @@
+use crate::expression::{py_expr_list, PyExpr};
+
+use datafusion_expr::{logical_plan::Aggregate, logical_plan::Distinct, Expr, LogicalPlan};
+
+use crate::sql::exceptions::py_type_err;
+use pyo3::prelude::*;
+
+#[pyclass(name = "Aggregate", module = "dask_planner", subclass)]
+#[derive(Clone)]
+pub struct PyAggregate {
+ aggregate: Option,
+ distinct: Option,
+}
+
+#[pymethods]
+impl PyAggregate {
+ /// Determine the PyExprs that should be "Distinct-ed"
+ #[pyo3(name = "getDistinctColumns")]
+ pub fn distinct_columns(&self) -> PyResult> {
+ match &self.distinct {
+ Some(e) => Ok(e.input.schema().field_names()),
+ None => Err(py_type_err(
+ "distinct_columns invoked for non distinct instance",
+ )),
+ }
+ }
+
+ /// Returns a Vec of the group expressions
+ #[pyo3(name = "getGroupSets")]
+ pub fn group_expressions(&self) -> PyResult> {
+ match &self.aggregate {
+ Some(e) => py_expr_list(&e.input, &e.group_expr),
+ None => Ok(vec![]),
+ }
+ }
+
+ /// Returns the inner Aggregate Expr(s)
+ #[pyo3(name = "getNamedAggCalls")]
+ pub fn agg_expressions(&self) -> PyResult> {
+ match &self.aggregate {
+ Some(e) => py_expr_list(&e.input, &e.aggr_expr),
+ None => Ok(vec![]),
+ }
+ }
+
+ #[pyo3(name = "getAggregationFuncName")]
+ pub fn agg_func_name(&self, expr: PyExpr) -> PyResult {
+ _agg_func_name(&expr.expr)
+ }
+
+ #[pyo3(name = "getArgs")]
+ pub fn aggregation_arguments(&self, expr: PyExpr) -> PyResult> {
+ self._aggregation_arguments(&expr.expr)
+ }
+
+ #[pyo3(name = "isAggExprDistinct")]
+ pub fn distinct_agg_expr(&self, expr: PyExpr) -> PyResult {
+ _distinct_agg_expr(&expr.expr)
+ }
+
+ #[pyo3(name = "isDistinctNode")]
+ pub fn distinct_node(&self) -> PyResult {
+ Ok(self.distinct.is_some())
+ }
+}
+
+impl PyAggregate {
+ fn _aggregation_arguments(&self, expr: &Expr) -> PyResult> {
+ match expr {
+ Expr::Alias(expr, _) => self._aggregation_arguments(expr.as_ref()),
+ Expr::AggregateFunction { fun: _, args, .. }
+ | Expr::AggregateUDF { fun: _, args, .. } => match &self.aggregate {
+ Some(e) => py_expr_list(&e.input, args),
+ None => Ok(vec![]),
+ },
+ _ => Err(py_type_err(
+ "Encountered a non Aggregate type in aggregation_arguments",
+ )),
+ }
+ }
+}
+
+fn _agg_func_name(expr: &Expr) -> PyResult {
+ match expr {
+ Expr::Alias(expr, _) => _agg_func_name(expr.as_ref()),
+ Expr::AggregateFunction { fun, .. } => Ok(fun.to_string()),
+ Expr::AggregateUDF { fun, .. } => Ok(fun.name.clone()),
+ _ => Err(py_type_err(
+ "Encountered a non Aggregate type in agg_func_name",
+ )),
+ }
+}
+
+fn _distinct_agg_expr(expr: &Expr) -> PyResult {
+ match expr {
+ Expr::Alias(expr, _) => _distinct_agg_expr(expr.as_ref()),
+ Expr::AggregateFunction { distinct, .. } => Ok(*distinct),
+ Expr::AggregateUDF { .. } => {
+ // DataFusion does not support DISTINCT in UDAFs
+ Ok(false)
+ }
+ _ => Err(py_type_err(
+ "Encountered a non Aggregate type in distinct_agg_expr",
+ )),
+ }
+}
+
+impl TryFrom for PyAggregate {
+ type Error = PyErr;
+
+ fn try_from(logical_plan: LogicalPlan) -> Result {
+ match logical_plan {
+ LogicalPlan::Aggregate(aggregate) => Ok(PyAggregate {
+ aggregate: Some(aggregate),
+ distinct: None,
+ }),
+ LogicalPlan::Distinct(distinct) => Ok(PyAggregate {
+ aggregate: None,
+ distinct: Some(distinct),
+ }),
+ _ => Err(py_type_err("unexpected plan")),
+ }
+ }
+}
diff --git a/dask_planner/src/sql/logical/analyze_table.rs b/dask_planner/src/sql/logical/analyze_table.rs
new file mode 100644
index 000000000..9e4761d90
--- /dev/null
+++ b/dask_planner/src/sql/logical/analyze_table.rs
@@ -0,0 +1,119 @@
+use crate::sql::exceptions::py_type_err;
+use crate::sql::logical;
+use pyo3::prelude::*;
+
+use datafusion_expr::logical_plan::{Extension, UserDefinedLogicalNode};
+use datafusion_expr::{Expr, LogicalPlan};
+
+use fmt::Debug;
+use std::{any::Any, fmt, sync::Arc};
+
+use datafusion_common::{DFSchema, DFSchemaRef};
+
+#[derive(Clone)]
+pub struct AnalyzeTablePlanNode {
+ pub schema: DFSchemaRef,
+ pub table_name: String,
+ pub schema_name: Option,
+ pub columns: Vec,
+}
+
+impl Debug for AnalyzeTablePlanNode {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ self.fmt_for_explain(f)
+ }
+}
+
+impl UserDefinedLogicalNode for AnalyzeTablePlanNode {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn inputs(&self) -> Vec<&LogicalPlan> {
+ vec![]
+ }
+
+ fn schema(&self) -> &DFSchemaRef {
+ &self.schema
+ }
+
+ fn expressions(&self) -> Vec {
+ // there is no need to expose any expressions here since DataFusion would
+ // not be able to do anything with expressions that are specific to
+ // ANALYZE TABLE {table_name}
+ vec![]
+ }
+
+ fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(
+ f,
+ "Analyze Table: table_name: {:?}, columns: {:?}",
+ self.table_name, self.columns
+ )
+ }
+
+ fn from_template(
+ &self,
+ _exprs: &[Expr],
+ _inputs: &[LogicalPlan],
+ ) -> Arc {
+ Arc::new(AnalyzeTablePlanNode {
+ schema: Arc::new(DFSchema::empty()),
+ table_name: self.table_name.clone(),
+ schema_name: self.schema_name.clone(),
+ columns: self.columns.clone(),
+ })
+ }
+}
+
+#[pyclass(name = "AnalyzeTable", module = "dask_planner", subclass)]
+pub struct PyAnalyzeTable {
+ pub(crate) analyze_table: AnalyzeTablePlanNode,
+}
+
+#[pymethods]
+impl PyAnalyzeTable {
+ #[pyo3(name = "getTableName")]
+ fn get_table_name(&self) -> PyResult {
+ Ok(self.analyze_table.table_name.clone())
+ }
+
+ #[pyo3(name = "getSchemaName")]
+ fn get_schema_name(&self) -> PyResult