feat(datafusion): initial implementation for Arrow Datafusion backend

.github/workflows/ibis-backends.yml

@@ -452,6 +452,95 @@ jobs:
           name: ${{ matrix.backend.name }}-${{ matrix.python-version }}
           path: junit.xml
 
+  test_datafusion:
+    name: DataFusion ${{ matrix.datafusion-version }} ubuntu-latest python-${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        datafusion-version:
+          - master
+          - latest
+        python-version:
+          - "3.7"
+          - "3.9"
+    steps:
+      - name: checkout ibis
+        uses: actions/checkout@v2
+        with:
+          path: ibis
+
+      - name: checkout datafusion
+        uses: actions/checkout@v2
+        if: ${{ matrix.datafusion-version == 'master' }}
+        with:
+          path: datafusion
+          repository: datafusion-contrib/datafusion-python
+
+      - name: install rust
+        uses: actions-rs/toolchain@v1
+        if: ${{ matrix.datafusion-version == 'master' }}
+        with:
+          profile: minimal
+          toolchain: stable
+
+      - name: install python
+        id: install_python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: restore virtualenv
+        uses: syphar/restore-virtualenv@v1
+        id: cache-virtualenv
+        with:
+          requirement_files: ibis/poetry.lock
+          custom_cache_key_element: ${{ matrix.datafusion-version }}-${{ steps.install_python.outputs.python-version }}
+
+      - name: upgrade pip
+        run: pip install -U pip
+
+      - uses: syphar/restore-pip-download-cache@v1
+        if: ${{ !fromJSON(steps.cache-virtualenv.outputs.cache-hit) }}
+        with:
+          requirement_files: ibis/poetry.lock
+          custom_cache_key_element: ${{ matrix.datafusion-version }}-${{ steps.install_python.outputs.python-version }}
+
+      - name: install maturin and poetry
+        if: ${{ !fromJSON(steps.cache-virtualenv.outputs.cache-hit) }}
+        run: pip install poetry maturin
+
+      - name: install datafusion
+        if: ${{ matrix.datafusion-version == 'master' }}
+        working-directory: datafusion
+        run: maturin develop
+
+      - name: install ibis
+        if: ${{ !fromJSON(steps.cache-virtualenv.outputs.cache-hit) }}
+        working-directory: ibis
+        run: poetry install --extras datafusion
+
+      - name: download backend data
+        working-directory: ibis
+        run: python ci/datamgr.py download
+
+      - name: install backend data
+        working-directory: ibis
+        run: python ci/datamgr.py datafusion
+
+      - name: run tests
+        working-directory: ibis
+        run: ./ci/run_tests.sh
+        env:
+          PYTEST_BACKENDS: datafusion
+
+      - name: publish test report
+        uses: actions/upload-artifact@v2
+        if: success() || failure()
+        with:
+          name: ${{ matrix.backend }}-${{ matrix.python-version }}
+          path: ibis/junit.xml
+
   backends:
     # this job exists so that we can use a single job from this workflow to gate merging
     runs-on: ubuntu-latest
@@ -461,5 +550,6 @@ jobs:
       - test_postgres
       - test_pyspark
       - test_simple_backends
+      - test_datafusion
     steps:
       - run: exit 0

ci/datamgr.py

@@ -448,6 +448,15 @@ def dask(**params):
     """
 
 
+@cli.command()
+def datafusion(**params):
+    """
+    The datafusion backend does not need test data, but we still
+    have an option for the backend for consistency, and to not
+    have to avoid calling `./datamgr.py datafusion` in the CI.
+    """
+
+
 @cli.command()
 def csv(**params):
     """