merge beta

weaviate · Jan 31, 2024 · 9a2de06 · 9a2de06
1 parent 21d3552
commit 9a2de06
Show file tree

Hide file tree

Showing 274 changed files with 34,642 additions and 1,550 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,5 +1,10 @@
 [run]
-omit = *tests*/*,*__init__.py
+omit = *tests*/*,*__init__.py,weaviate/proto/**
+
 
 [report]
-omit = *tests*/*,*__init__.py
+omit = *tests*/*,*__init__.py
+exclude_lines =
+    pragma: not covered
+    @overload
+    @abstractmethod
diff --git a/.flake8 b/.flake8
@@ -1,4 +1,10 @@
 [flake8]
 max-line-length = 100
-exclude = .git, venv, .venv, .pytest_cache, dist, .idea, docs/conf.py, weaviate/proto/**/*.py
-ignore = E203, E266, E501, E731, W503
+exclude = .git, venv, .venv, .pytest_cache, dist, .idea, docs/conf.py, weaviate/collections/orm.py, weaviate/collections/classes/orm.py, weaviate/proto/**/*.py
+ignore = D100, D104, D105, D107, E203, E266, E501, E731, W503
+
+# D100: Missing docstring in public module
+# D104: Missing docstring in public package
+# D105: Missing docstring in magic method
+# D107: Missing docstring in __init__
+
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -13,6 +13,10 @@ on:
       - publishing.md
   pull_request:
 
+env:
+  OLD_WEAVIATE_VERSION: preview--9ede0f8
+  NEW_WEAVIATE_VERSION: preview--9ede0f8
+
 jobs:
   lint-and-format:
     name: Run Linter and Formatter
@@ -21,7 +25,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: psf/black@stable
         with:
-          version: "22.10"
+            version: "23.9.1"
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
@@ -51,6 +55,7 @@ jobs:
       - run: pip install -r requirements-devel.txt
       - name: Run mypy
         run: mypy --warn-unused-ignores --python-version ${{matrix.version}} ${{ matrix.folder }}
+      - uses: jakebailey/pyright-action@v2
 
   unit-tests:
     name: Run Unit Tests
@@ -76,72 +81,131 @@ jobs:
           name: coverage-report-${{ matrix.folder }}
           path: coverage-${{ matrix.folder }}.xml
 
+  integration-tests-v3:
+    name: Run Integration Tests v3
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        versions: [
+          { py: "3.8", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.9", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.10", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.11", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.11", weaviate: $OLD_WEAVIATE_VERSION},
+          { py: "3.12", weaviate: $NEW_WEAVIATE_VERSION}
+        ]
+
+        optional_dependencies: [false]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.versions.py }}
+          cache: 'pip' # caching pip dependencies
+      - run: |
+          pip install -r requirements-devel.txt
+          pip install .
+      - name: start weaviate
+        run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }}
+      - name: Run integration tests with auth secrets
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        env:
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
+          WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
+          OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
+#          OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}  disabled until we have a working key
+        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3
+      - name: Run integration tests without auth secrets (for forks)
+        if: ${{ github.event.pull_request.head.repo.fork }}
+        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3
+      - name: Archive code coverage results
+        if: matrix.versions.py == '3.10' && (github.ref_name != 'main')
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-report-integration-v3
+          path: coverage-integration-v3.xml
+
   integration-tests:
     name: Run Integration Tests
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        version: [ "3.8", "3.9", "3.10", "3.11", "3.12"]
+        versions: [
+          { py: "3.8", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.9", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.10", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.11", weaviate: $NEW_WEAVIATE_VERSION},
+          { py: "3.11", weaviate: $OLD_WEAVIATE_VERSION},
+          { py: "3.12", weaviate: $NEW_WEAVIATE_VERSION}
+        ]
+
         optional_dependencies: [false]
-        include:
-          - version: "3.11"
-            optional_dependencies: true
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           fetch-tags: true
       - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.version }}
+          python-version: ${{ matrix.versions.py }}
           cache: 'pip' # caching pip dependencies
       - run: |
           pip install -r requirements-devel.txt
           pip install .
       - name: start weaviate
-        run: /bin/bash ci/start_weaviate.sh
+        run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }}
       - name: Run integration tests with auth secrets
         if: ${{ !github.event.pull_request.head.repo.fork }}
         env:
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
           WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
           OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
-        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
+#          OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}  disabled until we have a working key
+        run: pytest -n auto -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
       - name: Run integration tests without auth secrets (for forks)
         if: ${{ github.event.pull_request.head.repo.fork }}
         run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
       - name: Archive code coverage results
-        if: matrix.version == '3.10' && (github.ref_name != 'main')
+        if: matrix.versions.py == '3.10' && (github.ref_name != 'main')
         uses: actions/upload-artifact@v3
         with:
           name: coverage-report-integration
           path: coverage-integration.xml
 
   Codecov:
-    needs: [Unit-Tests, Integration-Tests]
+    needs: [Unit-Tests, Integration-Tests, Integration-Tests-v3]
     runs-on: ubuntu-latest
     if: github.ref_name != 'main'
     steps:
       - uses: actions/checkout@v3
-      - name: Download coverage artifacts integration
+      - name: Download coverage artifacts mock
         uses: actions/download-artifact@v3
         with:
           name: coverage-report-mock_tests
-      - name: Download coverage artifacts integration
+      - name: Download coverage artifacts unit
         uses: actions/download-artifact@v3
         with:
           name: coverage-report-test
-      - name: Download coverage unit
+      - name: Download coverage integration
         uses: actions/download-artifact@v3
         with:
           name: coverage-report-integration
+      - name: Download coverage integration v3
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage-report-integration-v3
       - name: Codecov
         uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: true
-          files: ./coverage-integration.xml, ./coverage-test.xml, ./coverage-mock_tests.xml
+          files: ./coverage-integration.xml, ./coverage-integration-v3.xml, ./coverage-test.xml, ./coverage-mock_tests.xml
           verbose: true
 
 
@@ -173,32 +237,39 @@ jobs:
   test-package:
     needs: [build-package]
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        server: [
+          $OLD_WEAVIATE_VERSION,
+          $NEW_WEAVIATE_VERSION,
+        ]
     steps:
       - name: Download build artifact to append to release
         uses: actions/download-artifact@v3
         with:
           name: weaviate-python-client-wheel
       - run: |
           pip install weaviate_client-*.whl
-          pip install pytest pytest-benchmark pytest-profiling
+          pip install pytest pytest-benchmark pytest-profiling grpcio grpcio-tools pytest-xdist
       - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - run: rm -r weaviate
       - name: start weaviate
-        run: /bin/bash ci/start_weaviate.sh
+        run: /bin/bash ci/start_weaviate.sh ${{ matrix.server }}
       - name: Run integration tests with auth secrets
         if: ${{ !github.event.pull_request.head.repo.fork }}
         env:
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
           WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
           OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
-        run: pytest -v  integration
+        run: pytest -v -n auto integration
       - name: Run integration tests without auth secrets (for forks)
         if: ${{ github.event.pull_request.head.repo.fork }}
-        run: pytest -v integration
+        run: pytest -v -n auto integration
 
 
   build-and-publish:

diff --git a/.gitignore b/.gitignore
@@ -16,7 +16,10 @@ nohup.out
 .coverage
 coverage.xml
 .pytest_cache
+.mypy_cache
 
 image.png
 .vscode/
-scratch/
+scratch/
+
+*-test.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,34 +1,52 @@
 exclude: ^proto/
 repos:
 -   repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 23.9.1
     hooks:
     -   id: black
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.3.0
+  rev: v4.4.0
   hooks:
     - id: no-commit-to-branch
     - id: trailing-whitespace
 
--   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
-    hooks:
+- repo: https://github.com/PyCQA/flake8
+  rev: 6.1.0
+  hooks:
     - id: flake8
+      name: linting
       additional_dependencies: [
         'flake8-bugbear==22.10.27',
         'flake8-comprehensions==3.10.1',
         'flake8-builtins==2.0.1'
       ]
+    - id: flake8
+      name: docstrings
+      additional_dependencies: [
+        'flake8-docstrings==1.7.0'
+      ]
+      files: '^weaviate/collections'
 
 -   repo: local
     hooks:
     - id: mypy
       name: mypy
       entry: ./run-mypy.sh
       language: python
+      language_version: "3.11"
       # use require_serial so that script
       # is only called once per commit
       require_serial: true
       # Print the number of files as a sanity-check
-      verbose: true
+      verbose: true
+
+- repo: local
+  hooks:
+    - id: pyright
+      name: pyright
+      entry: pyright
+      language: node
+      pass_filenames: false
+      types: [python]
+      additional_dependencies: [[email protected]]
diff --git a/ci/docker-compose-async.yml b/ci/docker-compose-async.yml
@@ -9,9 +9,10 @@ services:
       - '8090'
       - --scheme
       - http
-    image: semitechnologies/weaviate:1.23.0-rc.0
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
     ports:
       - "8090:8090"
+      - "50060:50051"
     restart: on-failure:0
     environment:
       QUERY_DEFAULTS_LIMIT: 25

diff --git a/ci/docker-compose-azure.yml b/ci/docker-compose-azure.yml
@@ -10,7 +10,7 @@ services:
       - --scheme
       - http
       - --write-timeout=600s
-    image: semitechnologies/weaviate:1.23.0-rc.0
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
     ports:
       - 8081:8081
     restart: on-failure:0

diff --git a/ci/docker-compose-cluster.yml b/ci/docker-compose-cluster.yml
@@ -2,10 +2,11 @@
 version: '3.4'
 services:
   weaviate-node-1:
-    image: semitechnologies/weaviate:1.23.0-rc.0
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
     restart: on-failure:0
     ports:
       - "8087:8080"
+      - "50058:50051"
     environment:
       CONTEXTIONARY_URL: contextionary:9999
       QUERY_DEFAULTS_LIMIT: 20
@@ -25,9 +26,10 @@ services:
       - '8080'
       - --scheme
       - http
-    image: semitechnologies/weaviate:1.23.0-rc.0
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
     ports:
       - 8088:8080
+      - "50059:50051"
       - 6061:6060
     restart: on-failure:0
     environment:

diff --git a/ci/docker-compose-openai.yml → ci/docker-compose-generative.yml b/ci/docker-compose-openai.yml → ci/docker-compose-generative.yml
@@ -9,14 +9,15 @@ services:
       - '8086'
       - --scheme
       - http
-    image: semitechnologies/weaviate:1.23.0-rc.0
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
     ports:
       - 8086:8086
+      - "50057:50051"
     restart: on-failure:0
     environment:
       QUERY_DEFAULTS_LIMIT: 25
       AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
       PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
       DEFAULT_VECTORIZER_MODULE: 'text2vec-openai'
-      ENABLE_MODULES: 'text2vec-openai,generative-openai'
+      ENABLE_MODULES: 'text2vec-openai,generative-openai,text2vec-cohere,generative-cohere'
       CLUSTER_HOSTNAME: 'node1'