pr

LudoCorporateShark · Aug 27, 2024 · f686c35 · f686c35
2 parents 3c2176d + 2150747
commit f686c35
Show file tree

Hide file tree

Showing 277 changed files with 22,354 additions and 1 deletion.
diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml
@@ -16,6 +16,13 @@ on:
 permissions:
   contents: read
 
+<<<<<<< HEAD
+=======
+env:
+  # Configure a constant location for the uv cache
+  UV_CACHE_DIR: /tmp/.uv-cache
+
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
 jobs:
   paths-filter:
     runs-on: ubuntu-latest
@@ -49,19 +56,44 @@ jobs:
       matrix:
         python-version: ["3.11"]
         os: [ubuntu-latest]
+<<<<<<< HEAD
     steps:
       - uses: actions/checkout@v4
       - name: Install poetry
         run: pipx install poetry
+=======
+    defaults:
+      run:
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv
+        if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }}
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Set up uv
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: irm https://astral.sh/uv/install.ps1 | iex
+        shell: powershell
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+<<<<<<< HEAD
           cache: "poetry"
+=======
+      - name: Restore uv cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{  env.UV_CACHE_DIR  }}
+          key: uv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/uv.lock') }}
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
       - name: Install dependencies with hnswlib native disabled
         if: matrix.os == 'macos-latest' && matrix.python-version == '3.11'
         run: |
           export HNSWLIB_NO_NATIVE=1
+<<<<<<< HEAD
           python -m pip install --upgrade pip setuptools wheel
           cd python && poetry install --with tests
       - name: Install dependencies with hnswlib native enabled
@@ -70,6 +102,13 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel
           cd python
           poetry install --with tests
+=======
+          uv sync --all-extras --dev
+      - name: Install dependencies with hnswlib native enabled
+        if: matrix.os != 'macos-latest' || matrix.python-version != '3.11'
+        run: |
+          uv sync --all-extras --dev
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
       - name: Install Ollama
         if: matrix.os == 'ubuntu-latest'
         run: |
@@ -98,7 +137,11 @@ jobs:
       - name: Run Integration Tests
         id: run_tests
         shell: bash
+<<<<<<< HEAD
         env: # Set Azure credentials secret as an input
+=======
+        env:
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
           HNSWLIB_NO_NATIVE: 1
           Python_Integration_Tests: Python_Integration_Tests
           AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # azure-text-embedding-ada-002
@@ -111,6 +154,10 @@ jobs:
           OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }}
           OPENAI_TEXT_MODEL_ID: ${{ vars.OPENAI_TEXT_MODEL_ID }}
           OPENAI_EMBEDDING_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
+<<<<<<< HEAD
+=======
+          OPENAI_TEXT_TO_IMAGE_MODEL_ID: ${{ vars.OPENAI_TEXT_TO_IMAGE_MODEL_ID }}
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           PINECONE_API_KEY: ${{ secrets.PINECONE__APIKEY }}
           POSTGRES_CONNECTION_STRING: ${{secrets.POSTGRES__CONNECTIONSTR}}
@@ -124,6 +171,11 @@ jobs:
           MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}}
           MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }}
           MISTRALAI_EMBEDDING_MODEL_ID: ${{ vars.MISTRALAI_EMBEDDING_MODEL_ID }}
+<<<<<<< HEAD
+=======
+          ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}}
+          ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
           OLLAMA_MODEL: "${{ matrix.os == 'ubuntu-latest' && vars.OLLAMA_MODEL || '' }}" # phi3
           GOOGLE_AI_GEMINI_MODEL_ID: ${{ vars.GOOGLE_AI_GEMINI_MODEL_ID }}
           GOOGLE_AI_EMBEDDING_MODEL_ID: ${{ vars.GOOGLE_AI_EMBEDDING_MODEL_ID }}
@@ -132,13 +184,19 @@ jobs:
           VERTEX_AI_GEMINI_MODEL_ID: ${{ vars.VERTEX_AI_GEMINI_MODEL_ID }}
           VERTEX_AI_EMBEDDING_MODEL_ID: ${{ vars.VERTEX_AI_EMBEDDING_MODEL_ID }}
           REDIS_CONNECTION_STRING: ${{ vars.REDIS_CONNECTION_STRING }}
+<<<<<<< HEAD
         run: |
           cd python
           poetry run pytest -n logical --dist loadfile --dist worksteal ./tests/integration ./tests/samples -v --junitxml=pytest.xml
+=======
+        run: |          
+          uv run pytest -n logical --dist loadfile --dist worksteal ./tests/integration ./tests/samples -v --junitxml=pytest.xml
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
       - name: Surface failing tests
         if: always()
         uses: pmeier/pytest-results-action@main
         with:
+<<<<<<< HEAD
           # A list of JUnit XML files, directories containing the former, and wildcard
           # patterns to process.
           # See @actions/glob for supported patterns.
@@ -152,6 +210,15 @@ jobs:
           fail-on-empty: true
           # (Optional) Title of the test results section in the workflow summary
           title: Test results
+=======
+          path: python/pytest.xml
+          summary: true
+          display-options: fEX
+          fail-on-empty: true
+          title: Test results
+      - name: Minimize uv cache
+        run: uv cache prune --ci
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
 
   python-integration-tests:
     needs: paths-filter
@@ -163,19 +230,44 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
         os: [ubuntu-latest, windows-latest, macos-latest]
+<<<<<<< HEAD
     steps:
       - uses: actions/checkout@v4
       - name: Install poetry
         run: pipx install poetry
+=======
+    defaults:
+      run:
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv
+        if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }}
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Set up uv
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: irm https://astral.sh/uv/install.ps1 | iex
+        shell: powershell
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+<<<<<<< HEAD
           cache: "poetry"
+=======
+      - name: Restore uv cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.UV_CACHE_DIR }}
+          key: uv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/uv.lock') }}
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
       - name: Install dependencies with hnswlib native disabled
         if: matrix.os == 'macos-latest' && matrix.python-version == '3.11'
         run: |
           export HNSWLIB_NO_NATIVE=1
+<<<<<<< HEAD
           python -m pip install --upgrade pip setuptools wheel
           cd python && poetry install --with tests
 
@@ -198,11 +290,34 @@ jobs:
           ollama pull ${{ vars.OLLAMA_MODEL }}
           ollama list
 
+=======
+          uv sync --all-extras --dev
+      - name: Install dependencies with hnswlib native enabled
+        if: matrix.os != 'macos-latest' || matrix.python-version != '3.11'
+        run: |
+          uv sync --all-extras --dev
+      - name: Install Ollama
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          if ${{ vars.OLLAMA_MODEL != '' }}; then
+            curl -fsSL https://ollama.com/install.sh | sh
+            ollama serve &
+            sleep 5
+          fi
+      - name: Pull model in Ollama
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          if ${{ vars.OLLAMA_MODEL != '' }}; then
+            ollama pull ${{ vars.OLLAMA_MODEL }}
+            ollama list
+          fi
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
       - name: Google auth
         uses: google-github-actions/auth@v2
         with:
           project_id: ${{ vars.VERTEX_AI_PROJECT_ID }}
           credentials_json: ${{ secrets.VERTEX_AI_SERVICE_ACCOUNT_KEY }}
+<<<<<<< HEAD
 
       - name: Set up gcloud
         uses: google-github-actions/setup-gcloud@v2
@@ -211,6 +326,17 @@ jobs:
         id: run_tests
         shell: bash
         env: # Set Azure credentials secret as an input
+=======
+      - name: Set up gcloud
+        uses: google-github-actions/setup-gcloud@v2      
+      - name: Setup Redis Stack Server
+        if: matrix.os == 'ubuntu-latest'
+        run: docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
+      - name: Run Integration Tests
+        id: run_tests
+        shell: bash
+        env: 
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
           HNSWLIB_NO_NATIVE: 1
           Python_Integration_Tests: Python_Integration_Tests
           AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # azure-text-embedding-ada-002
@@ -223,6 +349,10 @@ jobs:
           OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_ID }}
           OPENAI_TEXT_MODEL_ID: ${{ vars.OPENAI_TEXT_MODEL_ID }}
           OPENAI_EMBEDDING_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
+<<<<<<< HEAD
+=======
+          OPENAI_TEXT_TO_IMAGE_MODEL_ID: ${{ vars.OPENAI_TEXT_TO_IMAGE_MODEL_ID }}
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           PINECONE_API_KEY: ${{ secrets.PINECONE__APIKEY }}
           POSTGRES_CONNECTION_STRING: ${{secrets.POSTGRES__CONNECTIONSTR}}
@@ -236,6 +366,11 @@ jobs:
           MISTRALAI_API_KEY: ${{secrets.MISTRALAI_API_KEY}}
           MISTRALAI_CHAT_MODEL_ID: ${{ vars.MISTRALAI_CHAT_MODEL_ID }}
           MISTRALAI_EMBEDDING_MODEL_ID: ${{ vars.MISTRALAI_EMBEDDING_MODEL_ID }}
+<<<<<<< HEAD
+=======
+          ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}}
+          ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
           OLLAMA_MODEL: "${{ matrix.os == 'ubuntu-latest' && vars.OLLAMA_MODEL || '' }}" # phi3
           GOOGLE_AI_GEMINI_MODEL_ID: ${{ vars.GOOGLE_AI_GEMINI_MODEL_ID }}
           GOOGLE_AI_EMBEDDING_MODEL_ID: ${{ vars.GOOGLE_AI_EMBEDDING_MODEL_ID }}
@@ -245,13 +380,28 @@ jobs:
           VERTEX_AI_EMBEDDING_MODEL_ID: ${{ vars.VERTEX_AI_EMBEDDING_MODEL_ID }}
           REDIS_CONNECTION_STRING: ${{ vars.REDIS_CONNECTION_STRING }}
         run: |
+<<<<<<< HEAD
           if ${{ matrix.os == 'ubuntu-latest' }}; then
             docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
           fi
 
           cd python
           poetry run pytest -n logical --dist loadfile --dist worksteal ./tests/integration -v
           poetry run pytest -n logical --dist loadfile --dist worksteal ./tests/samples -v
+=======
+          uv run pytest -n logical --dist loadfile --dist worksteal ./tests/integration ./tests/samples -v --junitxml=pytest.xml
+      - name: Surface failing tests
+        if: always()
+        uses: pmeier/pytest-results-action@main
+        with:
+          path: python/pytest.xml
+          summary: true
+          display-options: fEX
+          fail-on-empty: true
+          title: Test results
+      - name: Minimize uv cache
+        run: uv cache prune --ci
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39
 
   # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
   python-integration-tests-check:

diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml
@@ -15,6 +15,7 @@ jobs:
         python-version: ["3.10"]
     runs-on: ubuntu-latest
     continue-on-error: true
+<<<<<<< HEAD
     steps:
       - uses: actions/checkout@v4
       - name: Install poetry
@@ -26,3 +27,31 @@ jobs:
       - name: Install dependencies
         run: cd python && poetry install
       - uses: pre-commit/[email protected]
+=======
+    defaults:
+      run:
+        working-directory: python
+    env:
+      # Configure a constant location for the uv cache
+      UV_CACHE_DIR: /tmp/.uv-cache
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Restore uv cache
+        uses: actions/cache@v4
+        with:
+          path: ${{  env.UV_CACHE_DIR  }}
+          key: uv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/uv.lock') }}
+      - name: Install the project
+        run: uv sync --all-extras --dev
+      - uses: pre-commit/[email protected]
+        with:
+          extra_args: --config python/.pre-commit-config.yaml
+      - name: Minimize uv cache
+        run: uv cache prune --ci
+>>>>>>> 21507472dbfb2f82c4cee14827624e2d14dc0b39