Sync backup save

.github/workflows/deploy_backend.yml

@@ -0,0 +1,59 @@
+name: Deploy Backend
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tags:
+        description: 'Branch or tag (e.g: v0.0.1)'
+        required: false
+        type: string
+
+env:
+  IMAGE_NAME: cohere-toolkit-backend
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    # needs: build-and-push-backend-image
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Auth
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+
+      - name: Configure Docker
+        run: gcloud auth configure-docker
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ vars.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Deploy the image to Cloud Run
+      - name: Deploy to Cloud Run
+        run: |
+          gcloud run deploy ${{ env.IMAGE_NAME }} \
+            --project=${{ vars.GCP_PROJECT_ID }} \
+            --image=${{ steps.meta.outputs.tags }} \
+            --platform=managed \
+            --region=us-central1 \
+            --allow-unauthenticated \
+            --update-env-vars=DATABASE_URL=${{ secrets.DATABASE_URL }},COHERE_API_KEY=${{ secrets.COHERE_API_KEY }},MINIMAP_API_URL=${{ vars.MINIMAP_API_URL }} \
+            --memory=4Gi \
+            --cpu=4 \
+            --max-instances=1

.github/workflows/docker_push_backend.yml

@@ -11,37 +11,61 @@ on:
         type: string
 
 env:
-  REGISTRY: ghcr.io
   IMAGE_NAME: cohere-toolkit-backend
 
 jobs:
   build-and-push-backend-image:
     runs-on: ubuntu-latest
-    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
     permissions:
       contents: read
-      packages: write
+
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+
+      - name: Login
+        uses: google-github-actions/setup-gcloud@v0
         with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          project_id: ${{ vars.GCP_PROJECT_ID }}
+          service_account_email: ${{ secrets.SA_EMAIL }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Configure Docker
+        run: gcloud auth configure-docker --quiet
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - run: 'echo "$KEY" > ./key.json'
+        shell: bash
+        env:
+          KEY: ${{ secrets.GCP_SA_KEY }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/cohere-ai/${{ env.IMAGE_NAME }}
-      - name: Build and push Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        uses: docker/metadata-action@v3
         with:
-          context: .
-          tags: ${{ steps.meta.outputs.tags }}
-          file: ./src/backend/Dockerfile
-          push: true
+          images: ${{ vars.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build Docker image
+        run: |
+          docker build -t ${{ env.IMAGE_NAME }} -f ./src/backend/Dockerfile .
+
+      - name: Tag Docker image
+        run: |
+          docker tag ${{ env.IMAGE_NAME }} ${{ steps.meta.outputs.tags }}
 
+      - name: Push Docker image
+        run: |
+          docker push ${{ steps.meta.outputs.tags }}
 
+      - name: Deploy to Cloud Run
+        run: |
+          gcloud run deploy ${{ env.IMAGE_NAME }} \
+            --project=${{ vars.GCP_PROJECT_ID }} \
+            --image=${{ steps.meta.outputs.tags }} \
+            --platform=managed \
+            --region=us-central1 \
+            --allow-unauthenticated \
+            --update-env-vars=DATABASE_URL=${{ secrets.DATABASE_URL }},COHERE_API_KEY=${{ secrets.COHERE_API_KEY }},MINIMAP_API_URL=${{ vars.MINIMAP_API_URL }} \
+            --memory=4Gi \
+            --cpu=4 \
+            --max-instances=1

.github/workflows/docker_push_frontend.yml

@@ -11,33 +11,35 @@ on:
         type: string
 
 env:
-  REGISTRY: ghcr.io
   IMAGE_NAME: cohere-toolkit-frontend
 
 jobs:
   build-and-push-frontend-image:
     runs-on: ubuntu-latest
-    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
     permissions:
       contents: read
-      packages: write
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
         with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          project_id: ${{ env.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Authenticate Docker to Artifact Registry
+        run: gcloud auth configure-docker ${{ env.REGISTRY }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@v3
         with:
-          images: ${{ env.REGISTRY }}/cohere-ai/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
       - name: Build and push Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        uses: docker/build-push-action@v2
         with:
           context: ./src/interfaces/coral_web
           tags: ${{ steps.meta.outputs.tags }}

cloudbuild.yaml

@@ -1,8 +1,8 @@
 substitutions:
-  _GCP_PROJECT_ID: <GCP_PROJECT_ID>
-  _GCP_REGION: <GCP_REGION>
-  _DATABASE_URL: <DB_CONNECTION_STRING>
-  _COHERE_API_KEY: <COHERE_API_KEY>
+  _GCP_PROJECT_ID: $PROJECT_ID
+  _GCP_REGION: $REGION
+  _DATABASE_URL: $DATABASE_URL
+  _COHERE_API_KEY: $COHERE_API_KEY
 
 steps:
 # 1. BACKEND STEPS
@@ -15,8 +15,8 @@ steps:
 # c) Deploy container image to Cloud Run
 - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
   entrypoint: gcloud
-  args: ['run', 'deploy', 'toolkit-backend', 
-  '--image', 'gcr.io/$_GCP_PROJECT_ID/toolkit-backend', 
+  args: ['run', 'deploy', 'toolkit-backend',
+  '--image', 'gcr.io/$_GCP_PROJECT_ID/toolkit-backend',
   '--region', '$_GCP_REGION',
   '--port', '8000',
   '--set-env-vars', 'DATABASE_URL=$_DATABASE_URL',
@@ -33,10 +33,10 @@ steps:
 # e) Apply backend Alembic migrations
 - name: gcr.io/cloud-builders/curl
   entrypoint: bash
-  args: 
+  args:
     - '-c'
     - 'curl -X POST "$(cat /workspace/backend_url.txt)/migrate"'
-  
+
 # 2. FRONTEND STEPS
 # a) Build frontend container image
 - name: 'gcr.io/cloud-builders/docker'
@@ -50,7 +50,7 @@ steps:
   args:
     - '-c'
     - |
-        gcloud run deploy toolkit-frontend --port 4000 --memory 2Gi --image gcr.io/$_GCP_PROJECT_ID/toolkit-frontend --region $_GCP_REGION  --allow-unauthenticated --set-env-vars NEXT_PUBLIC_API_HOSTNAME=$(cat /workspace/backend_url.txt) 
+        gcloud run deploy toolkit-frontend --port 4000 --memory 2Gi --image gcr.io/$_GCP_PROJECT_ID/toolkit-frontend --region $_GCP_REGION  --allow-unauthenticated --set-env-vars NEXT_PUBLIC_API_HOSTNAME=$(cat /workspace/backend_url.txt)
 images:
 - gcr.io/$_GCP_PROJECT_ID/toolkit-backend
 - gcr.io/$_GCP_PROJECT_ID/toolkit-frontend

src/backend/Dockerfile

@@ -28,7 +28,7 @@ ARG INSTALL_COMMUNITY_DEPS
 COPY pyproject.toml poetry.lock ./
 
 # Install poetry
-RUN pip install --no-cache-dir poetry==1.6.1
+RUN pip install --no-cache-dir --upgrade pip setuptools wheel poetry
 
 # Conditional installation of dependencies
 RUN if [ "$INSTALL_COMMUNITY_DEPS" = "true" ]; then \
@@ -40,7 +40,7 @@ RUN if [ "$INSTALL_COMMUNITY_DEPS" = "true" ]; then \
 COPY src/backend/ src/backend/
 COPY src/community src/community/
 
-# Copy environment variables optionally 
+# Copy environment variables optionally
 # IMPORTANT: Can't be put in the docker-compose, will break tests
 COPY .en[v] .env
 

src/backend/config/tools.py

@@ -11,13 +11,14 @@
     ReadFileTool,
     SearchFileTool,
     TavilyInternetSearch,
+    LangChainMinimapRetriever,
 )
 
 """
-List of available tools. Each tool should have a name, implementation, is_visible and category. 
+List of available tools. Each tool should have a name, implementation, is_visible and category.
 They can also have kwargs if necessary.
 
-You can switch the visibility of a tool by changing the is_visible parameter to True or False. 
+You can switch the visibility of a tool by changing the is_visible parameter to True or False.
 If a tool is not visible, it will not be shown in the frontend.
 
 If you want to add a new tool, check the instructions on how to implement a retriever in the documentation.
@@ -32,6 +33,7 @@ class ToolName(StrEnum):
     Python_Interpreter = "Python_Interpreter"
     Calculator = "Calculator"
     Tavily_Internet_Search = "Internet_Search"
+    Minimap = "Minimap"
 
 
 ALL_TOOLS = {
@@ -52,6 +54,22 @@ class ToolName(StrEnum):
         category=Category.DataLoader,
         description="Retrieves documents from Wikipedia using LangChain.",
     ),
+    ToolName.Minimap: ManagedTool(
+        name=ToolName.Minimap,
+        implementation=LangChainMinimapRetriever,
+        parameter_definitions={
+            "query": {
+                "description": "Query for searching the news. It can be a topic, a named entities, or a quoted keyword. Summarizations should be in the form of a bulleted list where each bullet point is its own core topic.",
+                "type": "str",
+                "required": True,
+            }
+        },
+        is_visible=True,
+        is_available=LangChainMinimapRetriever.is_available(),
+        error_message="Minimap API not available.",
+        category=Category.DataLoader,
+        description="Fetches the most relevant news and content from Minimap.ai.",
+    ),
     ToolName.Search_File: ManagedTool(
         name=ToolName.Search_File,
         implementation=SearchFileTool,

src/backend/tools/__init__.py

@@ -3,6 +3,7 @@
 from backend.tools.lang_chain import LangChainVectorDBRetriever, LangChainWikiRetriever
 from backend.tools.python_interpreter import PythonInterpreter
 from backend.tools.tavily import TavilyInternetSearch
+from backend.tools.minimap import LangChainMinimapRetriever
 
 __all__ = [
     "Calculator",
@@ -12,4 +13,5 @@
     "TavilyInternetSearch",
     "ReadFileTool",
     "SearchFileTool",
+    "LangChainMinimapRetriever",
 ]