feat: Deploy llm-assistant stack on the GPU node

ansible/gpus.yml

@@ -10,8 +10,13 @@
         add_persistent_storage_device: /dev/vdb
         add_persistent_storage_mount_point: /mnt/models
     - role: setup_nvidia_ctk
+    - role: deploy_litellm
+      vars:
+        deploy_litellm_configfile: ../ollama/litellm-config.yaml
+        deploy_litellm_project_name: ollama
     - role: docker_compose
       vars:
+        docker_compose_envfile: ../ollama/.env
         docker_compose_project_name: ollama
         docker_compose_srcfile: ../ollama/docker-compose.yaml
     - role: pull_ollama_model

ansible/roles/deploy_litellm/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+deploy_litellm_configfile: ../ollama/litellm-config.yaml
+deploy_litellm_project_name: ollama

ansible/roles/deploy_litellm/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Create /opt/{{ deploy_litellm_project_name }}
+  become: true
+  ansible.builtin.file:
+    path: "/opt/{{ deploy_litellm_project_name }}"
+    state: directory
+    owner: deploy
+    group: deploy
+    mode: "0755"
+- name: Copy litellm-config.yaml to project directory
+  become: true
+  ansible.builtin.copy:
+    src: "{{ deploy_litellm_configfile }}"
+    dest: "/opt/{{ deploy_litellm_project_name }}/litellm-config.yaml"
+    owner: deploy
+    group: deploy
+    mode: "0644"

ansible/roles/docker_compose/defaults/main.yml

@@ -1,3 +1,4 @@
 ---
+docker_compose_envfile: ../app/.env.example
 docker_compose_project_name: app
 docker_compose_srcfile: ../app/docker-compose.yaml

ansible/roles/docker_compose/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create {{ docker_compose_project_name }}
+- name: Create /opt/{{ docker_compose_project_name }}
   become: true
   ansible.builtin.file:
     path: "/opt/{{ docker_compose_project_name }}"
@@ -8,6 +8,15 @@
     group: deploy
     mode: "0755"
 
+- name: Copy env file for {{ docker_compose_project_name }}
+  become: true
+  ansible.builtin.template:
+    src: "{{ docker_compose_envfile }}"
+    dest: "/opt/{{ docker_compose_project_name }}/.env"
+    owner: deploy
+    group: deploy
+    mode: "0644"
+
 - name: Copy {{ docker_compose_srcfile }}
   become: true
   ansible.builtin.copy:

ollama/.env.example

@@ -0,0 +1,4 @@
+DISCORD_BOT_TOKEN=<DISCORD_BOT_TOKEN>
+DISCORD_GUILD_ID=<DISCORD_GUILD_ID>
+OPENAI_API_KEY=<OPENAI_API_KEY>
+LITELLM_WORKERS_NUM=2

ollama/docker-compose.yaml

@@ -1,7 +1,9 @@
+name: vait
+
 services:
   ollama:
     container_name: ollama
-    image: ollama/ollama:0.1.25
+    image: ollama/ollama:0.1.29
     ports:
       - 11434:11434
     volumes:
@@ -13,5 +15,25 @@ services:
             - driver: nvidia
               count: all
               capabilities: [gpu]
-volumes:
-  ollama:
+  litellm:
+    container_name: litellm
+    image: ghcr.io/berriai/litellm:main-v1.33.9
+    ports:
+      - 4000:4000
+    volumes:
+      - ./litellm-config.yaml:/app/config.yaml
+    command: "--config /app/config.yaml --num_workers $LITELLM_WORKERS_NUM"
+    environment:
+      - OPENAI_API_KEY=$OPENAI_API_KEY
+    depends_on:
+      - ollama
+  llm-bot:
+    container_name: llm-bot
+    image: ghcr.io/bifrostlab/llm-assistant/llm-assistant:latest
+    environment:
+      - AI_SERVER_URL=http://litellm:4000
+      - DISCORD_BOT_TOKEN=$DISCORD_BOT_TOKEN
+      - DISCORD_GUILD_ID=$DISCORD_GUILD_ID
+    depends_on:
+      - ollama
+      - litellm

ollama/litellm-config.yaml

@@ -0,0 +1,29 @@
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: gpt-4
+    litellm_params:
+      model: gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: mixtral
+    litellm_params:
+      model: ollama/mixtral
+      api_base: http://ollama:11434
+  - model_name: mistral
+    litellm_params:
+      model: ollama/mistral
+      api_base: http://ollama:11434
+  - model_name: llama2
+    litellm_params:
+      model: ollama/llama2
+      api_base: http://ollama:11434
+
+litellm_params:
+  drop_params: True
+
+router_settings:
+  num_retries: 2
+  allowed_fails: 2
+  timeout: 60