Skip to content

Commit 6b62d08

Browse files
AstraBertAstraBert
committed
v3.0.0
1 parent 116b7e3 commit 6b62d08

File tree

7 files changed

+175
-21
lines changed

7 files changed

+175
-21
lines changed

‎.env

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,7 @@
11
VOLUME="/source/local-machine/dir:target/multi-container/app/dir"
2-
# VOLUME="c:/Users/User/:/User" e.g.
2+
# VOLUME="c:/Users/User/:/User" e.g.
3+
MODELS_PATH="/path/to/gguf/models"
4+
# MODELS_PATH="c:/Users/User/.cache/llama.cpp/"
5+
MODEL="your_favorite_model.gguf"
6+
# MODEL="stories260K.gguf"
7+
MAX_TOKENS="512"

‎README.md

+14-6
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,26 @@ git clone https://github.com/AstraBert/everything-ai.git
2323
cd everything-ai
2424
```
2525
### 2. Set your `.env` file
26-
Modify the `VOLUME` variable in the .env file so that you can mount your local file system into Docker container.
26+
Modify:
27+
- `VOLUME` variable in the .env file so that you can mount your local file system into Docker container.
28+
- `MODELS_PATH` variable in the .env file so that you can tell llama.cpp where you stored the GGUF models you downloaded.
29+
- `MODEL` variable in the .env file so that you can tell llama.cpp what model to use (use the actual name of the gguf file, and do not forget the .gguf extension!)
30+
- `MAX_TOKENS` variable in the .env file so that you can tell llama.cpp how many new tokens it can generate as output.
2731

28-
An example could be:
32+
An example of a `.env` file could be:
2933
```bash
3034
VOLUME="c:/Users/User/:/User/"
35+
MODELS_PATH="c:/Users/User/.cache/llama.cpp/"
36+
MODEL="stories260K.gguf"
37+
MAX_TOKENS="512"
3138
```
32-
This means that now everything that is under "c:/Users/User/" on your local machine is under "/User/" in your Docker container.
39+
This means that now everything that is under "c:/Users/User/" on your local machine is under "/User/" in your Docker container, that llama.cpp knows where to look for models and what model to look for, along with the maximum new tokens for its output.
3340

3441
### 3. Pull the necessary images
3542
```bash
36-
docker pull astrabert/everything-ai
37-
docker pull qdrant/qdrant
43+
docker pull astrabert/everything-ai:latest
44+
docker pull qdrant/qdrant:latest
45+
docker pull ghcr.io/ggerganov/llama.cpp:server
3846
```
3947
### 4. Run the multi-container app
4048
```bash
@@ -63,6 +71,7 @@ Choose the task among:
6371
- *protein-folding*: get the 3D structure of a protein from its amino-acid sequence, using ESM-2 backbone model - **GPU ONLY**
6472
- *autotrain*: fine-tune a model on a specific downstream task with autotrain-advanced, just by specifying you HF username, HF writing token and the path to a yaml config file for the training
6573
- *spaces-api-supabase*: use HF Spaces API in combination with Supabase PostgreSQL databases in order to unleash more powerful LLMs and larger RAG-oriented vector databases - **MULTILINGUAL**
74+
- *llama.cpp-and-qdrant*: same as *retrieval-text-generation*, but uses **llama.cpp** as inference engine, so you MUST NOT specify a model - **MULTILINGUAL**
6675
- *image-retrieval-search*: search an image database uploading a folder as database input. The folder should have the following structure:
6776

6877
```
@@ -87,4 +96,3 @@ Once everything is ready, you can head over to `localhost:7860` and start using
8796
</div>
8897

8998

90-
## Complete documentation is coming soon...🚀

‎compose.yaml

+12-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ services:
66
everything-ai:
77
image: astrabert/everything-ai
88
volumes:
9-
- ${VOLUME}
9+
- $VOLUME
1010
networks:
1111
- mynet
1212
ports:
@@ -19,4 +19,14 @@ services:
1919
volumes:
2020
- "./qdrant_storage:/qdrant/storage"
2121
networks:
22-
- mynet
22+
- mynet
23+
llama_server:
24+
image: ghcr.io/ggerganov/llama.cpp:server
25+
ports:
26+
- "8000:8000"
27+
volumes:
28+
- "$MODELS_PATH:/models"
29+
networks:
30+
- mynet
31+
command: "-m /models/$MODEL --port 8000 --host 0.0.0.0 -n $MAX_TOKENS"
32+

‎docker/Dockerfile

-3
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,6 @@ WORKDIR /app
77
# Add the current directory contents into the container at /app
88
ADD . /app
99

10-
#Upgrade gradio
11-
RUN pip install gradio_molecule3d
12-
1310
# Expose the port that the application will run on
1411
EXPOSE 8760
1512

‎docker/llama_cpp_int.py

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
from utils import Translation, PDFdatabase, NeuralSearcher
2+
import gradio as gr
3+
from qdrant_client import QdrantClient
4+
from sentence_transformers import SentenceTransformer
5+
from argparse import ArgumentParser
6+
import os
7+
8+
argparse = ArgumentParser()
9+
10+
argparse.add_argument(
11+
"-pf",
12+
"--pdf_file",
13+
help="Single pdf file or N pdfs reported like this: /path/to/file1.pdf,/path/to/file2.pdf,...,/path/to/fileN.pdf (there is no strict naming, you just need to provide them comma-separated)",
14+
required=False,
15+
default="No file"
16+
)
17+
18+
argparse.add_argument(
19+
"-d",
20+
"--directory",
21+
help="Directory where all your pdfs of interest are stored",
22+
required=False,
23+
default="No directory"
24+
)
25+
26+
argparse.add_argument(
27+
"-l",
28+
"--language",
29+
help="Language of the written content contained in the pdfs",
30+
required=False,
31+
default="Same as query"
32+
)
33+
34+
args = argparse.parse_args()
35+
36+
37+
pdff = args.pdf_file
38+
dirs = args.directory
39+
lan = args.language
40+
41+
42+
if pdff.replace("\\","").replace("'","") != "None" and dirs.replace("\\","").replace("'","") == "No directory":
43+
pdfs = pdff.replace("\\","/").replace("'","").split(",")
44+
else:
45+
pdfs = [os.path.join(dirs.replace("\\","/").replace("'",""), f) for f in os.listdir(dirs.replace("\\","/").replace("'","")) if f.endswith(".pdf")]
46+
47+
client = QdrantClient(host="host.docker.internal", port="6333")
48+
encoder = SentenceTransformer("all-MiniLM-L6-v2")
49+
50+
pdfdb = PDFdatabase(pdfs, encoder, client)
51+
pdfdb.preprocess()
52+
pdfdb.collect_data()
53+
pdfdb.qdrant_collection_and_upload()
54+
55+
56+
import requests
57+
58+
def llama_cpp_respond(query, max_new_tokens):
59+
url = "http://host.docker.internal:8000/completion"
60+
headers = {
61+
"Content-Type": "application/json"
62+
}
63+
data = {
64+
"prompt": query,
65+
"n_predict": int(max_new_tokens)
66+
}
67+
68+
response = requests.post(url, headers=headers, json=data)
69+
70+
a = response.json()
71+
return a["content"]
72+
73+
74+
def reply(max_new_tokens, message):
75+
global pdfdb
76+
txt = Translation(message, "en")
77+
if txt.original == "en" and lan.replace("\\","").replace("'","") == "None":
78+
txt2txt = NeuralSearcher(pdfdb.collection_name, pdfdb.client, pdfdb.encoder)
79+
results = txt2txt.search(message)
80+
response = llama_cpp_respond(results[0]["text"], max_new_tokens)
81+
return response
82+
elif txt.original == "en" and lan.replace("\\","").replace("'","") != "None":
83+
txt2txt = NeuralSearcher(pdfdb.collection_name, pdfdb.client, pdfdb.encoder)
84+
transl = Translation(message, lan.replace("\\","").replace("'",""))
85+
message = transl.translatef()
86+
results = txt2txt.search(message)
87+
t = Translation(results[0]["text"], txt.original)
88+
res = t.translatef()
89+
response = llama_cpp_respond(res, max_new_tokens)
90+
return response
91+
elif txt.original != "en" and lan.replace("\\","").replace("'","") == "None":
92+
txt2txt = NeuralSearcher(pdfdb.collection_name, pdfdb.client, pdfdb.encoder)
93+
results = txt2txt.search(message)
94+
transl = Translation(results[0]["text"], "en")
95+
translation = transl.translatef()
96+
response = llama_cpp_respond(translation, max_new_tokens)
97+
t = Translation(response, txt.original)
98+
res = t.translatef()
99+
return res
100+
else:
101+
txt2txt = NeuralSearcher(pdfdb.collection_name, pdfdb.client, pdfdb.encoder)
102+
transl = Translation(message, lan.replace("\\","").replace("'",""))
103+
message = transl.translatef()
104+
results = txt2txt.search(message)
105+
t = Translation(results[0]["text"], txt.original)
106+
res = t.translatef()
107+
response = llama_cpp_respond(res, max_new_tokens)
108+
tr = Translation(response, txt.original)
109+
ress = tr.translatef()
110+
return ress
111+
112+
demo = gr.Interface(
113+
reply,
114+
[
115+
gr.Textbox(
116+
label="Max new tokens",
117+
info="The number reported should not be higher than the one specified within the .env file",
118+
lines=3,
119+
value=f"512",
120+
),
121+
gr.Textbox(
122+
label="Input query",
123+
info="Write your input query here",
124+
lines=3,
125+
value=f"What are penguins?",
126+
)
127+
],
128+
title="everything-ai-llamacpp",
129+
outputs="textbox"
130+
)
131+
demo.launch(server_name="0.0.0.0", share=False)

‎docker/retrieval_text_generation.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,10 @@
4848
lan = args.language
4949

5050

51-
if pdff.replace("\\","").replace("'","") != "None" and dirs.replace("\\","").replace("'","") == "None":
52-
pdfs = pdff.replace("\\","").replace("'","").split(",")
51+
if pdff.replace("\\","").replace("'","") != "None" and dirs.replace("\\","").replace("'","") == "No directory":
52+
pdfs = pdff.replace("\\","/").replace("'","").split(",")
5353
else:
54-
pdfs = [os.path.join(dirs.replace("\\","").replace("'",""), f) for f in os.listdir(dirs.replace("\\","").replace("'","")) if f.endswith(".pdf")]
54+
pdfs = [os.path.join(dirs.replace("\\","/").replace("'",""), f) for f in os.listdir(dirs.replace("\\","/").replace("'","")) if f.endswith(".pdf")]
5555

5656
client = QdrantClient(host="host.docker.internal", port="6333")
5757
encoder = SentenceTransformer("all-MiniLM-L6-v2")

‎docker/select_and_run.py

+9-6
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
import subprocess as sp
22
import gradio as gr
33

4-
TASK_TO_SCRIPT = {"retrieval-text-generation": "retrieval_text_generation.py", "agnostic-text-generation": "agnostic_text_generation.py", "text-summarization": "text_summarization.py", "image-generation": "image_generation.py", "image-generation-pollinations": "image_generation_pollinations.py", "image-classification": "image_classification.py", "image-to-text": "image_to_text.py", "retrieval-image-search": "retrieval_image_search.py", "protein-folding": "protein_folding_with_esm.py", "video-generation": "video_generation.py", "speech-recognition": "speech_recognition.py", "spaces-api-supabase": "spaces_api_supabase.py", "audio-classification": "audio_classification.py", "autotrain": "autotrain_interface.py"}
4+
TASK_TO_SCRIPT = {"retrieval-text-generation": "retrieval_text_generation.py", "agnostic-text-generation": "agnostic_text_generation.py", "text-summarization": "text_summarization.py", "image-generation": "image_generation.py", "image-generation-pollinations": "image_generation_pollinations.py", "image-classification": "image_classification.py", "image-to-text": "image_to_text.py", "retrieval-image-search": "retrieval_image_search.py", "protein-folding": "protein_folding_with_esm.py", "video-generation": "video_generation.py", "speech-recognition": "speech_recognition.py", "spaces-api-supabase": "spaces_api_supabase.py", "audio-classification": "audio_classification.py", "autotrain": "autotrain_interface.py", "llama.cpp-and-qdrant": "llama_cpp_int.py"}
55

66

77
def build_command(tsk, mod="None", pdff="None", dirs="None", lan="None", imdim="512", gradioclient="None", supabaseurl="None", collectname="None", supenc="all-MiniLM-L6-v2", supdim="384"):
8-
if tsk != "retrieval-text-generation" and tsk != "image-generation-pollinations" and tsk != "retrieval-image-search" and tsk != "autotrain" and tsk != "protein-folding" and tsk != "spaces-api-supabase":
8+
if tsk != "retrieval-text-generation" and tsk != "image-generation-pollinations" and tsk != "retrieval-image-search" and tsk != "autotrain" and tsk != "protein-folding" and tsk != "spaces-api-supabase" and tsk != "llama.cpp-and-qdrant":
99
sp.run(f"python3 {TASK_TO_SCRIPT[tsk]} -m {mod}", shell=True)
1010
return f"python3 {TASK_TO_SCRIPT[tsk]} -m {mod}"
1111
elif tsk == "retrieval-text-generation":
1212
sp.run(f"python3 {TASK_TO_SCRIPT[tsk]} -m {mod} -pf '{pdff}' -d '{dirs}' -l '{lan}'", shell=True)
1313
return f"python3 {TASK_TO_SCRIPT[tsk]} -m {mod} -pf '{pdff}' -d '{dirs}' -l '{lan}'"
14+
elif tsk == "llama.cpp-and-qdrant":
15+
sp.run(f"python3 {TASK_TO_SCRIPT[tsk]} -pf '{pdff}' -d '{dirs}' -l '{lan}'", shell=True)
16+
return f"python3 {TASK_TO_SCRIPT[tsk]} -pf '{pdff}' -d '{dirs}' -l '{lan}'"
1417
elif tsk == "image-generation-pollinations" or tsk == "autotrain" or tsk == "protein-folding":
1518
sp.run(f"python3 {TASK_TO_SCRIPT[tsk]}", shell=True)
1619
return f"python3 {TASK_TO_SCRIPT[tsk]}"
@@ -41,15 +44,15 @@ def build_command(tsk, mod="None", pdff="None", dirs="None", lan="None", imdim="
4144
),
4245
gr.Textbox(
4346
label="PDF file(s)",
44-
info="Single pdf file or N pdfs reported like this: /path/to/file1.pdf,/path/to/file2.pdf,...,/path/to/fileN.pdf (there is no strict naming, you just need to provide them comma-separated): only available with 'retrieval-text-generation'",
47+
info="Single pdf file or N pdfs reported like this: /path/to/file1.pdf,/path/to/file2.pdf,...,/path/to/fileN.pdf (there is no strict naming, you just need to provide them comma-separated), please do not use '\\' as path separators: only available with 'retrieval-text-generation'",
4548
lines=3,
46-
value="None",
49+
value="No file",
4750
),
4851
gr.Textbox(
4952
label="Directory",
50-
info="Directory where all your pdfs or images (.jpg, .jpeg, .png) of interest are stored (only available with 'retrieval-text-generation' for pdfs and 'retrieval-image-search' for images)",
53+
info="Directory where all your pdfs or images (.jpg, .jpeg, .png) of interest are stored (only available with 'retrieval-text-generation' for pdfs and 'retrieval-image-search' for images). Please do not use '\\' as path separators",
5154
lines=3,
52-
value="None",
55+
value="No directory",
5356
),
5457
gr.Textbox(
5558
label="Language",

0 commit comments

Comments
 (0)