-
Notifications
You must be signed in to change notification settings - Fork 0
/
rag_config.py
85 lines (74 loc) · 6.5 KB
/
rag_config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
import os
class Configuration:
OPENAI_ENVIRONMENT = os.getenv("OPENAI_ENVIRONMENT", "DEV")
################# AZURE Settings ##############################################################################
AZURE_CLIENT_ID = os.getenv("AZURE_CLIENT_ID")
AZURE_TENANT_ID = os.getenv("AZURE_TENANT_ID")
# Update this with the scopes you've configured in Azure AD
#SCOPE = ["https://your-tenant-name.onmicrosoft.com/your-scope"]
AZURE_AD_WELL_KNOWN_URL = f"https://login.microsoftonline.com/{AZURE_TENANT_ID}/v2.0/.well-known/openid-configuration"
AZURE_AD_KEYS_URL = f"https://sts.windows.net/{AZURE_TENANT_ID}/discovery/v2.0/keys"
AZURE_AD_ISSUER_URL = f"https://sts.windows.net/{AZURE_TENANT_ID}/"
AZURE_AD_AUDIENCE = f"api://{AZURE_CLIENT_ID}"
AZURE_STORAGE_CONTAINER_SAS_TOKEN = os.getenv("AZURE_STORAGE_CONTAINER_SAS_TOKEN")
AZURE_STORAGE_ACCOUNT_URL = os.getenv("AZURE_STORAGE_ACCOUNT_URL","https:/funcst01.blob.core.windows.net")
AZURE_STORAGE_CONTAINER_NAME = os.getenv("AZURE_STORAGE_CONTAINER_NAME","openai-kb-upload")
AZURE_STORAGE_CONTAINER_LATEST_FILE_PATH = os.getenv("AZURE_STORAGE_CONTAINER_LATEST_FILE_PATH","latest/")
AZURE_STORAGE_CONTAINER_ARCHIVE_FILE_PATH = os.getenv("AZURE_STORAGE_CONTAINER_ARCHIVE_FILE_PATH","archive/")
################# AZURE Settings ###############################################################################
################# OPENAI Settings ##############################################################################
OPENAI_API_BASE = os.getenv("OPENAI_API_BASE")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
OPENAI_GPT_DEPLOYMENT_NAME = "gpt-35-turbo-16k"
OPENAI_GPT_API_VERSION = "2023-05-15"
OPENAI_EMBEDDINGS_MODEL = "text-embedding-ada-002"
OPENAI_GPT_API_EMBEDDING_VERSION = "2023-03-15-preview"
################# OPENAI Settings ##############################################################################
################# DB Settings ##################################################################################
PGVECTOR_SCHEMA = os.getenv("PGVECTOR_SCHEMA", "pgnopenai")
PGVECTOR_DATABASE = os.getenv("PGVECTOR_DATABASE", "pgnopenai")
PGVECTOR_USER = os.getenv("PGVECTOR_USER", "pgnopenai_user")
PGVECTOR_HOST = os.getenv("PGVECTOR_HOST")
PGVECTOR_PORT = os.getenv("PGVECTOR_PORT", "5432")
PGVECTOR_COLLECTION_NAME = os.getenv("PGVECTOR_COLLECTION_NAME", "pg_collection")
PGVECTOR_PASSWORD = os.getenv("PGVECTOR_PASSWORD")
PG_LANGCHAIN_CONNECTION_STRING = f"postgresql+psycopg2://{PGVECTOR_USER}:{PGVECTOR_PASSWORD}@{PGVECTOR_HOST}:{PGVECTOR_PORT}/{PGVECTOR_DATABASE}?options=-csearch_path%3D{PGVECTOR_SCHEMA},public&sslmode=require"
PG_CONNECTION_STRING = "host='%s' dbname='%s' user='%s' password='%s' sslmode='require' port='%s'" % (PGVECTOR_HOST, PGVECTOR_DATABASE, PGVECTOR_USER, PGVECTOR_PASSWORD, PGVECTOR_PORT)
EMBEDDINGS_DELETE_QUERY = "DELETE FROM pgnopenai.langchain_pg_embedding WHERE cmetadata->>'source' LIKE %s;"
EMBEDDINGS_COUNT_QUERY = "SELECT COUNT(*) FROM pgnopenai.langchain_pg_embedding WHERE cmetadata->>'source' LIKE %s;"
INTERACTION_AUDIT_LOG_INSERT_QUERY = '''INSERT INTO openai_interaction_audit_log (start_timestamp, end_timestamp, user_query, context, model_response, prompt_token,
completion_token, total_token, prompt, feedback, conversation_history, metadata, status, interaction_detail)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING interaction_id;'''
OPENAI_UPLOAD_BATCH_INSERT_QUERY = '''INSERT INTO pgnopenai.openai_upload_batch(batch_start_timestamp, batch_end_timestamp, batch_upload_status, created_by, updated_by, updated_date)
VALUES (%s, %s, %s, %s, %s, %s)
RETURNING openai_upload_batch_id;'''
OPENAI_UPLOAD_BATCH_UPLOAD_QUERY = '''UPDATE pgnopenai.openai_upload_batch
SET batch_end_timestamp=%s, batch_upload_status=%s, updated_by=%s, updated_date=%s
WHERE openai_upload_batch_id=%s;'''
OPENAI_UPLOAD_FILE_INSERT_QUERY = '''INSERT INTO pgnopenai.openai_upload_file(
openai_upload_batch_id, upload_source_file_name, upload_destination_file_name, file_page_count, file_upload_status, upload_file_timestamp)
VALUES (%s, %s, %s, %s, %s, %s)
RETURNING openai_upload_file_id;'''
OPENAI_UPLOAD_FILE_UPDATE_QUERY = '''UPDATE pgnopenai.openai_upload_file
SET upload_destination_file_name=%s, file_page_count=%s, file_upload_status=%s,upload_file_timestamp=%s
WHERE openai_upload_file_id=%s;'''
### Following columns belongs to OPENAI_UPLOAD_FILE table
UPLOAD_DESTINATION_FILE_NAME_COLUMN = "upload_destination_file_name"
FILE_PAGE_COUNT_COLUMN = "file_page_count"
EMBEDDING_COUNT_NEW = "embedding_count_new"
EMBEDDING_COUNT_OLD = "embedding_count_old"
FILE_UPLOAD_STATUS_COLUMN = "file_upload_status"
################# DB Settings ####################################################################################
################# APPLICATION Settings ###########################################################################
APPLICATION_REFERENCE_FILES_INDEX = os.getenv("APPLICATION_REFERENCE_FILES_INDEX",["hgg.pdf",
"ABC.pdf", "abc1.pdf"])
APPLICATION_TEXT_SPLIT_CHUNK_SIZE = os.getenv("APPLICATION_TEXT_SPLIT_CHUNK_SIZE",1000)
APPLICATION_TEXT_SPLIT_CHUNK_OVERLAP = os.getenv("APPLICATION_TEXT_SPLIT_CHUNK_OVERLAP",200)
APPLICATION_OPENAI_EMBEDDING_CHUNK_SIZE = os.getenv("APPLICATION_OPENAI_EMBEDDING_CHUNK_SIZE",16)
APPLICATION_OPENAI_EMBEDDING_REQUEST_TIMEOUT = os.getenv("APPLICATION_OPENAI_EMBEDDING_REQUEST_TIMEOUT",30)
APPLICATION_OPENAI_EMBEDDING_MAX_RETRIES = os.getenv("APPLICATION_OPENAI_EMBEDDING_MAX_RETRIES",3)
APPLICATION_PGVECTOR_RETRIEVER_K = os.getenv("APPLICATION_PGVECTOR_RETRIEVER_K",10)
APPLICATION_PGVECTOR_RETRIEVER_FETCH_K = os.getenv("APPLICATION_PGVECTOR_RETRIEVER_FETCH_K",30)
APPLICATION_PGVECTOR_RETRIEVER_LAMBDA_MULT = os.getenv("APPLICATION_PGVECTOR_RETRIEVER_LAMBDA_MULT",0.75)
################# APPLICATION Settings #############################################################################