Changes default location for the database and adds the option to use …

…the previous behavior (#1907) * chore: Refactor loading of settings and handle missing keys * chore: Update .env.example file with new configuration options * Ignore database files in .gitignore * Refactor loading of settings and handle missing keys * Update DOWNLOAD_WEBHOOK_URL in base settings to use the correct URL for triggering flows in the Langflow store. * 🐛 (base.py): Fix condition to copy existing database to new location only if SAVE_DB_IN_CONFIG_DIR is true * 🐛 (base.py): update log message to accurately reflect the action being taken when creating a new database
langflow-ai · May 15, 2024 · 53dc025 · 53dc025
1 parent fdf1881
commit 53dc025
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 183 deletions.
diff --git a/.env.example b/.env.example
@@ -4,6 +4,19 @@
 #        Do not commit .env file to git
 #        Do not change .env.example file
 
+# Config directory
+# Directory where files, logs and database will be stored
+# Example: LANGFLOW_CONFIG_DIR=~/.langflow
+LANGFLOW_CONFIG_DIR=
+
+# Save database in the config directory
+# Values: true, false
+# If false, the database will be saved in Langflow's root directory
+# This means that the database will be deleted when Langflow is uninstalled
+# and that the database will not be shared between different virtual environments
+# Example: LANGFLOW_SAVE_DB_IN_CONFIG_DIR=true
+LANGFLOW_SAVE_DB_IN_CONFIG_DIR=
+
 # Database URL
 # Postgres example: LANGFLOW_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/langflow
 # SQLite example:
@@ -56,7 +69,6 @@ LANGFLOW_REMOVE_API_KEYS=
 # LANGFLOW_REDIS_CACHE_EXPIRE (default: 3600)
 LANGFLOW_CACHE_TYPE=
 
-
 # Set AUTO_LOGIN to false if you want to disable auto login
 # and use the login form to login. LANGFLOW_SUPERUSER and LANGFLOW_SUPERUSER_PASSWORD
 # must be set if AUTO_LOGIN is set to false

diff --git a/src/backend/.gitignore b/src/backend/.gitignore
@@ -131,3 +131,4 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+*.db
diff --git a/src/backend/base/langflow/services/settings/base.py b/src/backend/base/langflow/services/settings/base.py
@@ -7,13 +7,12 @@
 
 import orjson
 import yaml
+from langflow.services.settings.constants import VARIABLES_TO_GET_FROM_ENVIRONMENT
 from loguru import logger
-from pydantic import field_validator, validator
+from pydantic import field_validator
 from pydantic.fields import FieldInfo
 from pydantic_settings import BaseSettings, EnvSettingsSource, PydanticBaseSettingsSource, SettingsConfigDict
 
-from langflow.services.settings.constants import VARIABLES_TO_GET_FROM_ENVIRONMENT
-
 # BASE_COMPONENTS_PATH = str(Path(__file__).parent / "components")
 BASE_COMPONENTS_PATH = str(Path(__file__).parent.parent.parent / "components")
 
@@ -76,6 +75,10 @@ class Settings(BaseSettings):
 
     # Define the default LANGFLOW_DIR
     CONFIG_DIR: Optional[str] = None
+    # Define if langflow db should be saved in config dir or
+    # in the langflow directory
+    SAVE_DB_IN_CONFIG_DIR: bool = False
+    """Define if langflow database should be saved in LANGFLOW_CONFIG_DIR or in the langflow directory (i.e. in the package directory)."""
 
     DEV: bool = False
     DATABASE_URL: Optional[str] = None
@@ -113,7 +116,7 @@ class Settings(BaseSettings):
     variables_to_get_from_environment: list[str] = VARIABLES_TO_GET_FROM_ENVIRONMENT
     """List of environment variables to get from the environment and store in the database."""
 
-    @validator("CONFIG_DIR", pre=True, allow_reuse=True)
+    @field_validator("CONFIG_DIR", mode="before")
     def set_langflow_dir(cls, value):
         if not value:
             from platformdirs import user_cache_dir
@@ -136,8 +139,8 @@ def set_langflow_dir(cls, value):
 
         return str(value)
 
-    @validator("DATABASE_URL", pre=True)
-    def set_database_url(cls, value, values):
+    @field_validator("DATABASE_URL", mode="before")
+    def set_database_url(cls, value, info):
         if not value:
             logger.debug("No database_url provided, trying LANGFLOW_DATABASE_URL env variable")
             if langflow_database_url := os.getenv("LANGFLOW_DATABASE_URL"):
@@ -148,29 +151,36 @@ def set_database_url(cls, value, values):
                 # Originally, we used sqlite:///./langflow.db
                 # so we need to migrate to the new format
                 # if there is a database in that location
-                if not values["CONFIG_DIR"]:
+                if not info.data["CONFIG_DIR"]:
                     raise ValueError("CONFIG_DIR not set, please set it or provide a DATABASE_URL")
                 from langflow.version import is_pre_release  # type: ignore
 
+                if info.data["SAVE_DB_IN_CONFIG_DIR"]:
+                    database_dir = info.data["CONFIG_DIR"]
+                    logger.debug(f"Saving database to CONFIG_DIR: {database_dir}")
+                else:
+                    database_dir = Path(__file__).parent.parent.parent.resolve()
+                    logger.debug(f"Saving database to langflow directory: {database_dir}")
+
                 pre_db_file_name = "langflow-pre.db"
                 db_file_name = "langflow.db"
-                new_pre_path = f"{values['CONFIG_DIR']}/{pre_db_file_name}"
-                new_path = f"{values['CONFIG_DIR']}/{db_file_name}"
+                new_pre_path = f"{database_dir}/{pre_db_file_name}"
+                new_path = f"{database_dir}/{db_file_name}"
                 final_path = None
                 if is_pre_release:
                     if Path(new_pre_path).exists():
                         final_path = new_pre_path
-                    elif Path(new_path).exists():
+                    elif Path(new_path).exists() and info.data["SAVE_DB_IN_CONFIG_DIR"]:
                         # We need to copy the current db to the new location
                         logger.debug("Copying existing database to new location")
                         copy2(new_path, new_pre_path)
                         logger.debug(f"Copied existing database to {new_pre_path}")
-                    elif Path(f"./{db_file_name}").exists():
+                    elif Path(f"./{db_file_name}").exists() and info.data["SAVE_DB_IN_CONFIG_DIR"]:
                         logger.debug("Copying existing database to new location")
                         copy2(f"./{db_file_name}", new_pre_path)
                         logger.debug(f"Copied existing database to {new_pre_path}")
                     else:
-                        logger.debug(f"Database already exists at {new_pre_path}, using it")
+                        logger.debug(f"Creating new database at {new_pre_path}")
                         final_path = new_pre_path
                 else:
                     if Path(new_path).exists():
@@ -311,3 +321,6 @@ def load_settings_from_yaml(file_path: str) -> Settings:
             logger.debug(f"Loading {len(settings_dict[key])} {key} from {file_path}")
 
     return Settings(**settings_dict)
+    return Settings(**settings_dict)
+    return Settings(**settings_dict)
+    return Settings(**settings_dict)
diff --git a/src/backend/base/langflow/services/settings/service.py b/src/backend/base/langflow/services/settings/service.py
@@ -1,11 +1,10 @@
 import os
 
 import yaml
-from loguru import logger
-
 from langflow.services.base import Service
 from langflow.services.settings.auth import AuthSettings
 from langflow.services.settings.base import Settings
+from loguru import logger
 
 
 class SettingsService(Service):
@@ -31,7 +30,7 @@ def load_settings_from_yaml(cls, file_path: str) -> "SettingsService":
 
             for key in settings_dict:
                 if key not in Settings.model_fields.keys():
-                    raise KeyError(f"Key {key} not found in settings")
+                    logger.warning(f"Key {key} not found in settings")
                 logger.debug(f"Loading {len(settings_dict[key])} {key} from {file_path}")
 
         settings = Settings(**settings_dict)

diff --git a/tests/test_template.py b/tests/test_template.py
@@ -3,8 +3,7 @@
 
 import pytest
 from langflow.interface.utils import build_template_from_class
-from langflow.utils.constants import CHAT_OPENAI_MODELS, OPENAI_MODELS
-from langflow.utils.util import build_template_from_function, format_dict, get_base_classes, get_default_factory
+from langflow.utils.util import build_template_from_function, get_base_classes, get_default_factory
 from pydantic import BaseModel
 
 
@@ -88,171 +87,6 @@ def test_build_template_from_class():
         build_template_from_class("InvalidClass", type_to_cls_dict)
 
 
-# Test format_dict
-def test_format_dict():
-    # Test 1: Optional type removal
-    input_dict = {
-        "field1": {"type": "Optional[str]", "required": False},
-    }
-    expected_output = {
-        "field1": {
-            "type": "str",
-            "required": False,
-            "list": False,
-            "show": False,
-            "password": False,
-            "multiline": False,
-        },
-    }
-    assert format_dict(input_dict) == expected_output
-
-    # Test 2: List type processing
-    input_dict = {
-        "field1": {"type": "List[str]", "required": False},
-    }
-    expected_output = {
-        "field1": {
-            "type": "str",
-            "required": False,
-            "list": True,
-            "show": False,
-            "password": False,
-            "multiline": False,
-        },
-    }
-    assert format_dict(input_dict) == expected_output
-
-    # Test 3: Mapping type replacement
-    input_dict = {
-        "field1": {"type": "Mapping[str, int]", "required": False},
-    }
-    expected_output = {
-        "field1": {
-            "type": "dict[str, int]",  # Mapping type is replaced with dict which is replaced with code
-            "required": False,
-            "list": False,
-            "show": False,
-            "password": False,
-            "multiline": False,
-        },
-    }
-    assert format_dict(input_dict) == expected_output
-
-    # Test 4: Replace default value with actual value
-    input_dict = {
-        "field1": {"type": "str", "required": False, "default": "test"},
-    }
-    expected_output = {
-        "field1": {
-            "type": "str",
-            "required": False,
-            "list": False,
-            "show": False,
-            "password": False,
-            "multiline": False,
-            "value": "test",
-        },
-    }
-    assert format_dict(input_dict) == expected_output
-
-    # Test 5: Add password field
-    input_dict = {
-        "field1": {"type": "str", "required": False},
-        "api_key": {"type": "str", "required": False},
-    }
-    expected_output = {
-        "field1": {
-            "type": "str",
-            "required": False,
-            "list": False,
-            "show": False,
-            "password": False,
-            "multiline": False,
-        },
-        "api_key": {
-            "type": "str",
-            "required": False,
-            "list": False,
-            "show": True,
-            "password": True,
-            "multiline": False,
-        },
-    }
-    assert format_dict(input_dict) == expected_output
-
-    # Test 6: Add multiline
-    input_dict = {
-        "field1": {"type": "str", "required": False},
-        "prefix": {"type": "str", "required": False},
-    }
-    expected_output = {
-        "field1": {
-            "type": "str",
-            "required": False,
-            "list": False,
-            "show": False,
-            "password": False,
-            "multiline": False,
-        },
-        "prefix": {
-            "type": "str",
-            "required": False,
-            "list": False,
-            "show": True,
-            "password": False,
-            "multiline": True,
-        },
-    }
-    assert format_dict(input_dict) == expected_output
-
-    # Test 7: Check class name-specific cases (OpenAI, ChatOpenAI)
-    input_dict = {
-        "model_name": {"type": "str", "required": False},
-    }
-    expected_output_openai = {
-        "model_name": {
-            "type": "str",
-            "required": False,
-            "list": True,
-            "show": True,
-            "password": False,
-            "multiline": False,
-            "options": OPENAI_MODELS,
-            "value": "text-davinci-003",
-        },
-    }
-    expected_output_openai_chat = {
-        "model_name": {
-            "type": "str",
-            "required": False,
-            "list": True,
-            "show": True,
-            "password": False,
-            "multiline": False,
-            "options": CHAT_OPENAI_MODELS,
-            "value": "gpt-4-turbo-preview",
-        },
-    }
-    assert format_dict(input_dict, "OpenAI") == expected_output_openai
-    assert format_dict(input_dict, "ChatOpenAI") == expected_output_openai_chat
-
-    # Test 8: Replace dict type with str
-    input_dict = {
-        "field1": {"type": "Dict[str, int]", "required": False},
-    }
-    expected_output = {
-        "field1": {
-            "type": "Dict[str, int]",
-            "required": False,
-            "list": False,
-            "show": False,
-            "password": False,
-            "multiline": False,
-        },
-    }
-    assert format_dict(input_dict) == expected_output
-
-
 # Test get_base_classes
 def test_get_base_classes():
     base_classes_parent = get_base_classes(Parent)