From 26bc6dfb7baae67eeecec0c480c99f9b9df0f734 Mon Sep 17 00:00:00 2001 From: Travis Abendshien <46939827+CyanVoxel@users.noreply.github.com> Date: Mon, 25 Aug 2025 00:49:39 -0700 Subject: [PATCH 1/5] refactor: store DB version inside `versions` table --- docs/updates/schema_changes.md | 34 +++- .../core/library/alchemy/constants.py | 18 +++ src/tagstudio/core/library/alchemy/library.py | 146 ++++++++++++------ src/tagstudio/core/library/alchemy/models.py | 9 ++ 4 files changed, 158 insertions(+), 49 deletions(-) create mode 100644 src/tagstudio/core/library/alchemy/constants.py diff --git a/docs/updates/schema_changes.md b/docs/updates/schema_changes.md index aa8488376..16eb4194b 100644 --- a/docs/updates/schema_changes.md +++ b/docs/updates/schema_changes.md @@ -26,7 +26,19 @@ Replaced by the new SQLite format introduced in TagStudio [v9.5.0 Pre-Release 1] ## SQLite -Starting with TagStudio [v9.5.0-pr1](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr1), the library save format has been moved to a [SQLite](https://sqlite.org) format. Legacy JSON libraries are migrated (with the user's consent) to the new format when opening in current versions of the program. The save format versioning is now separate from the program's versioning number and stored inside a `DB_VERSION` attribute inside the SQLite file. +Starting with TagStudio [v9.5.0-pr1](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr1), the library save format has been moved to a [SQLite](https://sqlite.org) format. Legacy JSON libraries are migrated (with the user's consent) to the new format when opening in current versions of the program. The save format versioning is now separate from the program's versioning number. + +Versions **1-100** stored the database version in a table called `preferences` in a row with the `key` column of `"DB_VERSION"` inside the corresponding `value` column. + +Versions **>101** store the database version in a table called `versions` in a row with the `key` column of `'CURRENT'` inside the corresponding `value` column. The `versions` table also stores the initial database version in which the file was created with under the `'INITIAL'` key. Databases created before this key was introduced will always have `'INITIAL'` value of `100`. + +```mermaid +erDiagram + versions { + TEXT key PK "Values: ['INITIAL', 'CURRENT']" + INTEGER value + } +``` ### Versions 1 - 5 @@ -81,11 +93,25 @@ Migration from the legacy JSON format is provided via a walkthrough when opening ### Version 100 -| Used From | Used Until | Format | Location | -| ----------------------------------------------------------------------- | ---------- | ------ | ----------------------------------------------- | -| [v9.5.4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.4) | _Current_ | SQLite | ``/.TagStudio/ts_library.sqlite | +| Used From | Used Until | Format | Location | +| ---------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [74383e3](https://github.com/TagStudioDev/TagStudio/commit/74383e3c3c12f72be1481ab0b86c7360b95c2d85) | [v9.5.4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.4) | SQLite | ``/.TagStudio/ts_library.sqlite | - Introduces built-in minor versioning - The version number divided by 100 (and floored) constitutes the **major** version. Major version indicate breaking changes that prevent libraries from being opened in TagStudio versions older than the ones they were created in. - Values more precise than this ("ones" through "tens" columns) constitute the **minor** version. These indicate minor changes that don't prevent a newer library from being opened in an older version of TagStudio, as long as the major version is not also increased. - Swaps `parent_id` and `child_id` values in the `tag_parents` table, which have erroneously been flipped since the first SQLite DB version. + +#### Version 101 + +| Used From | Used Until | Format | Location | +| ----------------------------------------------------------------------- | ---------- | ------ | ----------------------------------------------- | +| [v9.5.4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.4) | _Current_ | SQLite | ``/.TagStudio/ts_library.sqlite | + +- Deprecates the `preferences` table, set to be removed in a future TagStudio version. +- Introduces the `versions` table + - Has a string `key` column and an int `value` column + - The `key` column stores one of two values: `'INITIAL'` and `'CURRENT'` + - `'INITIAL'` stores the database version number in which in was created + - Pre-existing databases set this number to `100` + - `'CURRENT'` stores the current database version number diff --git a/src/tagstudio/core/library/alchemy/constants.py b/src/tagstudio/core/library/alchemy/constants.py new file mode 100644 index 000000000..2e575359e --- /dev/null +++ b/src/tagstudio/core/library/alchemy/constants.py @@ -0,0 +1,18 @@ +from sqlalchemy import text + +DB_VERSION_LEGACY_KEY: str = "DB_VERSION" +DB_VERSION_CURRENT_KEY: str = "CURRENT" +DB_VERSION_INITIAL_KEY: str = "INITIAL" +DB_VERSION: int = 101 + +TAG_CHILDREN_QUERY = text(""" +-- Note for this entire query that tag_parents.child_id is the parent id and tag_parents.parent_id is the child id due to bad naming +WITH RECURSIVE ChildTags AS ( + SELECT :tag_id AS child_id + UNION + SELECT tp.parent_id AS child_id + FROM tag_parents tp + INNER JOIN ChildTags c ON tp.child_id = c.child_id +) +SELECT * FROM ChildTags; +""") # noqa: E501 diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 08d9f66ef..8be1087b4 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -16,6 +16,7 @@ from uuid import uuid4 from warnings import catch_warnings +import sqlalchemy import structlog from humanfriendly import format_timespan from sqlalchemy import ( @@ -58,6 +59,13 @@ ) from tagstudio.core.enums import LibraryPrefs from tagstudio.core.library.alchemy import default_color_groups +from tagstudio.core.library.alchemy.constants import ( + DB_VERSION, + DB_VERSION_CURRENT_KEY, + DB_VERSION_INITIAL_KEY, + DB_VERSION_LEGACY_KEY, + TAG_CHILDREN_QUERY, +) from tagstudio.core.library.alchemy.db import make_tables from tagstudio.core.library.alchemy.enums import ( MAX_SQL_VARIABLES, @@ -81,6 +89,7 @@ TagAlias, TagColorGroup, ValueType, + Version, ) from tagstudio.core.library.alchemy.visitors import SQLBoolExpressionBuilder from tagstudio.core.library.json.library import Library as JsonLibrary @@ -92,21 +101,6 @@ logger = structlog.get_logger(__name__) -DB_VERSION_KEY: str = "DB_VERSION" -DB_VERSION: int = 100 - -TAG_CHILDREN_QUERY = text(""" --- Note for this entire query that tag_parents.child_id is the parent id and tag_parents.parent_id is the child id due to bad naming -WITH RECURSIVE ChildTags AS ( - SELECT :tag_id AS child_id - UNION - SELECT tp.parent_id AS child_id - FROM tag_parents tp - INNER JOIN ChildTags c ON tp.child_id = c.child_id -) -SELECT * FROM ChildTags; -""") # noqa: E501 - class ReservedNamespaceError(Exception): """Raise during an unauthorized attempt to create or modify a reserved namespace value. @@ -376,14 +370,9 @@ def open_sqlite_library(self, library_dir: Path, is_new: bool) -> LibraryStatus: ) self.engine = create_engine(connection_string, poolclass=poolclass) with Session(self.engine) as session: - # dont check db version when creating new library + # Don't check DB version when creating new library if not is_new: - db_result = session.scalar( - select(Preferences).where(Preferences.key == DB_VERSION_KEY) - ) - if db_result: - assert isinstance(db_result.value, int) - loaded_db_version = db_result.value + loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) # ======================== Library Database Version Checking ======================= # DB_VERSION 6 is the first supported SQLite DB version. @@ -444,22 +433,37 @@ def open_sqlite_library(self, library_dir: Path, is_new: bool) -> LibraryStatus: except IntegrityError: session.rollback() - # TODO: Completely rework this "preferences" system. + # Ensure version rows are present with catch_warnings(record=True): + # NOTE: The "Preferences" table is depreciated and will be removed in the future. + # The DB_VERSION is still being set to it in order to remain backwards-compatible + # with existing TagStudio versions until it is removed. + try: + session.add(Preferences(key=DB_VERSION_LEGACY_KEY, value=DB_VERSION)) + session.commit() + except IntegrityError: + session.rollback() + try: - session.add(Preferences(key=DB_VERSION_KEY, value=DB_VERSION)) + initial = DB_VERSION if is_new else 100 + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=initial)) session.commit() except IntegrityError: - logger.debug("preference already exists", pref=DB_VERSION_KEY) session.rollback() + try: + session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) + session.commit() + except IntegrityError: + session.rollback() + + # TODO: Remove this "Preferences" system. for pref in LibraryPrefs: with catch_warnings(record=True): try: session.add(Preferences(key=pref.name, value=pref.default)) session.commit() except IntegrityError: - logger.debug("preference already exists", pref=pref) session.rollback() for field in _FieldID: @@ -500,29 +504,29 @@ def open_sqlite_library(self, library_dir: Path, is_new: bool) -> LibraryStatus: # schema changes first if loaded_db_version < 8: - self.apply_db8_schema_changes(session) + self.__apply_db8_schema_changes(session) if loaded_db_version < 9: - self.apply_db9_schema_changes(session) + self.__apply_db9_schema_changes(session) # now the data changes if loaded_db_version == 6: - self.apply_repairs_for_db6(session) + self.__apply_repairs_for_db6(session) if loaded_db_version >= 6 and loaded_db_version < 8: - self.apply_db8_default_data(session) + self.__apply_db8_default_data(session) if loaded_db_version < 9: - self.apply_db9_filename_population(session) + self.__apply_db9_filename_population(session) if loaded_db_version < 100: - self.apply_db100_parent_repairs(session) + self.__apply_db100_parent_repairs(session) # Update DB_VERSION if loaded_db_version < DB_VERSION: - self.set_prefs(DB_VERSION_KEY, DB_VERSION) + self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) # everything is fine, set the library path self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def apply_repairs_for_db6(self, session: Session): + def __apply_repairs_for_db6(self, session: Session): """Apply database repairs introduced in DB_VERSION 7.""" logger.info("[Library][Migration] Applying patches to DB_VERSION: 6 library...") with session: @@ -543,11 +547,9 @@ def apply_repairs_for_db6(self, session: Session): .values(disambiguation_id=None) ) session.execute(disam_stmt) - session.flush() - session.commit() - def apply_db8_schema_changes(self, session: Session): + def __apply_db8_schema_changes(self, session: Session): """Apply database schema changes introduced in DB_VERSION 8.""" # TODO: Use Alembic for this part instead # Add the missing color_border column to the TagColorGroups table. @@ -565,7 +567,7 @@ def apply_db8_schema_changes(self, session: Session): ) session.rollback() - def apply_db8_default_data(self, session: Session): + def __apply_db8_default_data(self, session: Session): """Apply default data changes introduced in DB_VERSION 8.""" tag_colors: list[TagColorGroup] = default_color_groups.standard() tag_colors += default_color_groups.pastels() @@ -615,7 +617,7 @@ def apply_db8_default_data(self, session: Session): ) session.rollback() - def apply_db9_schema_changes(self, session: Session): + def __apply_db9_schema_changes(self, session: Session): """Apply database schema changes introduced in DB_VERSION 9.""" add_filename_column = text( "ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''" @@ -631,16 +633,15 @@ def apply_db9_schema_changes(self, session: Session): ) session.rollback() - def apply_db9_filename_population(self, session: Session): + def __apply_db9_filename_population(self, session: Session): """Populate the filename column introduced in DB_VERSION 9.""" for entry in self.all_entries(): session.merge(entry).filename = entry.path.name session.commit() logger.info("[Library][Migration] Populated filename column in entries table") - def apply_db100_parent_repairs(self, session: Session): - """Apply database repairs introduced in DB_VERSION 100.""" - logger.info("[Library][Migration] Applying patches to DB_VERSION 100 library...") + def __apply_db100_parent_repairs(self, session: Session): + """Swap the child_id and parent_id values in the TagParent table.""" with session: # Repair parent-child tag relationships that are the wrong way around. stmt = update(TagParent).values( @@ -648,9 +649,8 @@ def apply_db100_parent_repairs(self, session: Session): child_id=TagParent.parent_id, ) session.execute(stmt) - session.flush() - session.commit() + logger.info("[Library][Migration] Refactored TagParent column") @property def default_fields(self) -> list[BaseField]: @@ -1685,6 +1685,62 @@ def update_parent_tags(self, tag: Tag, parent_ids: list[int] | set[int], session ) session.add(parent_tag) + def get_version(self, key: str) -> int: + """Get a version value from the DB. + + Args: + key(str): The key for the name of the version type to set. + """ + with Session(self.engine) as session: + engine = sqlalchemy.inspect(self.engine) + try: + # "Version" table added in DB_VERSION 101 + if engine and engine.has_table("Version"): + version = session.scalar(select(Version).where(Version.key == key)) + assert version + return version.value + # NOTE: The "Preferences" table has been depreciated as of TagStudio 9.5.4 + # and is set to be removed in a future release. + else: + pref_version = session.scalar( + select(Preferences).where(Preferences.key == DB_VERSION_LEGACY_KEY) + ) + assert pref_version + assert isinstance(pref_version.value, int) + return pref_version.value + except Exception: + return 0 + + def set_version(self, key: str, value: int) -> None: + """Set a version value to the DB. + + Args: + key(str): The key for the name of the version type to set. + value(int): The version value to set. + """ + with Session(self.engine) as session: + try: + version = session.scalar(select(Version).where(Version.key == key)) + assert version + version.value = value + session.add(version) + session.commit() + + # If a depreciated "Preferences" table is found, update the version value to be read + # by older TagStudio versions. + engine = sqlalchemy.inspect(self.engine) + if engine and engine.has_table("Preferences"): + pref = session.scalar( + select(Preferences).where(Preferences.key == DB_VERSION_LEGACY_KEY) + ) + assert pref is not None + pref.value = value # pyright: ignore + session.add(pref) + session.commit() + except (IntegrityError, AssertionError) as e: + logger.error("[Library][ERROR] Couldn't add default tag color namespaces", error=e) + session.rollback() + def prefs(self, key: str | LibraryPrefs): # load given item from Preferences table with Session(self.engine) as session: diff --git a/src/tagstudio/core/library/alchemy/models.py b/src/tagstudio/core/library/alchemy/models.py index cd63f4ab0..bfd3ac36b 100644 --- a/src/tagstudio/core/library/alchemy/models.py +++ b/src/tagstudio/core/library/alchemy/models.py @@ -311,8 +311,17 @@ def slugify_field_key(mapper, connection, target): target.key = slugify(target.tag) +# NOTE: The "Preferences" table has been depreciated as of TagStudio 9.5.4 +# and is set to be removed in a future release. class Preferences(Base): __tablename__ = "preferences" key: Mapped[str] = mapped_column(primary_key=True) value: Mapped[dict] = mapped_column(JSON, nullable=False) + + +class Version(Base): + __tablename__ = "versions" + + key: Mapped[str] = mapped_column(primary_key=True) + value: Mapped[int] = mapped_column(nullable=False, default=0) From ad6453216fd823acc145c52571a51fb9bfb71022 Mon Sep 17 00:00:00 2001 From: Travis Abendshien <46939827+CyanVoxel@users.noreply.github.com> Date: Mon, 25 Aug 2025 00:54:16 -0700 Subject: [PATCH 2/5] tests: update search_library db file --- .../.TagStudio/ts_library.sqlite | Bin 98304 -> 114688 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/fixtures/search_library/.TagStudio/ts_library.sqlite b/tests/fixtures/search_library/.TagStudio/ts_library.sqlite index 0f0a9154536ed64b5d4ece35cedc316be3be67ee..fcb8e8cc5b24f1f5f395bbcd1cd8d9a8c07bf77e 100644 GIT binary patch delta 396 zcmZo@U~6b#pCB!GiGhJZ28dyRZ=#N|@FfPlvTjzO5F`I@kc15X?}u&-~9e3qH8Tug1;9EY0W~8WiN}7m~`& z2;_MBd4_m8`lN7y)xKolf64z9sP-?Z=2TQ`{d?o`Bi&Xi&d5<%eQ)jcHfB+|w!EP|)kPNPuU>BE_Wo!&ANleN~ zg(-m1lFmV{jv=lJA&yQyt_o1m$-O+Pla&}1Ctu* Date: Mon, 25 Aug 2025 01:11:53 -0700 Subject: [PATCH 3/5] chore: add copyright info to library constants.py --- src/tagstudio/core/library/alchemy/constants.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tagstudio/core/library/alchemy/constants.py b/src/tagstudio/core/library/alchemy/constants.py index 2e575359e..53a5218ab 100644 --- a/src/tagstudio/core/library/alchemy/constants.py +++ b/src/tagstudio/core/library/alchemy/constants.py @@ -1,3 +1,8 @@ +# Copyright (C) 2025 +# Licensed under the GPL-3.0 License. +# Created for TagStudio: https://github.com/CyanVoxel/TagStudio + + from sqlalchemy import text DB_VERSION_LEGACY_KEY: str = "DB_VERSION" From c29425ad1ec92b6ed010470ec5409571466a4635 Mon Sep 17 00:00:00 2001 From: Travis Abendshien <46939827+CyanVoxel@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:18:50 -0700 Subject: [PATCH 4/5] fix: only backup db if loaded version is lower --- src/tagstudio/core/library/alchemy/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 8be1087b4..3c429eda2 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -497,7 +497,7 @@ def open_sqlite_library(self, library_dir: Path, is_new: bool) -> LibraryStatus: # Apply any post-SQL migration patches. if not is_new: # save backup if patches will be applied - if loaded_db_version != DB_VERSION: + if loaded_db_version < DB_VERSION: self.library_dir = library_dir self.save_library_backup_to_disk() self.library_dir = None From a8eca270c513fef5d2f21d5a1d54b68237724040 Mon Sep 17 00:00:00 2001 From: Travis Abendshien <46939827+CyanVoxel@users.noreply.github.com> Date: Thu, 28 Aug 2025 12:39:36 -0700 Subject: [PATCH 5/5] chore: mark Preferences as @deprecated --- src/tagstudio/core/library/alchemy/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tagstudio/core/library/alchemy/models.py b/src/tagstudio/core/library/alchemy/models.py index bfd3ac36b..a1fc92e7b 100644 --- a/src/tagstudio/core/library/alchemy/models.py +++ b/src/tagstudio/core/library/alchemy/models.py @@ -7,6 +7,7 @@ from sqlalchemy import JSON, ForeignKey, ForeignKeyConstraint, Integer, event from sqlalchemy.orm import Mapped, mapped_column, relationship +from typing_extensions import deprecated from tagstudio.core.constants import TAG_ARCHIVED, TAG_FAVORITE from tagstudio.core.library.alchemy.db import Base, PathType @@ -313,6 +314,7 @@ def slugify_field_key(mapper, connection, target): # NOTE: The "Preferences" table has been depreciated as of TagStudio 9.5.4 # and is set to be removed in a future release. +@deprecated("Use `Version` for storing version, and `ts_ignore` system for file exclusion.") class Preferences(Base): __tablename__ = "preferences"