diff --git a/docs/updates/schema_changes.md b/docs/updates/schema_changes.md index 16eb4194b..2d721d8be 100644 --- a/docs/updates/schema_changes.md +++ b/docs/updates/schema_changes.md @@ -14,9 +14,9 @@ Legacy (JSON) library save format versions were tied to the release version of t ### Versions 1.0.0 - 9.4.2 -| Used From | Used Until | Format | Location | -| --------- | ----------------------------------------------------------------------- | ------ | --------------------------------------------- | -| v1.0.0 | [v9.4.2](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.4.2) | JSON | ``/.TagStudio/ts_library.json | +| Used From | Format | Location | +| --------- | ------ | --------------------------------------------- | +| v1.0.0 | JSON | ``/.TagStudio/ts_library.json | The legacy database format for public TagStudio releases [v9.1](https://github.com/TagStudioDev/TagStudio/tree/Alpha-v9.1) through [v9.4.2](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.4.2). Variations of this format had been used privately since v1.0.0. @@ -48,9 +48,9 @@ These versions were used while developing the new SQLite file format, outside an ### Version 6 -| Used From | Used Until | Format | Location | -| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | ------ | ----------------------------------------------- | -| [v9.5.0-pr1](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr1) | [v9.5.0-pr1](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr1) | SQLite | ``/.TagStudio/ts_library.sqlite | +| Used From | Format | Location | +| ------------------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [v9.5.0-pr1](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr1) | SQLite | ``/.TagStudio/ts_library.sqlite | The first public version of the SQLite save file format. @@ -60,9 +60,9 @@ Migration from the legacy JSON format is provided via a walkthrough when opening ### Version 7 -| Used From | Used Until | Format | Location | -| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | ------ | ----------------------------------------------- | -| [v9.5.0-pr2](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr2) | [v9.5.0-pr3](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr3) | SQLite | ``/.TagStudio/ts_library.sqlite | +| Used From | Format | Location | +| ------------------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [v9.5.0-pr2](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr2) | SQLite | ``/.TagStudio/ts_library.sqlite | - Repairs "Description" fields to use a TEXT_LINE key instead of a TEXT_BOX key. - Repairs tags that may have a disambiguation_id pointing towards a deleted tag. @@ -71,9 +71,9 @@ Migration from the legacy JSON format is provided via a walkthrough when opening ### Version 8 -| Used From | Used Until | Format | Location | -| ------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ------ | ----------------------------------------------- | -| [v9.5.0-pr4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr4) | [v9.5.1](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.1) | SQLite | ``/.TagStudio/ts_library.sqlite | +| Used From | Format | Location | +| ------------------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [v9.5.0-pr4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.0-pr4) | SQLite | ``/.TagStudio/ts_library.sqlite | - Adds the `color_border` column to the `tag_colors` table. Used for instructing the [secondary color](../library/tag_color.md#secondary-color) to apply to a tag's border as a new optional behavior. - Adds three new default colors: "Burgundy (TagStudio Shades)", "Dark Teal (TagStudio Shades)", and "Dark Lavender (TagStudio Shades)". @@ -83,9 +83,9 @@ Migration from the legacy JSON format is provided via a walkthrough when opening ### Version 9 -| Used From | Used Until | Format | Location | -| ----------------------------------------------------------------------- | ----------------------------------------------------------------------- | ------ | ----------------------------------------------- | -| [v9.5.2](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.2) | [v9.5.3](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.3) | SQLite | ``/.TagStudio/ts_library.sqlite | +| Used From | Format | Location | +| ----------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [v9.5.2](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.2) | SQLite | ``/.TagStudio/ts_library.sqlite | - Adds the `filename` column to the `entries` table. Used for sorting entries by filename in search results. @@ -93,20 +93,20 @@ Migration from the legacy JSON format is provided via a walkthrough when opening ### Version 100 -| Used From | Used Until | Format | Location | -| ---------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | ------ | ----------------------------------------------- | -| [74383e3](https://github.com/TagStudioDev/TagStudio/commit/74383e3c3c12f72be1481ab0b86c7360b95c2d85) | [v9.5.4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.4) | SQLite | ``/.TagStudio/ts_library.sqlite | +| Used From | Format | Location | +| ---------------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [74383e3](https://github.com/TagStudioDev/TagStudio/commit/74383e3c3c12f72be1481ab0b86c7360b95c2d85) | SQLite | ``/.TagStudio/ts_library.sqlite | - Introduces built-in minor versioning - The version number divided by 100 (and floored) constitutes the **major** version. Major version indicate breaking changes that prevent libraries from being opened in TagStudio versions older than the ones they were created in. - Values more precise than this ("ones" through "tens" columns) constitute the **minor** version. These indicate minor changes that don't prevent a newer library from being opened in an older version of TagStudio, as long as the major version is not also increased. -- Swaps `parent_id` and `child_id` values in the `tag_parents` table, which have erroneously been flipped since the first SQLite DB version. +- Swaps `parent_id` and `child_id` values in the `tag_parents` table #### Version 101 -| Used From | Used Until | Format | Location | -| ----------------------------------------------------------------------- | ---------- | ------ | ----------------------------------------------- | -| [v9.5.4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.4) | _Current_ | SQLite | ``/.TagStudio/ts_library.sqlite | +| Used From | Format | Location | +| ----------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [v9.5.4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.4) | SQLite | ``/.TagStudio/ts_library.sqlite | - Deprecates the `preferences` table, set to be removed in a future TagStudio version. - Introduces the `versions` table @@ -115,3 +115,11 @@ Migration from the legacy JSON format is provided via a walkthrough when opening - `'INITIAL'` stores the database version number in which in was created - Pre-existing databases set this number to `100` - `'CURRENT'` stores the current database version number + +#### Version 102 + +| Used From | Format | Location | +| ----------------------------------------------------------------------- | ------ | ----------------------------------------------- | +| [v9.5.4](https://github.com/TagStudioDev/TagStudio/releases/tag/v9.5.4) | SQLite | ``/.TagStudio/ts_library.sqlite | + +- Applies repairs to the `tag_parents` table created in [version 100](#version-100), removing rows that reference tags that have been deleted. diff --git a/src/tagstudio/core/library/alchemy/constants.py b/src/tagstudio/core/library/alchemy/constants.py index 2032d613a..83aab71b0 100644 --- a/src/tagstudio/core/library/alchemy/constants.py +++ b/src/tagstudio/core/library/alchemy/constants.py @@ -11,7 +11,7 @@ DB_VERSION_LEGACY_KEY: str = "DB_VERSION" DB_VERSION_CURRENT_KEY: str = "CURRENT" DB_VERSION_INITIAL_KEY: str = "INITIAL" -DB_VERSION: int = 101 +DB_VERSION: int = 102 TAG_CHILDREN_QUERY = text(""" WITH RECURSIVE ChildTags AS ( diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index c24643b23..64aeac4f6 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -11,7 +11,7 @@ import shutil import time import unicodedata -from collections.abc import Iterable, Iterator, MutableSequence +from collections.abc import Iterable, Iterator from dataclasses import dataclass from datetime import UTC, datetime from os import makedirs @@ -534,21 +534,23 @@ def open_sqlite_library(self, library_dir: Path, is_new: bool) -> LibraryStatus: self.save_library_backup_to_disk() self.library_dir = None - # schema changes first + # NOTE: Depending on the data, some data and schema changes need to be applied in + # different orders. This chain of methods can likely be cleaned up and/or moved. if loaded_db_version < 8: self.__apply_db8_schema_changes(session) if loaded_db_version < 9: self.__apply_db9_schema_changes(session) - - # now the data changes if loaded_db_version == 6: self.__apply_repairs_for_db6(session) + if loaded_db_version >= 6 and loaded_db_version < 8: self.__apply_db8_default_data(session) if loaded_db_version < 9: self.__apply_db9_filename_population(session) if loaded_db_version < 100: self.__apply_db100_parent_repairs(session) + if loaded_db_version < 102: + self.__apply_db102_repairs(session) # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist self.migrate_sql_to_ts_ignore(library_dir) @@ -566,12 +568,12 @@ def __apply_repairs_for_db6(self, session: Session): logger.info("[Library][Migration] Applying patches to DB_VERSION: 6 library...") with session: # Repair "Description" fields with a TEXT_LINE key instead of a TEXT_BOX key. - desc_stmd = ( + desc_stmt = ( update(ValueType) .where(ValueType.key == FieldID.DESCRIPTION.name) .values(type=FieldTypeEnum.TEXT_BOX.name) ) - session.execute(desc_stmd) + session.execute(desc_stmt) session.flush() # Repair tags that may have a disambiguation_id pointing towards a deleted tag. @@ -685,7 +687,16 @@ def __apply_db100_parent_repairs(self, session: Session): ) session.execute(stmt) session.commit() - logger.info("[Library][Migration] Refactored TagParent column") + logger.info("[Library][Migration] Refactored TagParent table") + + def __apply_db102_repairs(self, session: Session): + """Repair tag_parents rows with references to deleted tags.""" + with session: + all_tag_ids: list[int] = [t.id for t in self.tags] + stmt = delete(TagParent).where(TagParent.parent_id.not_in(all_tag_ids)) + session.execute(stmt) + session.commit() + logger.info("[Library][Migration] Verified TagParent table data") def migrate_sql_to_ts_ignore(self, library_dir: Path): # Do not continue if existing '.ts_ignore' file is found @@ -794,7 +805,7 @@ def get_entries(self, entry_ids: Iterable[int]) -> list[Entry]: entries = dict((e.id, e) for e in session.scalars(statement)) return [entries[id] for id in entry_ids] - def get_entries_full(self, entry_ids: MutableSequence[int]) -> Iterator[Entry]: + def get_entries_full(self, entry_ids: list[int] | set[int]) -> Iterator[Entry]: """Load entry and join with all joins and all tags.""" with Session(self.engine) as session: statement = select(Entry).where(Entry.id.in_(set(entry_ids))) @@ -1112,17 +1123,17 @@ def update_entry_path(self, entry_id: int | Entry, path: Path) -> bool: def remove_tag(self, tag_id: int): with Session(self.engine, expire_on_commit=False) as session: try: - child_tags = session.scalars( - select(TagParent).where(TagParent.child_id == tag_id) - ).all() aliases = session.scalars(select(TagAlias).where(TagAlias.tag_id == tag_id)) - - for alias in aliases or []: + for alias in aliases: session.delete(alias) + session.flush() - for child_tag in child_tags or []: - session.delete(child_tag) - session.expunge(child_tag) + tag_parents = session.scalars( + select(TagParent).where(TagParent.parent_id == tag_id) + ).all() + for tag_parent in tag_parents: + session.delete(tag_parent) + session.flush() disam_stmt = ( update(Tag) @@ -1131,6 +1142,7 @@ def remove_tag(self, tag_id: int): ) session.execute(disam_stmt) session.flush() + session.query(Tag).filter_by(id=tag_id).delete() session.commit() @@ -1397,9 +1409,9 @@ def delete_namespace(self, namespace: Namespace | str): def add_tag( self, tag: Tag, - parent_ids: MutableSequence[int] | None = None, - alias_names: MutableSequence[str] | None = None, - alias_ids: MutableSequence[int] | None = None, + parent_ids: list[int] | set[int] | None = None, + alias_names: list[str] | set[str] | None = None, + alias_ids: list[int] | set[int] | None = None, ) -> Tag | None: with Session(self.engine, expire_on_commit=False) as session: try: @@ -1422,7 +1434,7 @@ def add_tag( return None def add_tags_to_entries( - self, entry_ids: int | list[int], tag_ids: int | MutableSequence[int] + self, entry_ids: int | list[int] | set[int], tag_ids: int | list[int] | set[int] ) -> int: """Add one or more tags to one or more entries. @@ -1451,7 +1463,7 @@ def add_tags_to_entries( return total_added def remove_tags_from_entries( - self, entry_ids: int | list[int], tag_ids: int | MutableSequence[int] + self, entry_ids: int | list[int] | set[int], tag_ids: int | list[int] | set[int] ) -> bool: """Remove one or more tags from one or more entries.""" entry_ids_ = [entry_ids] if isinstance(entry_ids, int) else entry_ids @@ -1604,16 +1616,22 @@ def get_tag_hierarchy(self, tag_ids: Iterable[int]) -> dict[int, Tag]: for tag in tags: all_tags[tag.id] = tag for tag in all_tags.values(): - # Sqlalchemy tracks this as a change to the parent_tags field - tag.parent_tags = {all_tags[p] for p in all_tag_parents.get(tag.id, [])} - # When calling session.add with this tag instance sqlalchemy will - # attempt to create TagParents that already exist. - - state: InstanceState[Tag] = inspect(tag) - # Prevent sqlalchemy from thinking any fields are different from what's committed - # committed_state contains original values for fields that have changed. - # empty when no fields have changed - state.committed_state.clear() + try: + # Sqlalchemy tracks this as a change to the parent_tags field + tag.parent_tags = {all_tags[p] for p in all_tag_parents.get(tag.id, [])} + # When calling session.add with this tag instance sqlalchemy will + # attempt to create TagParents that already exist. + + state: InstanceState[Tag] = inspect(tag) + # Prevent sqlalchemy from thinking fields are different from what's committed + # committed_state contains original values for fields that have changed. + # empty when no fields have changed + state.committed_state.clear() + except KeyError as e: + logger.error( + "[LIBRARY][get_tag_hierarchy] Tag referenced by TagParent does not exist!", + error=e, + ) return all_tags @@ -1669,9 +1687,9 @@ def remove_parent_tag(self, base_id: int, remove_tag_id: int) -> bool: def update_tag( self, tag: Tag, - parent_ids: MutableSequence[int] | None = None, - alias_names: MutableSequence[str] | None = None, - alias_ids: MutableSequence[int] | None = None, + parent_ids: list[int] | set[int] | None = None, + alias_names: list[str] | set[str] | None = None, + alias_ids: list[int] | set[int] | None = None, ) -> None: """Edit a Tag in the Library.""" self.add_tag(tag, parent_ids, alias_names, alias_ids) @@ -1728,8 +1746,8 @@ def update_color(self, old_color_group: TagColorGroup, new_color_group: TagColor def update_aliases( self, tag: Tag, - alias_ids: MutableSequence[int], - alias_names: MutableSequence[str], + alias_ids: list[int] | set[int], + alias_names: list[str] | set[str], session: Session, ): prev_aliases = session.scalars(select(TagAlias).where(TagAlias.tag_id == tag.id)).all() @@ -1745,7 +1763,7 @@ def update_aliases( alias = TagAlias(alias_name, tag.id) session.add(alias) - def update_parent_tags(self, tag: Tag, parent_ids: MutableSequence[int], session: Session): + def update_parent_tags(self, tag: Tag, parent_ids: list[int] | set[int], session: Session): if tag.id in parent_ids: parent_ids.remove(tag.id) diff --git a/src/tagstudio/qt/mixed/tag_search.py b/src/tagstudio/qt/mixed/tag_search.py index 13ca4b0d3..53990c378 100644 --- a/src/tagstudio/qt/mixed/tag_search.py +++ b/src/tagstudio/qt/mixed/tag_search.py @@ -197,7 +197,8 @@ def on_tag_modal_saved(): self.search_field.setFocus() self.update_tags() - from tagstudio.qt.modals.build_tag import BuildTagPanel # here due to circular imports + # TODO: Move this to a top-level import + from tagstudio.qt.mixed.build_tag import BuildTagPanel # here due to circular imports self.build_tag_modal: BuildTagPanel = BuildTagPanel(self.lib) self.add_tag_modal: PanelModal = PanelModal( @@ -375,7 +376,8 @@ def delete_tag(self, tag: Tag): pass def edit_tag(self, tag: Tag): - from tagstudio.qt.modals.build_tag import BuildTagPanel + # TODO: Move this to a top-level import + from tagstudio.qt.mixed.build_tag import BuildTagPanel def callback(btp: BuildTagPanel): self.lib.update_tag(