From f07be065d2dbd73bfc1730681dd3060ce073f858 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Mon, 9 Sep 2024 17:48:48 +0800 Subject: [PATCH] feat(kobo): sync On Deck as a Kobo collection --- .../V20240909112827__syncpoint_readlists.sql | 34 +++ .../gotson/komga/domain/model/SyncPoint.kt | 19 ++ .../domain/persistence/SyncPointRepository.kt | 44 ++++ .../domain/service/SyncPointLifecycle.kt | 60 ++++- .../infrastructure/jooq/main/SyncPointDao.kt | 243 ++++++++++++++++-- .../interfaces/api/kobo/KoboController.kt | 94 ++++++- .../interfaces/api/kobo/dto/SyncResultDto.kt | 9 +- .../komga/interfaces/api/kobo/dto/TagDto.kt | 18 ++ .../interfaces/api/kobo/dto/TagTypeDto.kt | 7 +- .../domain/service/SyncPointLifecycleTest.kt | 97 ++++++- 10 files changed, 569 insertions(+), 56 deletions(-) create mode 100644 komga/src/flyway/resources/db/migration/sqlite/V20240909112827__syncpoint_readlists.sql diff --git a/komga/src/flyway/resources/db/migration/sqlite/V20240909112827__syncpoint_readlists.sql b/komga/src/flyway/resources/db/migration/sqlite/V20240909112827__syncpoint_readlists.sql new file mode 100644 index 0000000000..d56d9d08f0 --- /dev/null +++ b/komga/src/flyway/resources/db/migration/sqlite/V20240909112827__syncpoint_readlists.sql @@ -0,0 +1,34 @@ +CREATE TABLE SYNC_POINT_READLIST +( + SYNC_POINT_ID varchar NOT NULL, + READLIST_ID varchar NOT NULL, + READLIST_NAME varchar NOT NULL, + READLIST_CREATED_DATE datetime NOT NULL, + READLIST_LAST_MODIFIED_DATE datetime NOT NULL, + SYNCED boolean NOT NULL default false, + PRIMARY KEY (SYNC_POINT_ID, READLIST_ID), + FOREIGN KEY (SYNC_POINT_ID) REFERENCES SYNC_POINT (ID) +); + +create index if not exists idx__sync_point_readlist__sync_point_id + on SYNC_POINT_READLIST (SYNC_POINT_ID); + +CREATE TABLE SYNC_POINT_READLIST_BOOK +( + SYNC_POINT_ID varchar NOT NULL, + READLIST_ID varchar NOT NULL, + BOOK_ID varchar NOT NULL, + PRIMARY KEY (SYNC_POINT_ID, READLIST_ID, BOOK_ID), + FOREIGN KEY (SYNC_POINT_ID) REFERENCES SYNC_POINT (ID) +); + +create index if not exists idx__sync_point_readlist_book__sync_point_id_readlist_id + on SYNC_POINT_READLIST_BOOK (SYNC_POINT_ID, READLIST_ID); + +CREATE TABLE SYNC_POINT_READLIST_REMOVED_SYNCED +( + SYNC_POINT_ID varchar NOT NULL, + READLIST_ID varchar NOT NULL, + PRIMARY KEY (SYNC_POINT_ID, READLIST_ID), + FOREIGN KEY (SYNC_POINT_ID) REFERENCES SYNC_POINT (ID) +); diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/SyncPoint.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/SyncPoint.kt index 38fb209cdd..3f4c40b7ed 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/SyncPoint.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/SyncPoint.kt @@ -19,4 +19,23 @@ data class SyncPoint( val metadataLastModifiedDate: ZonedDateTime, val synced: Boolean, ) + + data class ReadList( + val syncPointId: String, + val readListId: String, + val readListName: String, + val createdDate: ZonedDateTime, + val lastModifiedDate: ZonedDateTime, + val synced: Boolean, + ) { + companion object { + const val ON_DECK_ID = "KOMGA-ONDECK" + } + + data class Book( + val syncPointId: String, + val readListId: String, + val bookId: String, + ) + } } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SyncPointRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SyncPointRepository.kt index a2041b65b9..a944cdcee4 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SyncPointRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SyncPointRepository.kt @@ -13,6 +13,12 @@ interface SyncPointRepository { search: BookSearch, ): SyncPoint + fun addOnDeck( + syncPointId: String, + user: KomgaUser, + filterOnLibraryIds: Collection?, + ) + fun findByIdOrNull(syncPointId: String): SyncPoint? fun findBooksById( @@ -49,12 +55,50 @@ interface SyncPointRepository { pageable: Pageable, ): Page + fun findReadListsById( + syncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page + + fun findReadListsAdded( + fromSyncPointId: String, + toSyncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page + + fun findReadListsChanged( + fromSyncPointId: String, + toSyncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page + + fun findReadListsRemoved( + fromSyncPointId: String, + toSyncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page + + fun findBookIdsByReadListIds( + syncPointId: String, + readListIds: Collection, + ): List + fun markBooksSynced( syncPointId: String, forRemovedBooks: Boolean, bookIds: Collection, ) + fun markReadListsSynced( + syncPointId: String, + forRemovedReadLists: Boolean, + readListIds: Collection, + ) + fun deleteByUserId(userId: String) fun deleteByUserIdAndApiKeyIds( diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/SyncPointLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/SyncPointLifecycle.kt index 9db08e0812..d72592be6d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/SyncPointLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/SyncPointLifecycle.kt @@ -18,17 +18,24 @@ class SyncPointLifecycle( user: KomgaUser, apiKeyId: String?, libraryIds: List?, - ): SyncPoint = - syncPointRepository.create( - user, - apiKeyId, - BookSearch( - libraryIds = user.getAuthorizedLibraryIds(libraryIds), - mediaStatus = setOf(Media.Status.READY), - mediaProfile = listOf(MediaProfile.EPUB), - deleted = false, - ), - ) + ): SyncPoint { + val authorizedLibraryIds = user.getAuthorizedLibraryIds(libraryIds) + val syncPoint = + syncPointRepository.create( + user, + apiKeyId, + BookSearch( + libraryIds = authorizedLibraryIds, + mediaStatus = setOf(Media.Status.READY), + mediaProfile = listOf(MediaProfile.EPUB), + deleted = false, + ), + ) + + syncPointRepository.addOnDeck(syncPoint.id, user, authorizedLibraryIds) + + return syncPoint + } /** * Retrieve a page of un-synced books and mark them as synced. @@ -83,4 +90,35 @@ class SyncPointLifecycle( ): Page = syncPointRepository.findBooksReadProgressChanged(fromSyncPointId, toSyncPointId, true, pageable) .also { page -> syncPointRepository.markBooksSynced(toSyncPointId, false, page.content.map { it.bookId }) } + + fun takeReadLists( + toSyncPointId: String, + pageable: Pageable, + ): Page = + syncPointRepository.findReadListsById(toSyncPointId, true, pageable) + .also { page -> syncPointRepository.markReadListsSynced(toSyncPointId, false, page.content.map { it.readListId }) } + + fun takeReadListsAdded( + fromSyncPointId: String, + toSyncPointId: String, + pageable: Pageable, + ): Page = + syncPointRepository.findReadListsAdded(fromSyncPointId, toSyncPointId, true, pageable) + .also { page -> syncPointRepository.markReadListsSynced(toSyncPointId, false, page.content.map { it.readListId }) } + + fun takeReadListsChanged( + fromSyncPointId: String, + toSyncPointId: String, + pageable: Pageable, + ): Page = + syncPointRepository.findReadListsChanged(fromSyncPointId, toSyncPointId, true, pageable) + .also { page -> syncPointRepository.markReadListsSynced(toSyncPointId, false, page.content.map { it.readListId }) } + + fun takeReadListsRemoved( + fromSyncPointId: String, + toSyncPointId: String, + pageable: Pageable, + ): Page = + syncPointRepository.findReadListsRemoved(fromSyncPointId, toSyncPointId, true, pageable) + .also { page -> syncPointRepository.markReadListsSynced(toSyncPointId, true, page.content.map { it.readListId }) } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SyncPointDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SyncPointDao.kt index 0ed7e25f38..48f7f7d580 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SyncPointDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SyncPointDao.kt @@ -4,11 +4,14 @@ import com.github.f4b6a3.tsid.TsidCreator import org.gotson.komga.domain.model.BookSearch import org.gotson.komga.domain.model.KomgaUser import org.gotson.komga.domain.model.SyncPoint +import org.gotson.komga.domain.model.SyncPoint.ReadList.Companion.ON_DECK_ID import org.gotson.komga.domain.persistence.SyncPointRepository import org.gotson.komga.infrastructure.jooq.toCondition import org.gotson.komga.jooq.main.Tables import org.gotson.komga.language.toZonedDateTime import org.jooq.DSLContext +import org.jooq.Field +import org.jooq.Record1 import org.jooq.SelectConditionStep import org.jooq.impl.DSL import org.springframework.data.domain.Page @@ -24,6 +27,7 @@ import java.time.ZoneId @Component class SyncPointDao( private val dsl: DSLContext, + private val bookCommonDao: BookCommonDao, ) : SyncPointRepository { private val b = Tables.BOOK private val m = Tables.MEDIA @@ -33,6 +37,9 @@ class SyncPointDao( private val sp = Tables.SYNC_POINT private val spb = Tables.SYNC_POINT_BOOK private val spbs = Tables.SYNC_POINT_BOOK_REMOVED_SYNCED + private val sprl = Tables.SYNC_POINT_READLIST + private val sprlb = Tables.SYNC_POINT_READLIST_BOOK + private val sprls = Tables.SYNC_POINT_READLIST_REMOVED_SYNCED @Transactional override fun create( @@ -43,7 +50,7 @@ class SyncPointDao( val conditions = search.toCondition().and(user.restrictions.toCondition(dsl)) val syncPointId = TsidCreator.getTsid256().toString() - val createdAt = LocalDateTime.now() + val createdAt = LocalDateTime.now(ZoneId.of("Z")) dsl.insertInto( sp, @@ -91,6 +98,43 @@ class SyncPointDao( return findByIdOrNull(syncPointId)!! } + @Transactional + override fun addOnDeck( + syncPointId: String, + user: KomgaUser, + filterOnLibraryIds: Collection?, + ) { + val createdAt = LocalDateTime.now(ZoneId.of("Z")) + val onDeckFields: Array> = arrayOf(DSL.`val`(syncPointId), DSL.`val`(ON_DECK_ID), b.ID) + + val (query, _, queryMostRecentDate) = bookCommonDao.getBooksOnDeckQuery(user.id, user.restrictions, filterOnLibraryIds, onDeckFields) + + val count = + dsl.insertInto(sprlb) + .select(query) + .execute() + + // only add the read list entry if some books were added + if (count > 0) { + val mostRecentDate = dsl.fetch(queryMostRecentDate).into(LocalDateTime::class.java).firstOrNull() ?: createdAt + + dsl.insertInto( + sprl, + sprl.SYNC_POINT_ID, + sprl.READLIST_ID, + sprl.READLIST_NAME, + sprl.READLIST_CREATED_DATE, + sprl.READLIST_LAST_MODIFIED_DATE, + ).values( + syncPointId, + ON_DECK_ID, + "On Deck", + createdAt, + mostRecentDate, + ).execute() + } + } + override fun findByIdOrNull(syncPointId: String): SyncPoint? = dsl.selectFrom(sp) .where(sp.ID.eq(syncPointId)) @@ -118,7 +162,7 @@ class SyncPointDao( } } - return queryToPage(query, pageable) + return queryToPageBook(query, pageable) } override fun findBooksAdded( @@ -141,7 +185,7 @@ class SyncPointDao( ), ) - return queryToPage(query, pageable) + return queryToPageBook(query, pageable) } override fun findBooksRemoved( @@ -167,7 +211,7 @@ class SyncPointDao( ) } - return queryToPage(query, pageable) + return queryToPageBook(query, pageable) } override fun findBooksChanged( @@ -195,7 +239,7 @@ class SyncPointDao( .or(spb.BOOK_METADATA_LAST_MODIFIED_DATE.ne(spbFrom.BOOK_METADATA_LAST_MODIFIED_DATE)), ) - return queryToPage(query, pageable) + return queryToPageBook(query, pageable) } override fun findBooksReadProgressChanged( @@ -230,9 +274,104 @@ class SyncPointDao( ), ) - return queryToPage(query, pageable) + return queryToPageBook(query, pageable) + } + + override fun findReadListsById( + syncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page { + val query = + dsl.selectFrom(sprl) + .where(sprl.SYNC_POINT_ID.eq(syncPointId)) + .apply { + if (onlyNotSynced) { + and(sprl.SYNCED.isFalse) + } + } + + return queryToPageReadList(query, pageable) + } + + override fun findReadListsAdded( + fromSyncPointId: String, + toSyncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page { + val to = sprl.`as`("to") + val from = sprl.`as`("from") + val query = + dsl.select(*to.fields()) + .from(to) + .leftOuterJoin(from).on(to.READLIST_ID.eq(from.READLIST_ID).and(from.SYNC_POINT_ID.eq(fromSyncPointId))) + .where(to.SYNC_POINT_ID.eq(toSyncPointId)) + .apply { if (onlyNotSynced) and(to.SYNCED.isFalse) } + .and(from.READLIST_ID.isNull) + + return queryToPageReadList(query, pageable) + } + + override fun findReadListsChanged( + fromSyncPointId: String, + toSyncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page { + val from = sprl.`as`("from") + val query = + dsl.select(*sprl.fields()) + .from(sprl) + .join(from).on(sprl.READLIST_ID.eq(from.READLIST_ID)) + .where(sprl.SYNC_POINT_ID.eq(toSyncPointId)) + .and(from.SYNC_POINT_ID.eq(fromSyncPointId)) + .apply { if (onlyNotSynced) and(sprl.SYNCED.isFalse) } + .and( + sprl.READLIST_LAST_MODIFIED_DATE.ne(from.READLIST_LAST_MODIFIED_DATE) + .or(sprl.READLIST_NAME.ne(from.READLIST_NAME)), + ) + + return queryToPageReadList(query, pageable) + } + + override fun findReadListsRemoved( + fromSyncPointId: String, + toSyncPointId: String, + onlyNotSynced: Boolean, + pageable: Pageable, + ): Page { + val from = sprl.`as`("from") + val to = sprl.`as`("to") + val query = + dsl.select(*from.fields()) + .from(from) + .leftOuterJoin(to).on(from.READLIST_ID.eq(to.READLIST_ID).and(to.SYNC_POINT_ID.eq(toSyncPointId))) + .where(from.SYNC_POINT_ID.eq(fromSyncPointId)) + .apply { + if (onlyNotSynced) + and( + from.READLIST_ID.notIn( + dsl.select(sprls.READLIST_ID).from(sprls).where(sprls.SYNC_POINT_ID.eq(toSyncPointId)), + ), + ) + } + .and(to.READLIST_ID.isNull) + + return queryToPageReadList(query, pageable) } + override fun findBookIdsByReadListIds( + syncPointId: String, + readListIds: Collection, + ): List = + dsl.select(*sprlb.fields()) + .from(sprlb) + .where(sprlb.SYNC_POINT_ID.eq(syncPointId)) + .and(sprlb.READLIST_ID.`in`(readListIds)) + .fetchInto(sprlb) + .map { SyncPoint.ReadList.Book(it.syncPointId, it.readlistId, it.bookId) } + override fun markBooksSynced( syncPointId: String, forRemovedBooks: Boolean, @@ -256,17 +395,31 @@ class SyncPointDao( } } + override fun markReadListsSynced( + syncPointId: String, + forRemovedReadLists: Boolean, + readListIds: Collection, + ) { + // removed read lists are not present in the 'to' SyncPoint, only in the 'from' SyncPoint + // we store status in a separate table + if (readListIds.isNotEmpty()) { + if (forRemovedReadLists) + dsl.batch( + dsl.insertInto(sprls, sprls.SYNC_POINT_ID, sprls.READLIST_ID).values(null as String?, null).onDuplicateKeyIgnore(), + ).also { step -> + readListIds.map { step.bind(syncPointId, it) } + }.execute() + else + dsl.update(sprl) + .set(sprl.SYNCED, true) + .where(sprl.SYNC_POINT_ID.eq(syncPointId)) + .and(sprl.READLIST_ID.`in`(readListIds)) + .execute() + } + } + override fun deleteByUserId(userId: String) { - dsl.deleteFrom(spbs).where( - spbs.SYNC_POINT_ID.`in`( - dsl.select(sp.ID).from(sp).where(sp.USER_ID.eq(userId)), - ), - ).execute() - dsl.deleteFrom(spb).where( - spb.SYNC_POINT_ID.`in`( - dsl.select(sp.ID).from(sp).where(sp.USER_ID.eq(userId)), - ), - ).execute() + deleteSubEntities(dsl.select(sp.ID).from(sp).where(sp.USER_ID.eq(userId))) dsl.deleteFrom(sp).where(sp.USER_ID.eq(userId)).execute() } @@ -274,32 +427,37 @@ class SyncPointDao( userId: String, apiKeyIds: Collection, ) { - dsl.deleteFrom(spbs).where( - spbs.SYNC_POINT_ID.`in`( - dsl.select(sp.ID).from(sp).where(sp.USER_ID.eq(userId).and(sp.API_KEY_ID.`in`(apiKeyIds))), - ), - ).execute() - dsl.deleteFrom(spb).where( - spb.SYNC_POINT_ID.`in`( - dsl.select(sp.ID).from(sp).where(sp.USER_ID.eq(userId).and(sp.API_KEY_ID.`in`(apiKeyIds))), - ), - ).execute() + deleteSubEntities(dsl.select(sp.ID).from(sp).where(sp.USER_ID.eq(userId).and(sp.API_KEY_ID.`in`(apiKeyIds)))) dsl.deleteFrom(sp).where(sp.USER_ID.eq(userId).and(sp.API_KEY_ID.`in`(apiKeyIds))).execute() } + private fun deleteSubEntities(condition: SelectConditionStep>) { + dsl.deleteFrom(sprls).where(sprls.SYNC_POINT_ID.`in`(condition)).execute() + dsl.deleteFrom(sprlb).where(sprlb.SYNC_POINT_ID.`in`(condition)).execute() + dsl.deleteFrom(sprl).where(sprl.SYNC_POINT_ID.`in`(condition)).execute() + dsl.deleteFrom(spbs).where(spbs.SYNC_POINT_ID.`in`(condition)).execute() + dsl.deleteFrom(spb).where(spb.SYNC_POINT_ID.`in`(condition)).execute() + } + override fun deleteOne(syncPointId: String) { + dsl.deleteFrom(sprls).where(sprls.SYNC_POINT_ID.eq(syncPointId)).execute() + dsl.deleteFrom(sprlb).where(sprlb.SYNC_POINT_ID.eq(syncPointId)).execute() + dsl.deleteFrom(sprl).where(sprl.SYNC_POINT_ID.eq(syncPointId)).execute() dsl.deleteFrom(spbs).where(spbs.SYNC_POINT_ID.eq(syncPointId)).execute() dsl.deleteFrom(spb).where(spb.SYNC_POINT_ID.eq(syncPointId)).execute() dsl.deleteFrom(sp).where(sp.ID.eq(syncPointId)).execute() } override fun deleteAll() { + dsl.deleteFrom(sprls).execute() + dsl.deleteFrom(sprlb).execute() + dsl.deleteFrom(sprl).execute() dsl.deleteFrom(spbs).execute() dsl.deleteFrom(spb).execute() dsl.deleteFrom(sp).execute() } - private fun queryToPage( + private fun queryToPageBook( query: SelectConditionStep<*>, pageable: Pageable, ): Page { @@ -332,4 +490,35 @@ class SyncPointDao( count.toLong(), ) } + + private fun queryToPageReadList( + query: SelectConditionStep<*>, + pageable: Pageable, + ): Page { + val count = dsl.fetchCount(query) + + val items = + query + .apply { if (pageable.isPaged) limit(pageable.pageSize).offset(pageable.offset) } + .fetchInto(sprl) + .map { + SyncPoint.ReadList( + syncPointId = it.syncPointId, + readListId = it.readlistId, + readListName = it.readlistName, + createdDate = it.readlistCreatedDate.atZone(ZoneId.of("Z")), + lastModifiedDate = it.readlistLastModifiedDate.atZone(ZoneId.of("Z")), + synced = it.synced, + ) + } + + return PageImpl( + items, + if (pageable.isPaged) + PageRequest.of(pageable.pageNumber, pageable.pageSize, Sort.unsorted()) + else + PageRequest.of(0, maxOf(count, 20), Sort.unsorted()), + count.toLong(), + ) + } } diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/KoboController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/KoboController.kt index 8d125c806c..971c8cdbfd 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/KoboController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/KoboController.kt @@ -38,8 +38,11 @@ import org.gotson.komga.interfaces.api.kobo.dto.BookmarkDto import org.gotson.komga.interfaces.api.kobo.dto.ChangedEntitlementDto import org.gotson.komga.interfaces.api.kobo.dto.ChangedProductMetadataDto import org.gotson.komga.interfaces.api.kobo.dto.ChangedReadingStateDto +import org.gotson.komga.interfaces.api.kobo.dto.ChangedTagDto +import org.gotson.komga.interfaces.api.kobo.dto.DeletedTagDto import org.gotson.komga.interfaces.api.kobo.dto.KoboBookMetadataDto import org.gotson.komga.interfaces.api.kobo.dto.NewEntitlementDto +import org.gotson.komga.interfaces.api.kobo.dto.NewTagDto import org.gotson.komga.interfaces.api.kobo.dto.ReadingStateDto import org.gotson.komga.interfaces.api.kobo.dto.ReadingStateStateUpdateDto import org.gotson.komga.interfaces.api.kobo.dto.ReadingStateUpdateResultDto @@ -50,10 +53,12 @@ import org.gotson.komga.interfaces.api.kobo.dto.StatisticsDto import org.gotson.komga.interfaces.api.kobo.dto.StatusDto import org.gotson.komga.interfaces.api.kobo.dto.StatusInfoDto import org.gotson.komga.interfaces.api.kobo.dto.SyncResultDto +import org.gotson.komga.interfaces.api.kobo.dto.TagItemDto import org.gotson.komga.interfaces.api.kobo.dto.TestsDto import org.gotson.komga.interfaces.api.kobo.dto.WrappedReadingStateDto import org.gotson.komga.interfaces.api.kobo.dto.toBookEntitlementDto import org.gotson.komga.interfaces.api.kobo.dto.toDto +import org.gotson.komga.interfaces.api.kobo.dto.toWrappedTagDto import org.gotson.komga.interfaces.api.kobo.persistence.KoboDtoRepository import org.gotson.komga.language.toUTCZoned import org.springframework.data.domain.Page @@ -264,10 +269,38 @@ class KoboController( else Page.empty() - logger.debug { "Library sync: ${booksAdded.numberOfElements} books added, ${booksChanged.numberOfElements} books changed, ${booksRemoved.numberOfElements} books removed, ${changedReadingState.numberOfElements} books with changed reading state" } + val readListsAdded = + if (changedReadingState.isLast && maxRemainingCount > 0) + syncPointLifecycle.takeReadListsAdded(fromSyncPoint.id, toSyncPoint.id, Pageable.ofSize(maxRemainingCount)).also { + maxRemainingCount -= it.numberOfElements + shouldContinueSync = shouldContinueSync || it.hasNext() + } + else + Page.empty() + + val readListsChanged = + if (readListsAdded.isLast && maxRemainingCount > 0) + syncPointLifecycle.takeReadListsChanged(fromSyncPoint.id, toSyncPoint.id, Pageable.ofSize(maxRemainingCount)).also { + maxRemainingCount -= it.numberOfElements + shouldContinueSync = shouldContinueSync || it.hasNext() + } + else + Page.empty() + + val readListsRemoved = + if (readListsChanged.isLast && maxRemainingCount > 0) + syncPointLifecycle.takeReadListsRemoved(fromSyncPoint.id, toSyncPoint.id, Pageable.ofSize(maxRemainingCount)).also { + maxRemainingCount -= it.numberOfElements + shouldContinueSync = shouldContinueSync || it.hasNext() + } + else + Page.empty() + + logger.debug { "Library sync: ${booksAdded.numberOfElements} books added, ${booksChanged.numberOfElements} books changed, ${booksRemoved.numberOfElements} books removed, ${changedReadingState.numberOfElements} books with changed reading state, $readListsAdded readlists added, $readListsChanged readlists changed, $readListsRemoved removed" } val metadata = koboDtoRepository.findBookMetadataByIds((booksAdded.content + booksChanged.content).map { it.bookId }, getDownloadUrlBuilder(authToken)).associateBy { it.entitlementId } val readProgress = readProgressRepository.findAllByBookIdsAndUserId((booksAdded.content + booksChanged.content + changedReadingState.content).map { it.bookId }, principal.user.id).associateBy { it.bookId } + val readListsBooks = syncPointRepository.findBookIdsByReadListIds(toSyncPoint.id, (readListsAdded.content + readListsChanged.content).map { it.readListId }).groupBy { it.readListId } buildList { addAll( @@ -319,24 +352,63 @@ class KoboController( } }, ) + addAll( + readListsAdded.content.map { + NewTagDto(it.toWrappedTagDto(readListsBooks[it.readListId]?.map { b -> TagItemDto(b.bookId) })) + }, + ) + addAll( + readListsChanged.content.map { + ChangedTagDto(it.toWrappedTagDto(readListsBooks[it.readListId]?.map { b -> TagItemDto(b.bookId) })) + }, + ) + addAll( + readListsRemoved.content.map { + DeletedTagDto(it.toWrappedTagDto()) + }, + ) } } else { // no starting point, sync everything - val books = syncPointLifecycle.takeBooks(toSyncPoint.id, Pageable.ofSize(komgaProperties.kobo.syncItemLimit)) - shouldContinueSync = books.hasNext() + var maxRemainingCount = komgaProperties.kobo.syncItemLimit - logger.debug { "Library sync: ${books.numberOfElements} books" } + val books = + syncPointLifecycle.takeBooks(toSyncPoint.id, Pageable.ofSize(maxRemainingCount)).also { + maxRemainingCount -= it.numberOfElements + shouldContinueSync = it.hasNext() + } + + val readLists = + if (books.isLast && maxRemainingCount > 0) + syncPointLifecycle.takeReadLists(toSyncPoint.id, Pageable.ofSize(maxRemainingCount)).also { + maxRemainingCount -= it.numberOfElements + shouldContinueSync = shouldContinueSync || it.hasNext() + } + else + Page.empty() + + logger.debug { "Library sync: ${books.numberOfElements} books, ${readLists.numberOfElements} readlists" } val metadata = koboDtoRepository.findBookMetadataByIds(books.content.map { it.bookId }, getDownloadUrlBuilder(authToken)).associateBy { it.entitlementId } val readProgress = readProgressRepository.findAllByBookIdsAndUserId(books.content.map { it.bookId }, principal.user.id).associateBy { it.bookId } + val readListsBooks = syncPointRepository.findBookIdsByReadListIds(toSyncPoint.id, readLists.content.map { it.readListId }).groupBy { it.readListId } - books.content.map { - NewEntitlementDto( - BookEntitlementContainerDto( - bookEntitlement = it.toBookEntitlementDto(false), - bookMetadata = metadata[it.bookId]!!, - readingState = readProgress[it.bookId]?.toDto() ?: getEmptyReadProgressForBook(it.bookId, it.createdDate), - ), + buildList { + addAll( + books.content.map { + NewEntitlementDto( + BookEntitlementContainerDto( + bookEntitlement = it.toBookEntitlementDto(false), + bookMetadata = metadata[it.bookId]!!, + readingState = readProgress[it.bookId]?.toDto() ?: getEmptyReadProgressForBook(it.bookId, it.createdDate), + ), + ) + }, + ) + addAll( + readLists.content.map { + NewTagDto(it.toWrappedTagDto(readListsBooks[it.readListId]?.map { b -> TagItemDto(b.bookId) })) + }, ) } } diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/SyncResultDto.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/SyncResultDto.kt index 2b8c313303..9db0661e33 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/SyncResultDto.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/SyncResultDto.kt @@ -22,12 +22,17 @@ data class ChangedProductMetadataDto( @JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class) data class NewTagDto( - val newTag: TagDto, + val newTag: WrappedTagDto, ) : SyncResultDto @JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class) data class ChangedTagDto( - val changedTag: TagDto, + val changedTag: WrappedTagDto, +) : SyncResultDto + +@JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class) +data class DeletedTagDto( + val deletedTag: WrappedTagDto, ) : SyncResultDto @JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class) diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagDto.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagDto.kt index c877091598..ad3d6a3af4 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagDto.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagDto.kt @@ -2,6 +2,7 @@ package org.gotson.komga.interfaces.api.kobo.dto import com.fasterxml.jackson.databind.PropertyNamingStrategies import com.fasterxml.jackson.databind.annotation.JsonNaming +import org.gotson.komga.domain.model.SyncPoint import java.time.ZonedDateTime @JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class) @@ -13,3 +14,20 @@ data class TagDto( val type: TagTypeDto, val items: List? = null, ) + +@JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class) +data class WrappedTagDto( + val tag: TagDto, +) + +fun SyncPoint.ReadList.toWrappedTagDto(items: List? = null) = + WrappedTagDto( + TagDto( + id = readListId, + created = createdDate, + lastModified = lastModifiedDate, + name = readListName, + type = TagTypeDto.USER_TAG, + items = items, + ), + ) diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagTypeDto.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagTypeDto.kt index 65b86ff0b4..6c0391edc8 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagTypeDto.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/kobo/dto/TagTypeDto.kt @@ -1,10 +1,11 @@ package org.gotson.komga.interfaces.api.kobo.dto -import com.fasterxml.jackson.databind.PropertyNamingStrategies -import com.fasterxml.jackson.databind.annotation.JsonNaming +import com.fasterxml.jackson.annotation.JsonProperty -@JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy::class) enum class TagTypeDto { + @JsonProperty("SystemTag") SYSTEM_TAG, + + @JsonProperty("UserTag") USER_TAG, } diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/service/SyncPointLifecycleTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/service/SyncPointLifecycleTest.kt index 4207778755..f684b019be 100644 --- a/komga/src/test/kotlin/org/gotson/komga/domain/service/SyncPointLifecycleTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/domain/service/SyncPointLifecycleTest.kt @@ -8,6 +8,7 @@ import org.gotson.komga.domain.model.ContentRestrictions import org.gotson.komga.domain.model.KomgaUser import org.gotson.komga.domain.model.Media import org.gotson.komga.domain.model.MediaType +import org.gotson.komga.domain.model.SyncPoint.ReadList.Companion.ON_DECK_ID import org.gotson.komga.domain.model.makeBook import org.gotson.komga.domain.model.makeLibrary import org.gotson.komga.domain.model.makeSeries @@ -29,6 +30,7 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import java.time.LocalDateTime +import java.time.ZonedDateTime import java.time.temporal.ChronoUnit @SpringBootTest @@ -39,13 +41,12 @@ class SyncPointLifecycleTest( @Autowired private val userRepository: KomgaUserRepository, @Autowired private val bookRepository: BookRepository, @Autowired private val bookMetadataRepository: BookMetadataRepository, + @Autowired private val bookLifecycle: BookLifecycle, @Autowired private val seriesMetadataRepository: SeriesMetadataRepository, @Autowired private val seriesRepository: SeriesRepository, @Autowired private val seriesLifecycle: SeriesLifecycle, @Autowired private val mediaRepository: MediaRepository, ) { - @Autowired - private lateinit var bookLifecycle: BookLifecycle private val library1 = makeLibrary() private val library2 = makeLibrary() private val library3 = makeLibrary() @@ -269,4 +270,96 @@ class SyncPointLifecycleTest( .containsAnyElementsOf(listOf(book1.id, book2.id, book3.id)) .doesNotContainAnyElementsOf((page1 + page2).map { it.bookId }) } + + @Test + fun `given syncpoint when books are read then syncpoint diff contains on deck read list`() { + // given + val book1 = makeBook("book 1", libraryId = library1.id).copy(fileHash = "hash", fileSize = 12, fileLastModified = LocalDateTime.now(), number = 1) + val book2 = makeBook("book 2", libraryId = library1.id).copy(fileHash = "hash", fileSize = 12, fileLastModified = LocalDateTime.now(), number = 2) + val book3 = makeBook("book 3", libraryId = library1.id).copy(fileHash = "hash", fileSize = 12, fileLastModified = LocalDateTime.now(), number = 3) + + makeSeries(name = "series1", libraryId = library1.id).let { series -> + seriesLifecycle.createSeries(series).let { created -> + seriesLifecycle.addBooks(created, listOf(book1, book2, book3)) + } + } + + bookRepository.findAll().forEach { mediaRepository.findById(it.id).let { media -> mediaRepository.update(media.copy(status = Media.Status.READY, mediaType = MediaType.EPUB.type)) } } + + // first sync point + val syncPoint1 = syncPointLifecycle.createSyncPoint(user1, null, listOf(library1.id)) + val syncPoint1ReadLists = syncPointRepository.findReadListsById(syncPoint1.id, false, Pageable.unpaged()) + + assertThat(syncPoint1ReadLists).isEmpty() + + // book marked as read + bookLifecycle.markReadProgressCompleted(book1.id, user1) + val syncPoint2 = syncPointLifecycle.createSyncPoint(user1, null, listOf(library1.id)) + + // on deck is present and has 1 book + val syncPoint2ReadLists = syncPointRepository.findReadListsById(syncPoint2.id, false, Pageable.unpaged()) + val rlAdded1to2 = syncPointRepository.findReadListsAdded(syncPoint1.id, syncPoint2.id, false, Pageable.unpaged()) + val rlChanged1to2 = syncPointRepository.findReadListsChanged(syncPoint1.id, syncPoint2.id, false, Pageable.unpaged()) + val rlRemoved1to2 = syncPointRepository.findReadListsRemoved(syncPoint1.id, syncPoint2.id, false, Pageable.unpaged()) + val syncPoint2Page1 = syncPointLifecycle.takeReadListsAdded(syncPoint1.id, syncPoint2.id, Pageable.ofSize(1)) + val syncPoint2Page2 = syncPointLifecycle.takeReadListsAdded(syncPoint1.id, syncPoint2.id, Pageable.ofSize(1)) + + assertThat(syncPoint2ReadLists).hasSize(1) + assertThat(rlAdded1to2).containsExactlyInAnyOrderElementsOf(syncPoint2ReadLists) + assertThat(rlChanged1to2).isEmpty() + assertThat(rlRemoved1to2).isEmpty() + with(syncPoint2ReadLists.first()) { + assertThat(this.readListId).isEqualTo(ON_DECK_ID) + assertThat(this.createdDate).isCloseTo(ZonedDateTime.now(), within(1, ChronoUnit.SECONDS)) + assertThat(this.lastModifiedDate).isCloseTo(ZonedDateTime.now(), within(1, ChronoUnit.SECONDS)) + } + assertThat(syncPoint2Page1).containsExactlyInAnyOrderElementsOf(rlAdded1to2) + assertThat(syncPoint2Page2).isEmpty() + val syncPoint2OnDeckBooks = syncPointRepository.findBookIdsByReadListIds(syncPoint2.id, listOf(ON_DECK_ID)) + assertThat(syncPoint2OnDeckBooks.map { it.bookId }) + .hasSize(1) + .containsExactlyInAnyOrder(book2.id) + + // 2nd book marked as read, on deck is still present but has changed + bookLifecycle.markReadProgressCompleted(book2.id, user1) + val syncPoint3 = syncPointLifecycle.createSyncPoint(user1, null, listOf(library1.id)) + + val syncPoint3ReadLists = syncPointRepository.findReadListsById(syncPoint3.id, false, Pageable.unpaged()) + val rlAdded2to3 = syncPointRepository.findReadListsAdded(syncPoint2.id, syncPoint3.id, false, Pageable.unpaged()) + val rlChanged2to3 = syncPointRepository.findReadListsChanged(syncPoint2.id, syncPoint3.id, false, Pageable.unpaged()) + val rlRemoved2to3 = syncPointRepository.findReadListsRemoved(syncPoint2.id, syncPoint3.id, false, Pageable.unpaged()) + val syncPoint3Page1 = syncPointLifecycle.takeReadListsChanged(syncPoint2.id, syncPoint3.id, Pageable.ofSize(1)) + val syncPoint3Page2 = syncPointLifecycle.takeReadListsChanged(syncPoint2.id, syncPoint3.id, Pageable.ofSize(1)) + + assertThat(syncPoint3ReadLists.map { it.readListId }).containsExactlyInAnyOrder(ON_DECK_ID) + assertThat(rlChanged2to3.map { it.readListId }).containsExactlyInAnyOrder(ON_DECK_ID) + assertThat(rlAdded2to3).isEmpty() + assertThat(rlRemoved2to3).isEmpty() + assertThat(syncPoint3Page1).containsExactlyInAnyOrderElementsOf(rlChanged2to3) + assertThat(syncPoint3Page2).isEmpty() + + val syncPoint3OnDeckBooks = syncPointRepository.findBookIdsByReadListIds(syncPoint3.id, listOf(ON_DECK_ID)) + assertThat(syncPoint3OnDeckBooks.map { it.bookId }) + .hasSize(1) + .containsExactlyInAnyOrder(book3.id) + + // 3rd book marked as read, whole series is read now - on deck is not present anymore + bookLifecycle.markReadProgressCompleted(book3.id, user1) + val syncPoint4 = syncPointLifecycle.createSyncPoint(user1, null, listOf(library1.id)) + + val syncPoint4ReadLists = syncPointRepository.findReadListsById(syncPoint4.id, false, Pageable.unpaged()) + val rlAdded3to4 = syncPointRepository.findReadListsAdded(syncPoint3.id, syncPoint4.id, false, Pageable.unpaged()) + val rlChanged3to4 = syncPointRepository.findReadListsChanged(syncPoint3.id, syncPoint4.id, false, Pageable.unpaged()) + val rlRemoved3to4 = syncPointRepository.findReadListsRemoved(syncPoint3.id, syncPoint4.id, false, Pageable.unpaged()) + val syncPoint4Page1 = syncPointLifecycle.takeReadListsRemoved(syncPoint3.id, syncPoint4.id, Pageable.ofSize(1)) + val syncPoint4Page2 = syncPointLifecycle.takeReadListsRemoved(syncPoint3.id, syncPoint4.id, Pageable.ofSize(1)) + + assertThat(syncPoint4ReadLists).isEmpty() + assertThat(rlAdded3to4).isEmpty() + assertThat(rlChanged3to4).isEmpty() + assertThat(rlRemoved3to4).hasSize(1) + assertThat(rlRemoved3to4.map { it.readListId }).containsExactlyInAnyOrder(ON_DECK_ID) + assertThat(syncPoint4Page1).containsExactlyInAnyOrderElementsOf(rlRemoved3to4) + assertThat(syncPoint4Page2).isEmpty() + } }