From 494bdf28a110c566843e20ae558181fb4e0b4a32 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Fri, 31 Dec 2021 10:16:54 +0800 Subject: [PATCH] feat(api): search series by completeness --- .../org/gotson/komga/application/tasks/Task.kt | 5 +++-- .../komga/application/tasks/TaskHandler.kt | 2 +- .../komga/application/tasks/TaskReceiver.kt | 5 +++-- .../gotson/komga/domain/model/SeriesSearch.kt | 3 +++ .../komga/infrastructure/jooq/SeriesDtoDao.kt | 16 +++++++++------- .../komga/infrastructure/search/LuceneEntity.kt | 1 + .../search/SearchIndexLifecycle.kt | 12 +++++++----- .../api/rest/SeriesCollectionController.kt | 2 ++ .../interfaces/api/rest/SeriesController.kt | 4 ++++ .../scheduler/SearchIndexController.kt | 2 ++ 10 files changed, 35 insertions(+), 17 deletions(-) diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt index 9cc4f76995c..db882a739b3 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt @@ -2,6 +2,7 @@ package org.gotson.komga.application.tasks import org.gotson.komga.domain.model.BookMetadataPatchCapability import org.gotson.komga.domain.model.CopyMode +import org.gotson.komga.infrastructure.search.LuceneEntity import java.io.Serializable const val HIGHEST_PRIORITY = 8 @@ -79,9 +80,9 @@ sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable { override fun toString(): String = "RepairExtension(bookId='$bookId', priority='$priority')" } - class RebuildIndex(priority: Int = DEFAULT_PRIORITY) : Task(priority) { + class RebuildIndex(val entities: Set?, priority: Int = DEFAULT_PRIORITY) : Task(priority) { override fun uniqueId() = "REBUILD_INDEX" - override fun toString(): String = "RebuildIndex(priority='$priority')" + override fun toString(): String = "RebuildIndex(priority='$priority',entities='${entities?.map { it.type }}')" } class DeleteBook(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt index c334fe47029..2c3c0965934 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt @@ -122,7 +122,7 @@ class TaskHandler( bookLifecycle.hashAndPersist(book) } ?: logger.warn { "Cannot execute task $task: Book does not exist" } - is Task.RebuildIndex -> searchIndexLifecycle.rebuildIndex() + is Task.RebuildIndex -> searchIndexLifecycle.rebuildIndex(task.entities) is Task.DeleteBook -> { bookRepository.findByIdOrNull(task.bookId)?.let { book -> diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt index 89e9942367e..60ff928a911 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt @@ -15,6 +15,7 @@ import org.gotson.komga.infrastructure.jms.QUEUE_TASKS import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE import org.gotson.komga.infrastructure.jms.QUEUE_TYPE import org.gotson.komga.infrastructure.jms.QUEUE_UNIQUE_ID +import org.gotson.komga.infrastructure.search.LuceneEntity import org.springframework.data.domain.Sort import org.springframework.jms.core.JmsTemplate import org.springframework.stereotype.Service @@ -117,8 +118,8 @@ class TaskReceiver( submitTask(Task.ImportBook(sourceFile, seriesId, copyMode, destinationName, upgradeBookId, priority)) } - fun rebuildIndex(priority: Int = DEFAULT_PRIORITY) { - submitTask(Task.RebuildIndex(priority)) + fun rebuildIndex(priority: Int = DEFAULT_PRIORITY, entities: Set? = null) { + submitTask(Task.RebuildIndex(entities, priority)) } fun deleteBook(bookId: String, priority: Int = DEFAULT_PRIORITY) { diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/SeriesSearch.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/SeriesSearch.kt index 4f834ba8bc1..42f29f89a1f 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/SeriesSearch.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/SeriesSearch.kt @@ -8,6 +8,7 @@ open class SeriesSearch( val metadataStatus: Collection? = null, val publishers: Collection? = null, val deleted: Boolean? = null, + val complete: Boolean? = null, ) { enum class SearchField { NAME, TITLE, TITLE_SORT @@ -22,6 +23,7 @@ class SeriesSearchWithReadProgress( metadataStatus: Collection? = null, publishers: Collection? = null, deleted: Boolean? = null, + complete: Boolean? = null, val languages: Collection? = null, val genres: Collection? = null, val tags: Collection? = null, @@ -37,4 +39,5 @@ class SeriesSearchWithReadProgress( metadataStatus = metadataStatus, publishers = publishers, deleted = deleted, + complete = complete, ) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt index c3bbf9bdc86..c227330fce2 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt @@ -87,7 +87,7 @@ class SeriesDtoDao( collectionId: String, search: SeriesSearchWithReadProgress, userId: String, - pageable: Pageable + pageable: Pageable, ): Page { val conditions = search.toCondition().and(cs.COLLECTION_ID.eq(collectionId)) val joinConditions = search.toJoinConditions().copy(selectCollectionNumber = true, collection = true) @@ -98,7 +98,7 @@ class SeriesDtoDao( override fun findAllRecentlyUpdated( search: SeriesSearchWithReadProgress, userId: String, - pageable: Pageable + pageable: Pageable, ): Page { val conditions = search.toCondition() .and(s.CREATED_DATE.ne(s.LAST_MODIFIED_DATE)) @@ -143,7 +143,7 @@ class SeriesDtoDao( private fun selectBase( userId: String, - joinConditions: JoinConditions = JoinConditions() + joinConditions: JoinConditions = JoinConditions(), ): SelectOnConditionStep = dsl.selectDistinct(*groupFields) .apply { if (joinConditions.selectCollectionNumber) select(cs.NUMBER) } @@ -205,7 +205,7 @@ class SeriesDtoDao( dtos, if (pageable.isPaged) PageRequest.of(pageable.pageNumber, pageable.pageSize, pageSort) else PageRequest.of(0, maxOf(count, 20), pageSort), - count.toLong() + count.toLong(), ) } @@ -248,7 +248,7 @@ class SeriesDtoDao( booksUnreadCount, booksInProgressCount, dr.toDto(genres, tags), - bmar.toDto(aggregatedAuthors, aggregatedTags) + bmar.toDto(aggregatedAuthors, aggregatedTags), ) } @@ -262,6 +262,8 @@ class SeriesDtoDao( if (!publishers.isNullOrEmpty()) c = c.and(lower(d.PUBLISHER).`in`(publishers.map { it.lowercase() })) if (deleted == true) c = c.and(s.DELETED_DATE.isNotNull) if (deleted == false) c = c.and(s.DELETED_DATE.isNull) + if (complete == false) c = c.and(d.TOTAL_BOOK_COUNT.isNotNull.and(d.TOTAL_BOOK_COUNT.ne(s.BOOK_COUNT))) + if (complete == true) c = c.and(d.TOTAL_BOOK_COUNT.isNotNull.and(d.TOTAL_BOOK_COUNT.eq(s.BOOK_COUNT))) if (!languages.isNullOrEmpty()) c = c.and(lower(d.LANGUAGE).`in`(languages.map { it.lowercase() })) if (!genres.isNullOrEmpty()) c = c.and(lower(g.GENRE).`in`(genres.map { it.lowercase() })) if (!tags.isNullOrEmpty()) c = c.and(lower(st.TAG).`in`(tags.map { it.lowercase() }).or(lower(bmat.TAG).`in`(tags.map { it.lowercase() }))) @@ -322,7 +324,7 @@ class SeriesDtoDao( booksUnreadCount: Int, booksInProgressCount: Int, metadata: SeriesMetadataDto, - booksMetadata: BookMetadataAggregationDto + booksMetadata: BookMetadataAggregationDto, ) = SeriesDto( id = id, @@ -378,6 +380,6 @@ class SeriesDtoDao( summaryNumber = summaryNumber, created = createdDate.toCurrentTimeZone(), - lastModified = lastModifiedDate.toCurrentTimeZone() + lastModified = lastModifiedDate.toCurrentTimeZone(), ) } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/LuceneEntity.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/LuceneEntity.kt index e17a282b813..4f144b1c905 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/LuceneEntity.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/LuceneEntity.kt @@ -69,6 +69,7 @@ fun SeriesDto.toDocument() = } if (booksMetadata.releaseDate != null) add(TextField("release_date", DateTools.dateToString(booksMetadata.releaseDate.toDate(), DateTools.Resolution.YEAR), Field.Store.NO)) add(TextField("deleted", deleted.toString(), Field.Store.NO)) + if (metadata.totalBookCount != null) add(TextField("complete", (metadata.totalBookCount == booksCount).toString(), Field.Store.NO)) add(StringField(LuceneEntity.TYPE, LuceneEntity.Series.type, Field.Store.NO)) add(StringField(LuceneEntity.Series.id, id, Field.Store.YES)) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/SearchIndexLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/SearchIndexLifecycle.kt index 1e380e845bb..68d2f92a81b 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/SearchIndexLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/search/SearchIndexLifecycle.kt @@ -26,7 +26,7 @@ import kotlin.math.ceil import kotlin.time.measureTime private val logger = KotlinLogging.logger {} -private const val INDEX_VERSION = 3 +private const val INDEX_VERSION = 4 @Component class SearchIndexLifecycle( @@ -37,10 +37,12 @@ class SearchIndexLifecycle( private val luceneHelper: LuceneHelper, ) { - fun rebuildIndex() { - logger.info { "Rebuild all indexes" } + fun rebuildIndex(entities: Set?) { + val targetEntities = entities ?: LuceneEntity.values().toSet() - LuceneEntity.values().forEach { + logger.info { "Rebuild index for: ${targetEntities.map { it.type }}" } + + targetEntities.forEach { when (it) { LuceneEntity.Book -> rebuildIndex(it, { p: Pageable -> bookDtoRepository.findAll(BookSearchWithReadProgress(), "unused", p) }, { e: BookDto -> e.toDocument() }) LuceneEntity.Series -> rebuildIndex(it, { p: Pageable -> seriesDtoRepository.findAll(SeriesSearchWithReadProgress(), "unused", p) }, { e: SeriesDto -> e.toDocument() }) @@ -63,7 +65,7 @@ class SearchIndexLifecycle( indexWriter.deleteDocuments(Term(LuceneEntity.TYPE, entity.type)) (0 until pages).forEach { page -> - logger.info { "Processing page $page of $batchSize elements" } + logger.info { "Processing page ${page + 1} of $pages ($batchSize elements)" } val entityDocs = provider(PageRequest.of(page, batchSize)).content .map { toDoc(it) } indexWriter.addDocuments(entityDocs) diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesCollectionController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesCollectionController.kt index 55d980222e3..37ef7be3f85 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesCollectionController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesCollectionController.kt @@ -263,6 +263,7 @@ class SeriesCollectionController( @RequestParam(name = "age_rating", required = false) ageRatings: List?, @RequestParam(name = "release_year", required = false) release_years: List?, @RequestParam(name = "deleted", required = false) deleted: Boolean?, + @RequestParam(name = "complete", required = false) complete: Boolean?, @RequestParam(name = "unpaged", required = false) unpaged: Boolean = false, @Parameter(hidden = true) @Authors authors: List?, @Parameter(hidden = true) page: Pageable @@ -286,6 +287,7 @@ class SeriesCollectionController( readStatus = readStatus, publishers = publishers, deleted = deleted, + complete = complete, languages = languages, genres = genres, tags = tags, diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesController.kt index 45da9499fd8..e41328e9709 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesController.kt @@ -131,6 +131,7 @@ class SeriesController( @RequestParam(name = "age_rating", required = false) ageRatings: List?, @RequestParam(name = "release_year", required = false) release_years: List?, @RequestParam(name = "deleted", required = false) deleted: Boolean?, + @RequestParam(name = "complete", required = false) complete: Boolean?, @RequestParam(name = "unpaged", required = false) unpaged: Boolean = false, @Parameter(hidden = true) @Authors authors: List?, @Parameter(hidden = true) page: Pageable @@ -165,6 +166,7 @@ class SeriesController( readStatus = readStatus, publishers = publishers, deleted = deleted, + complete = complete, languages = languages, genres = genres, tags = tags, @@ -200,6 +202,7 @@ class SeriesController( @RequestParam(name = "age_rating", required = false) ageRatings: List?, @RequestParam(name = "release_year", required = false) release_years: List?, @RequestParam(name = "deleted", required = false) deleted: Boolean?, + @RequestParam(name = "complete", required = false) complete: Boolean?, @Parameter(hidden = true) @Authors authors: List?, @Parameter(hidden = true) page: Pageable ): List { @@ -218,6 +221,7 @@ class SeriesController( readStatus = readStatus, publishers = publishers, deleted = deleted, + complete = complete, languages = languages, genres = genres, tags = tags, diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/SearchIndexController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/SearchIndexController.kt index a798a6db7b7..a52dece0ed4 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/SearchIndexController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/SearchIndexController.kt @@ -3,6 +3,7 @@ package org.gotson.komga.interfaces.scheduler import mu.KotlinLogging import org.gotson.komga.application.tasks.HIGHEST_PRIORITY import org.gotson.komga.application.tasks.TaskReceiver +import org.gotson.komga.infrastructure.search.LuceneEntity import org.gotson.komga.infrastructure.search.LuceneHelper import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.context.annotation.Profile @@ -27,6 +28,7 @@ class SearchIndexController( logger.info { "Lucene index version: ${luceneHelper.getIndexVersion()}" } when (luceneHelper.getIndexVersion()) { 1, 2 -> taskReceiver.rebuildIndex(HIGHEST_PRIORITY) + 3 -> taskReceiver.rebuildIndex(HIGHEST_PRIORITY, setOf(LuceneEntity.Series)) } } }