diff --git a/komga/src/flyway/resources/db/migration/sqlite/V20230921112923__library_directory_exclusions.sql b/komga/src/flyway/resources/db/migration/sqlite/V20230921112923__library_directory_exclusions.sql new file mode 100644 index 0000000000..b52b72fc2a --- /dev/null +++ b/komga/src/flyway/resources/db/migration/sqlite/V20230921112923__library_directory_exclusions.sql @@ -0,0 +1,17 @@ +CREATE TABLE LIBRARY_EXCLUSIONS +( + LIBRARY_ID varchar NOT NULL, + EXCLUSION varchar NOT NULL, + PRIMARY KEY (LIBRARY_ID, EXCLUSION), + FOREIGN KEY (LIBRARY_ID) REFERENCES LIBRARY (ID) +); + +CREATE INDEX idx__library_exclusions__library_id on LIBRARY_EXCLUSIONS (LIBRARY_ID); + +INSERT INTO LIBRARY_EXCLUSIONS +WITH cte_exclusions(exclude) AS (VALUES ('#recycle'), + ('@eaDir'), + ('@Recycle')) +SELECT LIBRARY.ID, cte_exclusions.exclude +FROM LIBRARY + cross join cte_exclusions; diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt index 258d8d6f75..8ddafd14a4 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt @@ -26,6 +26,7 @@ data class Library( val scanCbx: Boolean = true, val scanPdf: Boolean = true, val scanEpub: Boolean = true, + val scanDirectoryExclusions: Set = emptySet(), val repairExtensions: Boolean = false, val convertToCbz: Boolean = false, val emptyTrashAfterScan: Boolean = false, diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt index 8d726d95f1..85c1b43f9f 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt @@ -6,7 +6,6 @@ import org.gotson.komga.domain.model.DirectoryNotFoundException import org.gotson.komga.domain.model.ScanResult import org.gotson.komga.domain.model.Series import org.gotson.komga.domain.model.Sidecar -import org.gotson.komga.infrastructure.configuration.KomgaProperties import org.gotson.komga.infrastructure.sidecar.SidecarBookConsumer import org.gotson.komga.infrastructure.sidecar.SidecarSeriesConsumer import org.springframework.stereotype.Service @@ -34,7 +33,6 @@ private val logger = KotlinLogging.logger {} @Service class FileSystemScanner( - private val komgaProperties: KomgaProperties, private val sidecarBookConsumers: List, private val sidecarSeriesConsumers: List, ) { @@ -55,6 +53,7 @@ class FileSystemScanner( scanCbx: Boolean = true, scanPdf: Boolean = true, scanEpub: Boolean = true, + directoryExclusions: Set = emptySet(), ): ScanResult { val scanForExtensions = buildList { if (scanCbx) addAll(listOf("cbz", "zip", "cbr", "rar")) @@ -63,7 +62,7 @@ class FileSystemScanner( } logger.info { "Scanning folder: $root" } logger.info { "Scan for extensions: $scanForExtensions" } - logger.info { "Excluded patterns: ${komgaProperties.librariesScanDirectoryExclusions}" } + logger.info { "Excluded directory patterns: $directoryExclusions" } logger.info { "Force directory modified time: $forceDirectoryModifiedTime" } if (!(Files.isDirectory(root) && Files.isReadable(root))) @@ -88,7 +87,7 @@ class FileSystemScanner( override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { logger.trace { "preVisit: $dir (regularFile:${attrs.isRegularFile}, directory:${attrs.isDirectory}, symbolicLink:${attrs.isSymbolicLink}, other:${attrs.isOther})" } if (dir.name.startsWith(".") || - komgaProperties.librariesScanDirectoryExclusions.any { exclude -> + directoryExclusions.any { exclude -> dir.pathString.contains(exclude, true) } ) return FileVisitResult.SKIP_SUBTREE diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt index b94e15206c..68e356908c 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt @@ -73,6 +73,7 @@ class LibraryContentLifecycle( library.scanCbx, library.scanPdf, library.scanEpub, + library.scanDirectoryExclusions, ) } catch (e: DirectoryNotFoundException) { library.copy(unavailableDate = LocalDateTime.now()).let { diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryLifecycle.kt index ef17202bdc..239b8484ac 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryLifecycle.kt @@ -77,6 +77,7 @@ class LibraryLifecycle( if (existing.scanPdf != updated.scanPdf) return true if (existing.scanEpub != updated.scanEpub) return true if (existing.scanForceModifiedTime != updated.scanForceModifiedTime) return true + if (existing.scanDirectoryExclusions != updated.scanDirectoryExclusions) return true return false } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt index 7022678f90..d961b4597a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt @@ -31,6 +31,7 @@ class KomgaProperties { @Deprecated("Moved to library options since 1.5.0") var librariesScanStartup: Boolean = false + @Deprecated("Moved to library options since 1.5.0") var librariesScanDirectoryExclusions: List = emptyList() var deleteEmptyReadLists: Boolean = true diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt index 8788839890..60e745163b 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt @@ -5,6 +5,8 @@ import org.gotson.komga.domain.persistence.LibraryRepository import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.LibraryRecord import org.jooq.DSLContext +import org.jooq.Record +import org.jooq.ResultQuery import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URL @@ -18,39 +20,51 @@ class LibraryDao( private val l = Tables.LIBRARY private val ul = Tables.USER_LIBRARY_SHARING + private val le = Tables.LIBRARY_EXCLUSIONS override fun findByIdOrNull(libraryId: String): Library? = findOne(libraryId) - ?.toDomain() + .fetchAndMap() + .firstOrNull() override fun findById(libraryId: String): Library = - findOne(libraryId)!! - .toDomain() + findOne(libraryId) + .fetchAndMap() + .first() private fun findOne(libraryId: String) = - dsl.selectFrom(l) + selectBase() .where(l.ID.eq(libraryId)) - .fetchOneInto(l) - override fun findAll(): Collection = - dsl.selectFrom(l) - .fetchInto(l) - .map { it.toDomain() } + selectBase() + .fetchAndMap() override fun findAllByIds(libraryIds: Collection): Collection = - dsl.selectFrom(l) + selectBase() .where(l.ID.`in`(libraryIds)) - .fetchInto(l) - .map { it.toDomain() } + .fetchAndMap() + + private fun selectBase() = + dsl.select() + .from(l) + .leftJoin(le).onKey() + + private fun ResultQuery.fetchAndMap(): Collection = + this.fetchGroups({ it.into(l) }, { it.into(le) }) + .map { (lr, ler) -> + lr.toDomain(ler.mapNotNull { it.exclusion }.toSet()) + } @Transactional override fun delete(libraryId: String) { + dsl.deleteFrom(le).where(le.LIBRARY_ID.eq(libraryId)).execute() dsl.deleteFrom(ul).where(ul.LIBRARY_ID.eq(libraryId)).execute() dsl.deleteFrom(l).where(l.ID.eq(libraryId)).execute() } @Transactional override fun deleteAll() { + dsl.deleteFrom(le).execute() dsl.deleteFrom(ul).execute() dsl.deleteFrom(l).execute() } @@ -87,6 +101,8 @@ class LibraryDao( .set(l.ONESHOTS_DIRECTORY, library.oneshotsDirectory) .set(l.UNAVAILABLE_DATE, library.unavailableDate) .execute() + + insertDirectoryExclusions(library) } @Transactional @@ -122,11 +138,32 @@ class LibraryDao( .set(l.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z"))) .where(l.ID.eq(library.id)) .execute() + + dsl.deleteFrom(le).where(le.LIBRARY_ID.eq(library.id)).execute() + insertDirectoryExclusions(library) } override fun count(): Long = dsl.fetchCount(l).toLong() + fun findDirectoryExclusions(libraryId: String): Set = + dsl.select(le.EXCLUSION) + .from(le) + .where(le.LIBRARY_ID.eq(libraryId)) + .fetchSet(le.EXCLUSION) + + private fun insertDirectoryExclusions(library: Library) { + if (library.scanDirectoryExclusions.isNotEmpty()) { + dsl.batch( + dsl.insertInto(le, le.LIBRARY_ID, le.EXCLUSION) + .values(null as String?, null), + ).also { step -> + library.scanDirectoryExclusions.forEach { + step.bind(library.id, it) + } + }.execute() + } + } - private fun LibraryRecord.toDomain() = + private fun LibraryRecord.toDomain(directoryExclusions: Set) = Library( name = name, root = URL(root), @@ -146,6 +183,7 @@ class LibraryDao( scanEpub = scanEpub, scanOnStartup = scanStartup, scanInterval = Library.ScanInterval.valueOf(scanInterval), + scanDirectoryExclusions = directoryExclusions, repairExtensions = repairExtensions, convertToCbz = convertToCbz, emptyTrashAfterScan = emptyTrashAfterScan, diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/LibraryController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/LibraryController.kt index 13ab34f519..e747bf752b 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/LibraryController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/LibraryController.kt @@ -98,6 +98,7 @@ class LibraryController( scanCbx = library.scanCbx, scanPdf = library.scanPdf, scanEpub = library.scanEpub, + scanDirectoryExclusions = library.scanDirectoryExclusions, repairExtensions = library.repairExtensions, convertToCbz = library.convertToCbz, emptyTrashAfterScan = library.emptyTrashAfterScan, @@ -165,6 +166,7 @@ class LibraryController( scanCbx = scanCbx ?: existing.scanCbx, scanPdf = scanPdf ?: existing.scanPdf, scanEpub = scanEpub ?: existing.scanEpub, + scanDirectoryExclusions = if (isSet("scanDirectoryExclusions")) scanDirectoryExclusions ?: emptySet() else existing.scanDirectoryExclusions, repairExtensions = repairExtensions ?: existing.repairExtensions, convertToCbz = convertToCbz ?: existing.convertToCbz, emptyTrashAfterScan = emptyTrashAfterScan ?: existing.emptyTrashAfterScan, diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryCreationDto.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryCreationDto.kt index ed9542799a..3b8a3f08fb 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryCreationDto.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryCreationDto.kt @@ -21,6 +21,7 @@ data class LibraryCreationDto( val scanCbx: Boolean = true, val scanPdf: Boolean = true, val scanEpub: Boolean = true, + val scanDirectoryExclusions: Set = emptySet(), val repairExtensions: Boolean = false, val convertToCbz: Boolean = false, val emptyTrashAfterScan: Boolean = false, diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryDto.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryDto.kt index 6dd6b51f6c..84cbf6bd5f 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryDto.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryDto.kt @@ -23,6 +23,7 @@ data class LibraryDto( val scanCbx: Boolean, val scanPdf: Boolean, val scanEpub: Boolean, + val scanDirectoryExclusions: Set, val repairExtensions: Boolean, val convertToCbz: Boolean, val emptyTrashAfterScan: Boolean, @@ -54,6 +55,7 @@ fun Library.toDto(includeRoot: Boolean) = LibraryDto( scanCbx = scanCbx, scanPdf = scanPdf, scanEpub = scanEpub, + scanDirectoryExclusions = scanDirectoryExclusions, repairExtensions = repairExtensions, convertToCbz = convertToCbz, emptyTrashAfterScan = emptyTrashAfterScan, diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryUpdateDto.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryUpdateDto.kt index 6521d59d3d..3d25163e9f 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryUpdateDto.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/dto/LibraryUpdateDto.kt @@ -30,6 +30,10 @@ class LibraryUpdateDto { val scanCbx: Boolean? = null val scanPdf: Boolean? = null val scanEpub: Boolean? = null + var scanDirectoryExclusions: Set? + by Delegates.observable(null) { prop, _, _ -> + isSet[prop.name] = true + } val repairExtensions: Boolean? = null val convertToCbz: Boolean? = null diff --git a/komga/src/main/resources/application.yml b/komga/src/main/resources/application.yml index a213d8b997..36a6889ab3 100644 --- a/komga/src/main/resources/application.yml +++ b/komga/src/main/resources/application.yml @@ -14,10 +14,6 @@ logging: komga: libraries-scan-cron: "0 0 */8 * * ?" - libraries-scan-directory-exclusions: - - "#recycle" # Synology - - "@eaDir" # Synology - - "@Recycle" # Qnap database: file: \${komga.config-dir}/database.sqlite lucene: diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt index cfa048583f..1a709d6e60 100644 --- a/komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt @@ -6,7 +6,6 @@ import org.apache.commons.io.FilenameUtils import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.catchThrowable import org.gotson.komga.domain.model.DirectoryNotFoundException -import org.gotson.komga.infrastructure.configuration.KomgaProperties import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments @@ -16,12 +15,7 @@ import java.nio.file.Path import java.util.stream.Stream class FileSystemScannerTest { - - private val komgaProperties = KomgaProperties().apply { - librariesScanDirectoryExclusions = listOf("#recycle") - } - - private val scanner = FileSystemScanner(komgaProperties, emptyList(), emptyList()) + private val scanner = FileSystemScanner(emptyList(), emptyList()) @Test fun `given unavailable root directory when scanning then throw exception`() { @@ -294,7 +288,7 @@ class FileSystemScannerTest { makeSubDir(recycle, "subtrash", listOf("trash2.cbz")) // when - val scan = scanner.scanRootFolder(root).series + val scan = scanner.scanRootFolder(root, directoryExclusions = setOf("#recycle")).series // then assertThat(scan).hasSize(2) diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt index a9c9136d49..e4ff2f7b94 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt @@ -74,6 +74,7 @@ class LibraryDaoTest( scanPdf = false, scanInterval = Library.ScanInterval.DAILY, scanOnStartup = true, + scanDirectoryExclusions = setOf("a", "b"), ) } @@ -111,6 +112,7 @@ class LibraryDaoTest( assertThat(modified.scanPdf).isEqualTo(updated.scanPdf) assertThat(modified.scanInterval).isEqualTo(updated.scanInterval) assertThat(modified.scanOnStartup).isEqualTo(updated.scanOnStartup) + assertThat(modified.scanDirectoryExclusions).containsExactlyInAnyOrderElementsOf(updated.scanDirectoryExclusions) } @Test diff --git a/komga/src/test/kotlin/org/gotson/komga/interfaces/api/rest/LibraryControllerTest.kt b/komga/src/test/kotlin/org/gotson/komga/interfaces/api/rest/LibraryControllerTest.kt index 667895e225..bd0522c5f2 100644 --- a/komga/src/test/kotlin/org/gotson/komga/interfaces/api/rest/LibraryControllerTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/interfaces/api/rest/LibraryControllerTest.kt @@ -5,10 +5,12 @@ import org.gotson.komga.domain.model.ROLE_USER import org.gotson.komga.domain.model.makeLibrary import org.gotson.komga.domain.persistence.LibraryRepository import org.hamcrest.Matchers -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll +import org.hamcrest.Matchers.hasItems +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc import org.springframework.boot.test.context.SpringBootTest @@ -17,7 +19,9 @@ import org.springframework.security.test.context.support.WithAnonymousUser import org.springframework.security.test.context.support.WithMockUser import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.get +import org.springframework.test.web.servlet.patch import org.springframework.test.web.servlet.post +import java.nio.file.Path @SpringBootTest @AutoConfigureMockMvc(printOnlyOnFailure = false) @@ -30,12 +34,12 @@ class LibraryControllerTest( private val library = makeLibrary(path = "file:/library1", id = "1") - @BeforeAll + @BeforeEach fun `setup library`() { libraryRepository.insert(library) } - @AfterAll + @AfterEach fun `teardown library`() { libraryRepository.deleteAll() } @@ -132,4 +136,55 @@ class LibraryControllerTest( } } } + + @Nested + inner class DirectoryExclusions { + @Test + @WithMockCustomUser + fun `given library with exclusions when getting libraries then exclusions are present`() { + libraryRepository.update(library.copy(scanDirectoryExclusions = setOf("test", "value"))) + + mockMvc.get(route) + .andExpect { + status { isOk() } + jsonPath("$[0].scanDirectoryExclusions.length()") { value(2) } + jsonPath("$[0].scanDirectoryExclusions") { hasItems("test", "value") } + } + + mockMvc.get("$route/${library.id}") + .andExpect { + status { isOk() } + jsonPath("$.scanDirectoryExclusions.length()") { value(2) } + jsonPath("$.scanDirectoryExclusions") { hasItems("test", "value") } + } + } + + @Test + @WithMockCustomUser(roles = [ROLE_ADMIN]) + fun `given library with exclusions when updating library then exclusions are updated`(@TempDir tmp: Path) { + libraryRepository.update(library.copy(root = tmp.toUri().toURL(), scanDirectoryExclusions = setOf("test", "value"))) + + // language=JSON + val jsonString = """ + { + "scanDirectoryExclusions": ["updated"] + } + """.trimIndent() + + mockMvc.patch("$route/${library.id}") { + contentType = MediaType.APPLICATION_JSON + content = jsonString + } + .andExpect { + status { isNoContent() } + } + + mockMvc.get("$route/${library.id}") + .andExpect { + status { isOk() } + jsonPath("$.scanDirectoryExclusions.length()") { value(1) } + jsonPath("$.scanDirectoryExclusions") { hasItems("updated") } + } + } + } }