Move subtitle files into moving soon via Bazarr (#80)

* [Bazarr] Create base classes

* [Bazarr] Use Bazarr as source for copying subtitles over

* [Build] Build for PR

* [Build] Fix compile, whoopsie

* [Build] Fix docker tag naming for number/merge type "branches"

* [Build] Fix docker tag naming for number/merge type "branches"

* [Build] Upgrade versions

* [Build] Upgrade versions

* [Build] Jib bug fix?

* [Bazarr] Fix method signature

* [Bazarr] HTTP parameters

* [MediaServer] Trace log for extra files

* [Bazarr] Set config defaults

* [Bazarr] Fix parameter naming

* [Bazarr] Expand debug logging

* [Bazarr] Cleanup

* [MediaServer] Add exception to debug logging for link creation

* [MediaServer] Improve extra file copying

* [MediaServer] Fix file copy

* [Build] Force UTF-8 where possible

* [Build] Fix arguments

* [Build] Force some more UTF-8

* [MediaServer] Enable more debug info

* [Build] Add charsets

* [Build] Explicitly use environment variable buildpack

* [Environment] Add some debug info

* [CI] Pass env variables everywhere

* [Build] Don't use multiline string for args

* [Build] Try another way of passing encoding

* [Bazarr] Fix radarr parameter

* [Build] Try another way of passing encoding

* [File-API] Temporarily switch to experimental Kotlin API while GraalVM and paketo builders are bugged

.github/workflows/gradle.yml

@@ -22,19 +22,39 @@ on:
     branches:
       - 'develop'
 
+  pull_request:
+    paths:
+      - src/**
+      - buildSrc/**
+      - .github/**
+      - build.gradle.kts
+      - settings.gradle.kts
+    branches:
+      - '**'
+
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
       - name: Set up JDK 23
         uses: actions/setup-java@v4
         with:
           java-version: '23'
           distribution: 'temurin'
           cache: 'gradle'
 
+      - name: Export branch name
+        uses: mad9000/actions-find-and-replace-string@5
+        id: branch_name
+        with:
+          source: ${{ github.ref_name }}
+          find: '/'
+          replace: '-'
+
       - name: Log in to Docker Hub to pull images without rate limit
         uses: docker/login-action@v3
         with:
@@ -51,11 +71,13 @@ jobs:
       - name: Setup Gradle for a non-wrapper project
         uses: gradle/actions/setup-gradle@v4
         with:
-          gradle-version: 8.10.1
+          gradle-version: 8.10.2
 
       - name: Build JVM OCI image
-        run: gradle jib
+        run: gradle jib -Djib.serialize=true # https://github.com/GoogleContainerTools/jib/issues/4301
         env:
+          LANG: "en_US.UTF-8"
+          LC_ALL: "en_US.UTF-8"
           USERNAME: ${{ github.actor }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -64,6 +86,9 @@ jobs:
       - name: Build Native OCI Image
         run: gradle bootBuildImage --publishImage --imagePlatform linux/amd64
         env:
+          LANG: "en_US.UTF-8"
+          LC_ALL: "en_US.UTF-8"
+          NATIVE_IMAGE_OPTIONS: "-Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8"
           USERNAME: ${{ github.actor }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -72,6 +97,9 @@ jobs:
 
       - name: Setup QEMU for ARM64 OCI Image
         uses: docker/setup-qemu-action@v3
+        env:
+          LANG: "en_US.UTF-8"
+          LC_ALL: "en_US.UTF-8"
         with:
           platforms: 'all'
 
@@ -84,6 +112,8 @@ jobs:
       - name: Build Native ARM64 OCI Image
         run: gradle bootBuildImage --publishImage --imagePlatform linux/arm64
         env:
+          LANG: "en_US.UTF-8"
+          LC_ALL: "en_US.UTF-8"
           USERNAME: ${{ github.actor }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -95,9 +125,9 @@ jobs:
         with:
           push: true
           tags: | 
-            ghcr.io/schaka/janitorr:native-${{ github.ref_name }}
+            ghcr.io/schaka/janitorr:native-${{ steps.branch_name.outputs.value }}
             ${{ (startsWith(github.ref, 'refs/tags/v') && 'ghcr.io/schaka/janitorr:native-stable') || '' }}
           sources: | 
-            ghcr.io/schaka/janitorr:native-amd64-${{ github.ref_name }}
-            ghcr.io/schaka/janitorr:native-arm64-${{ github.ref_name }}
+            ghcr.io/schaka/janitorr:native-amd64-${{ steps.branch_name.outputs.value }}
+            ghcr.io/schaka/janitorr:native-arm64-${{ steps.branch_name.outputs.value }}
           

build.gradle.kts

@@ -3,6 +3,7 @@ import net.nemerosa.versioning.VersioningExtension
 import org.gradle.plugins.ide.idea.model.IdeaModel
 import org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_22
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.resolve.compatibility
 import org.springframework.boot.gradle.dsl.SpringBootExtension
 import org.springframework.boot.gradle.tasks.aot.ProcessAot
 import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
@@ -98,7 +99,7 @@ configure<VersioningExtension> {
 extra {
     val build = getBuild()
     val versioning: VersioningExtension = extensions.getByName<VersioningExtension>("versioning")
-    val branch = versioning.info.branch
+    val branch = versioning.info.branch.replace("/", "-")
     val shortCommit = versioning.info.commit.take(8)
 
     project.extra["build.date-time"] = build.buildDateAndTime
@@ -133,13 +134,19 @@ extra {
 tasks.withType<BootRun> {
     jvmArgs(
         arrayOf(
-            "-Dspring.config.additional-location=optional:file:/config/application.yaml,optional:file:/workspace/application.yaml"
+            "-Dspring.config.additional-location=optional:file:/config/application.yaml,optional:file:/workspace/application.yaml",
+            "-Dsun.jnu.encoding=UTF-8",
+            "-Dfile.encoding=UTF-8"
         )
     )
 }
 
 tasks.withType<ProcessAot> {
-    args("-Dspring.config.additional-location=optional:file:/config/application.yaml,optional:file:/workspace/application.yaml")
+    args(
+        "-Dspring.config.additional-location=optional:file:/config/application.yaml,optional:file:/workspace/application.yaml",
+        "-Dsun.jnu.encoding=UTF-8",
+        "-Dfile.encoding=UTF-8"
+    )
 }
 
 tasks.withType<BootBuildImage> {
@@ -149,7 +156,7 @@ tasks.withType<BootBuildImage> {
     docker.publishRegistry.password = System.getenv("GITHUB_TOKEN") ?: "INVALID_PASSWORD"
 
     builder = "paketobuildpacks/builder-jammy-buildpackless-tiny"
-    buildpacks = listOf("paketobuildpacks/java-native-image", "paketobuildpacks/health-checker")
+    buildpacks = listOf("paketobuildpacks/environment-variables", "paketobuildpacks/java-native-image", "paketobuildpacks/health-checker")
     imageName = project.extra["native.image.name"] as String
     version = project.extra["docker.image.version"] as String
     tags = project.extra["native.image.tags"] as List<String>
@@ -161,10 +168,12 @@ tasks.withType<BootBuildImage> {
         "BPL_SPRING_AOT_ENABLED" to "true",
         "BP_HEALTH_CHECKER_ENABLED" to "true",
         "BP_JVM_CDS_ENABLED" to "true",
-        "BP_JVM_VERSION" to "22",
-        "BP_NATIVE_IMAGE_BUILD_ARGUMENTS" to """
-            -march=compatibility
-        """.trimIndent()
+        "BP_JVM_VERSION" to "23",
+        "BPE_NATIVE_IMAGE_OPTIONS" to "-Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8",
+        "BPE_LANG" to "en_US.UTF-8",
+        "BPE_LANGUAGE" to "LANGUAGE=en_US:en",
+        "BPE_LC_ALL" to "en_US.UTF-8",
+        "BP_NATIVE_IMAGE_BUILD_ARGUMENTS" to "-march=compatibility -H:+AddAllCharsets -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8"
     )
 }
 
@@ -179,7 +188,7 @@ jib {
         }
     }
     from {
-        image = "eclipse-temurin:22-jre-jammy"
+        image = "eclipse-temurin:23-jre-noble"
         auth {
             username = System.getenv("DOCKERHUB_USER")
             password = System.getenv("DOCKERHUB_PASSWORD")
@@ -196,7 +205,12 @@ jib {
         }
     }
     container {
-        jvmFlags = listOf("-Dspring.config.additional-location=optional:file:/config/application.yaml", "-Xms256m")
+        jvmFlags = listOf(
+            "-Dspring.config.additional-location=optional:file:/config/application.yaml",
+            "-Dsun.jnu.encoding=UTF-8",
+            "-Dfile.encoding=UTF-8",
+            "-Xms256m",
+        )
         mainClass = "com.github.schaka.janitorr.JanitorrApplicationKt"
         ports = listOf("8978")
         format = ImageFormat.Docker // OCI not yet supported

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

src/main/kotlin/com/github/schaka/janitorr/JanitorrApplication.kt

@@ -5,7 +5,9 @@ import com.github.schaka.janitorr.jellystat.JellystatClient
 import com.github.schaka.janitorr.mediaserver.MediaServerClient
 import com.github.schaka.janitorr.mediaserver.MediaServerUserClient
 import com.github.schaka.janitorr.mediaserver.emby.EmbyMediaServerClient
+import com.github.schaka.janitorr.servarr.RestClientProperties
 import com.github.schaka.janitorr.servarr.ServarrService
+import com.github.schaka.janitorr.servarr.bazarr.BazarrClient
 import com.github.schaka.janitorr.servarr.radarr.RadarrClient
 import com.github.schaka.janitorr.servarr.sonarr.SonarrClient
 import org.springframework.aot.hint.RuntimeHints
@@ -37,7 +39,9 @@ class JanitorrApplication {
             hints.proxies().registerJdkProxy(MediaServerUserClient::class.java)
             hints.proxies().registerJdkProxy(RadarrClient::class.java)
             hints.proxies().registerJdkProxy(SonarrClient::class.java)
+            hints.proxies().registerJdkProxy(BazarrClient::class.java)
             hints.proxies().registerJdkProxy(ServarrService::class.java)
+            hints.proxies().registerJdkProxy(RestClientProperties::class.java)
         }
     }
 }

src/main/kotlin/com/github/schaka/janitorr/cleanup/MediaCleanupSchedule.kt

@@ -9,6 +9,7 @@ import com.github.schaka.janitorr.mediaserver.library.LibraryType
 import com.github.schaka.janitorr.mediaserver.library.LibraryType.MOVIES
 import com.github.schaka.janitorr.mediaserver.library.LibraryType.TV_SHOWS
 import com.github.schaka.janitorr.servarr.ServarrService
+import com.github.schaka.janitorr.servarr.bazarr.BazarrRestService
 import com.github.schaka.janitorr.servarr.radarr.Radarr
 import com.github.schaka.janitorr.servarr.radarr.RadarrRestService
 import com.github.schaka.janitorr.servarr.sonarr.Sonarr
@@ -34,8 +35,9 @@ class MediaCleanupSchedule(
         private val log = LoggerFactory.getLogger(this::class.java.enclosingClass)
     }
 
+
     // run every hour
-    @CacheEvict(cacheNames = [SonarrRestService.CACHE_NAME, RadarrRestService.CACHE_NAME])
+    @CacheEvict(cacheNames = [SonarrRestService.CACHE_NAME, RadarrRestService.CACHE_NAME, BazarrRestService.CACHE_NAME_TV, BazarrRestService.CACHE_NAME_MOVIES])
     @Scheduled(fixedDelay = 1000 * 60 * 60)
     fun runSchedule() {
 

src/main/kotlin/com/github/schaka/janitorr/cleanup/TagBasedCleanupSchedule.kt

@@ -11,6 +11,7 @@ import com.github.schaka.janitorr.mediaserver.library.LibraryType.MOVIES
 import com.github.schaka.janitorr.mediaserver.library.LibraryType.TV_SHOWS
 import com.github.schaka.janitorr.servarr.LibraryItem
 import com.github.schaka.janitorr.servarr.ServarrService
+import com.github.schaka.janitorr.servarr.bazarr.BazarrRestService
 import com.github.schaka.janitorr.servarr.radarr.Radarr
 import com.github.schaka.janitorr.servarr.radarr.RadarrRestService
 import com.github.schaka.janitorr.servarr.sonarr.Sonarr
@@ -38,7 +39,7 @@ class TagBasedCleanupSchedule(
     }
 
     // run every hour
-    @CacheEvict(cacheNames = [SonarrRestService.CACHE_NAME, RadarrRestService.CACHE_NAME])
+    @CacheEvict(cacheNames = [SonarrRestService.CACHE_NAME, RadarrRestService.CACHE_NAME, BazarrRestService.CACHE_NAME_TV, BazarrRestService.CACHE_NAME_MOVIES])
     @Scheduled(fixedDelay = 1000 * 60 * 60)
     fun runSchedule() {
 

src/main/kotlin/com/github/schaka/janitorr/cleanup/WeeklyEpisodeCleanupSchedule.kt

@@ -1,6 +1,7 @@
 package com.github.schaka.janitorr.cleanup
 
 import com.github.schaka.janitorr.config.ApplicationProperties
+import com.github.schaka.janitorr.servarr.bazarr.BazarrRestService
 import com.github.schaka.janitorr.servarr.data_structures.Tag
 import com.github.schaka.janitorr.servarr.history.HistoryResponse
 import com.github.schaka.janitorr.servarr.radarr.RadarrRestService
@@ -40,7 +41,7 @@ class WeeklyEpisodeCleanupSchedule(
     }
 
     // run every hour
-    @CacheEvict(cacheNames = [SonarrRestService.CACHE_NAME, RadarrRestService.CACHE_NAME])
+    @CacheEvict(cacheNames = [SonarrRestService.CACHE_NAME, RadarrRestService.CACHE_NAME, BazarrRestService.CACHE_NAME_TV, BazarrRestService.CACHE_NAME_MOVIES])
     @Scheduled(fixedDelay = 1000 * 60 * 60)
     fun runSchedule() {
 

src/main/kotlin/com/github/schaka/janitorr/config/RuntimeEnvironment.kt

@@ -0,0 +1,29 @@
+package com.github.schaka.janitorr.config
+
+import jakarta.annotation.PostConstruct
+import org.slf4j.LoggerFactory
+import org.springframework.core.Ordered
+import org.springframework.core.annotation.Order
+import org.springframework.stereotype.Component
+import java.nio.charset.Charset
+
+@Order(Ordered.HIGHEST_PRECEDENCE)
+@Component
+class RuntimeEnvironment {
+    companion object {
+        private val log = LoggerFactory.getLogger(this::class.java.enclosingClass)
+    }
+
+
+    @PostConstruct
+    fun init() {
+        log.info("Default charset {}", Charset.defaultCharset().displayName())
+        log.info("sun.jnu.encoding {}", System.getProperty("sun.jnu.encoding"))
+        log.info("sun.stdout.encoding {}", System.getProperty("sun.stdout.encoding"))
+        log.info("sun.stderr.encoding {}", System.getProperty("sun.stderr.encoding"))
+        log.info("ENV JAVA_TOOL_OPTIONS {}", System.getenv("JAVA_TOOL_OPTIONS"))
+        log.info("ENV LANG {}", System.getenv("LANG"))
+        log.info("ENV LANGUAGE {}", System.getenv("LANGUAGE"))
+        log.info("ENV LC_ALL {}", System.getenv("LC_ALL"))
+    }
+}

src/main/kotlin/com/github/schaka/janitorr/mediaserver/AbstractMediaServerService.kt

@@ -4,12 +4,13 @@ import com.github.schaka.janitorr.cleanup.CleanupType
 import com.github.schaka.janitorr.jellystat.JellystatProperties
 import com.github.schaka.janitorr.mediaserver.filesystem.PathStructure
 import com.github.schaka.janitorr.mediaserver.library.LibraryType
-import com.github.schaka.janitorr.mediaserver.library.VirtualFolderResponse
 import com.github.schaka.janitorr.servarr.LibraryItem
 import org.slf4j.LoggerFactory
 import org.springframework.util.FileSystemUtils
 import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.StandardCopyOption
+import kotlin.io.path.Path
 import kotlin.io.path.exists
 import kotlin.io.path.listDirectoryEntries
 
@@ -47,7 +48,7 @@ abstract class AbstractMediaServerService {
         }
     }
 
-    protected fun createSymLink(source: Path, target: Path, type: String) {
+    private fun createSymLink(source: Path, target: Path, type: String) {
         if (!Files.exists(target)) {
             log.debug("Creating {} link from {} to {}", type, source, target)
             Files.createSymbolicLink(target, source)
@@ -56,13 +57,24 @@ abstract class AbstractMediaServerService {
         }
     }
 
+    private fun copyExtraFiles(files: List<String>, target: Path) {
+         // TODO: files already contain the full path, consider only adding the filename to an existing source (folder)
+        for (filePath in files) {
+            val source = Path(filePath)
+            val targetFolder = target.parent
+            val targetFile = targetFolder.resolve(source.fileName)
+            Files.copy(source, targetFile, StandardCopyOption.REPLACE_EXISTING)
+            log.debug("Copying extra files from {} to {}", filePath, targetFile)
+        }
+    }
+
     internal fun pathStructure(it: LibraryItem, leavingSoonParentPath: Path): PathStructure {
-        val rootPath = Path.of(it.rootFolderPath)
-        val itemFilePath = Path.of(it.filePath)
+        val rootPath = Path(it.rootFolderPath)
+        val itemFilePath = Path(it.filePath)
         val itemFolderName = itemFilePath.subtract(rootPath).firstOrNull()
 
-        val fileOrFolder =
-            itemFilePath.subtract(Path.of(it.parentPath)).firstOrNull() // contains filename and folder before it e.g. (Season 05) (ShowName-Episode01.mkv) or MovieName2013.mkv
+        // contains filename and folder before it e.g. (Season 05) (ShowName-Episode01.mkv) or MovieName2013.mkv
+        val fileOrFolder = itemFilePath.subtract(Path(it.parentPath)).firstOrNull()
 
         val sourceFolder = rootPath.resolve(itemFolderName)
         val sourceFile = sourceFolder.resolve(fileOrFolder)
@@ -74,7 +86,7 @@ abstract class AbstractMediaServerService {
     }
 
     fun cleanupPath(leavingSoonDir: String, libraryType: LibraryType, cleanupType: CleanupType) {
-        val path = Path.of(leavingSoonDir, libraryType.folderName, cleanupType.folderName)
+        val path = Path(leavingSoonDir, libraryType.folderName, cleanupType.folderName)
         FileSystemUtils.deleteRecursively(path)
         Files.createDirectories(path)
     }
@@ -105,6 +117,7 @@ abstract class AbstractMediaServerService {
                             val source = sourceSeasonFolder.resolve(fileName)
                             val target = targetSeasonFolder.resolve(fileName)
                             createSymLink(source, target, "episode")
+                            copyExtraFiles(it.extraFiles, target)
                         }
 
                     } else {
@@ -119,12 +132,17 @@ abstract class AbstractMediaServerService {
                         val target = structure.targetFile
                         Files.createDirectories(structure.targetFolder)
                         createSymLink(source, target, "movie")
+                        copyExtraFiles(it.extraFiles, target)
                     } else {
                         log.info("Can't find original movie folder - no links to create {}", source)
                     }
                 }
             } catch (e: Exception) {
-                log.error("Couldn't find path {}", it.parentPath)
+                if (log.isDebugEnabled){
+                    log.error("Couldn't find path {} - {}", it.parentPath, it, e)
+                } else {
+                    log.error("Couldn't find path {}", it.parentPath)
+                }
             }
         }
     }