Implement multiple project & file requests for Modrinth

src/main/kotlin/teksturepako/pakku/api/platforms/CurseForge.kt

@@ -204,23 +204,17 @@ object CurseForge : Platform(
             .toMutableSet()
     }
 
-    suspend fun requestMultipleProjectsWithFiles(
-        mcVersions: List<String>, loaders: List<String>, ids: List<String>
+    override suspend fun requestMultipleProjectsWithFiles(
+        mcVersions: List<String>, loaders: List<String>, ids: List<String>, numberOfFiles: Int
     ): MutableSet<Project>
     {
-        val gameVersionTypeIds = mcVersions.mapNotNull { version: String ->
-            requestGameVersionTypeId(version)
-        }
-
         val response = json.decodeFromString<GetMultipleProjectsResponse>(
             this.requestProjectBody("mods", MultipleProjectsRequest(ids.map(String::toInt)))
                 ?: return mutableSetOf()
         ).data
 
         val fileIds = response.flatMap { model ->
-            model.latestFilesIndexes
-                .filter { it.gameVersionTypeId in gameVersionTypeIds }
-                .map { it.fileId.toString() }
+            model.latestFilesIndexes.map { it.fileId.toString() }
         }
 
         val projectFiles = requestMultipleProjectFiles(mcVersions, loaders, fileIds)
@@ -236,7 +230,7 @@ object CurseForge : Platform(
             }
         }
 
-        return projects.toMutableSet()
+        return projects.map { it.apply { files = files.take(numberOfFiles).toMutableSet() } }.toMutableSet()
     }
 
     suspend fun requestGameVersionTypeId(mcVersion: String): Int?
@@ -248,7 +242,6 @@ object CurseForge : Platform(
     }
 
     fun ProjectFile.fetchAlternativeDownloadUrl(): ProjectFile =
-        // Request alternative download URL.
         if (this.url != "null") this else
         {
             val url = fetchAlternativeDownloadUrl(this.id, this.fileName)

src/main/kotlin/teksturepako/pakku/api/platforms/Modrinth.kt

@@ -26,6 +26,7 @@ object Modrinth : Platform(
     // -- API RATE LIMIT --
 
     private var requestsRemaining = 0
+    private var waiting = false
 
     override suspend fun HttpResponse.checkLimit(): HttpResponse
     {
@@ -41,7 +42,8 @@ object Modrinth : Platform(
                 }
                 rateLimit < 100 ->
                 {
-                    print("Warning: Waiting because of height Modrinth API usage")
+                    if (!waiting) println("Warning: Waiting 30 seconds, because of height Modrinth API usage")
+                    waiting = true
                     delay(30.seconds)
                 }
             }
@@ -102,7 +104,7 @@ object Modrinth : Platform(
                 ids.map { "%22$it%22" }.toString()
                     .replace("[", "%5B")
                     .replace("]","%5D")
-            }") ?: return mutableSetOf()
+            }".replace(" ", "")) ?: return mutableSetOf()
         ).mapNotNull { it.toProject() }.toMutableSet()
     }
 
@@ -180,10 +182,40 @@ object Modrinth : Platform(
                 ids.map { "%22$it%22" }.toString()
                     .replace("[", "%5B")
                     .replace("]","%5D")
-            }") ?: return mutableSetOf()
+            }".replace(" ", "")) ?: return mutableSetOf()
         )
             .filterFileModels(mcVersions, loaders)
             .flatMap { version -> version.toProjectFiles() }
             .toMutableSet()
     }
+
+    override suspend fun requestMultipleProjectsWithFiles(
+        mcVersions: List<String>, loaders: List<String>, ids: List<String>, numberOfFiles: Int
+    ): MutableSet<Project>
+    {
+        val response = json.decodeFromString<List<MrProjectModel>>(
+            this.requestProjectBody("projects?ids=${
+                ids.map { "%22$it%22" }.toString()
+                    .replace("[", "%5B")
+                    .replace("]","%5D")
+            }".replace(" ", "")) ?: return mutableSetOf()
+        )
+
+        val fileIds = response.flatMap { it.versions }
+
+        val projectFiles = requestMultipleProjectFiles(mcVersions, loaders, fileIds)
+
+        val projects = response.mapNotNull { it.toProject() }
+
+        projects.forEach { project ->
+            projectFiles.forEach { projectFile ->
+                if (project.id[this.serialName] == projectFile.parentId)
+                {
+                    project.files.add(projectFile)
+                }
+            }
+        }
+
+        return projects.map { it.apply { files = files.take(numberOfFiles).toMutableSet() } }.toMutableSet()
+    }
 }

src/main/kotlin/teksturepako/pakku/api/platforms/Multiplatform.kt

@@ -67,4 +67,24 @@ object Multiplatform : IProjectProvider
 
         return project
     }
+
+    suspend fun requestMultipleProjectsWithFiles(
+        mcVersions: List<String>, loaders: List<String>, ids: Map<String, List<String>>, numberOfFiles: Int
+    ): MutableSet<Project>
+    {
+        val result: MutableSet<Project> = mutableSetOf()
+
+        val projectsCf = CurseForge.requestMultipleProjectsWithFiles(mcVersions, loaders, ids[CurseForge.serialName]!!, numberOfFiles)
+        val projectsMr = Modrinth.requestMultipleProjectsWithFiles(mcVersions, loaders, ids[Modrinth.serialName]!!, numberOfFiles)
+
+        for (projectCf in projectsCf)
+        {
+            for (projectMr in projectsMr)
+            {
+                if (projectCf isAlmostTheSameAs projectMr) result += projectCf + projectMr
+            }
+        }
+
+        return result
+    }
 }

src/main/kotlin/teksturepako/pakku/api/platforms/Platform.kt

@@ -90,4 +90,8 @@ abstract class Platform(
             files.addAll(requestProjectFiles(mcVersions, loaders, projectId, fileId).take(numberOfFiles))
         }
     }
+
+    abstract suspend fun requestMultipleProjectsWithFiles(
+        mcVersions: List<String>, loaders: List<String>, ids: List<String>, numberOfFiles: Int
+    ): MutableSet<Project>
 }

src/main/kotlin/teksturepako/pakku/api/projects/Project.kt

@@ -28,7 +28,7 @@ data class Project(
     val slug: MutableMap<String, String>,
     val name: MutableMap<String, String>,
     val id: MutableMap<String, String>,
-    val files: MutableSet<ProjectFile>
+    var files: MutableSet<ProjectFile>
 )
 {
     /**

src/main/kotlin/teksturepako/pakku/api/projects/ProjectFile.kt

@@ -15,8 +15,8 @@ abstract class IProjectFile
     open var url: String? = null
     open val requiredDependencies: MutableSet<String>? = null
 
-    abstract val id: String
-    abstract val parentId: String
+    open val id: String = ""
+    open val parentId: String = ""
 }
 
 @Serializable
@@ -34,7 +34,7 @@ data class MrFile(
     override var url: String?,
 
     override val id: String,
-    override val parentId: String,
+    @SerialName("parent_id") override val parentId: String,
 
     val hashes: MutableMap<String, String>? = null,
     @SerialName("required_dependencies") override val requiredDependencies: MutableSet<String>?,
@@ -50,7 +50,7 @@ data class CfFile(
     override var url: String?,
 
     override val id: String,
-    override val parentId: String,
+    @SerialName("parent_id") override val parentId: String,
 
     @SerialName("required_dependencies") override val requiredDependencies: MutableSet<String>?,
 ) : ProjectFile(CurseForge.serialName)

src/main/kotlin/teksturepako/pakku/cli/ResolveDependencies.kt

@@ -16,7 +16,8 @@ suspend fun Project.resolveDependencies(
     reqHandlers: RequestHandlers,
     pakkuLock: PakkuLock,
     projectProvider: IProjectProvider,
-    platforms: List<Platform>
+    platforms: List<Platform>,
+    addAsProjects: Boolean = true
 )
 {
     val dependencies = this.requestDependencies(projectProvider, pakkuLock)
@@ -32,7 +33,7 @@ suspend fun Project.resolveDependencies(
             pakkuLock.getProject(dependencyIn)?.pakkuId?.let { pakkuId ->
                 pakkuLock.addPakkuLink(pakkuId, this)
             }
-        } else
+        } else if (addAsProjects)
         {
             /** Add dependency as a regular project and resolve dependencies for it too */
             debug { terminal.info(dependencyIn.toPrettyString()) }
@@ -41,9 +42,13 @@ suspend fun Project.resolveDependencies(
                 onRetry = reqHandlers.onRetry,
                 onSuccess = { dependency, _, depReqHandlers ->
                     runBlocking {
+                        /** Add dependency */
                         pakkuLock.add(dependency)
+
+                        /** Link dependency to parent project */
                         pakkuLock.addPakkuLink(dependency.pakkuId!!, this@resolveDependencies)
 
+                        /** Resolve dependencies for dependency */
                         dependency.resolveDependencies(terminal, depReqHandlers, pakkuLock, projectProvider, platforms)
                         terminal.info("${dependency.slug} added")
                     }

src/main/kotlin/teksturepako/pakku/cli/cmd/Import.kt

@@ -4,6 +4,8 @@ import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.core.terminal
 import com.github.ajalt.clikt.parameters.arguments.argument
 import kotlinx.coroutines.async
+import kotlinx.coroutines.joinAll
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import teksturepako.pakku.api.actions.Error
 import teksturepako.pakku.api.actions.createAdditionRequest
@@ -37,34 +39,29 @@ class Import : CliktCommand("Import modpack")
 
         projects.await().forEach { project ->
             files.await().forEach { file ->
-                project.apply { if (id[CurseForge.serialName] == file.parentId) this.files.add(file)}
+                if (project.id[CurseForge.serialName] == file.parentId) project.files.add(file)
             }
         }
 
-        projects.await().forEach { projectIn ->
-            projectIn.createAdditionRequest(
-                onError = { error ->
-                    if (error !is Error.CouldNotAdd) terminal.danger(error)
-                },
-                onRetry = { platform -> promptForProject(platform, terminal, pakkuLock) },
-                onSuccess = { project, _, reqHandlers ->
-                    runBlocking {
-                        pakkuLock.add(project)
-                        project.resolveDependencies(
-                            terminal = terminal,
-                            reqHandlers = reqHandlers,
-                            pakkuLock = pakkuLock,
-                            projectProvider = CurseForge,
-                            platforms = listOf(CurseForge)
-                        )
-                        terminal.success("${project.slug} added")
-                    }
-                },
-                pakkuLock = pakkuLock,
-                platforms = listOf(CurseForge)
-            )
-            terminal.println()
-        }
+        projects.await().map { projectIn ->
+            launch {
+                projectIn.createAdditionRequest(
+                    onError = { error ->
+                        if (error !is Error.CouldNotAdd) terminal.danger(error.message)
+                    },
+                    onRetry = { platform -> promptForProject(platform, terminal, pakkuLock) },
+                    onSuccess = { project, _, reqHandlers ->
+                        runBlocking {
+                            pakkuLock.add(project)
+                            project.resolveDependencies(terminal, reqHandlers, pakkuLock, CurseForge, listOf(CurseForge))
+                            terminal.success("${project.slug} added")
+                        }
+                    },
+                    pakkuLock,
+                    listOf(CurseForge)
+                )
+            }
+        }.joinAll()
 
         pakkuLock.write()
     }

src/main/kotlin/teksturepako/pakku/cli/cmd/Ls.kt

@@ -9,7 +9,14 @@ import teksturepako.pakku.api.platforms.Multiplatform
 class Ls : CliktCommand("List projects")
 {
     override fun run() = runBlocking {
-        val pakkuLock = PakkuLock.readOrNew()
+        val pakkuLock = PakkuLock.readToResult().fold(
+            onSuccess = { it },
+            onFailure = {
+                terminal.danger(it.message)
+                echo()
+                return@runBlocking
+            }
+        )
 
         for (project in pakkuLock.getAllProjects())
         {

src/main/kotlin/teksturepako/pakku/cli/cmd/Update.kt

@@ -10,7 +10,7 @@ import kotlinx.coroutines.*
 import kotlinx.coroutines.channels.ReceiveChannel
 import kotlinx.coroutines.channels.produce
 import teksturepako.pakku.api.data.PakkuLock
-import teksturepako.pakku.api.platforms.CurseForge
+import teksturepako.pakku.api.platforms.Multiplatform
 import teksturepako.pakku.api.projects.Project
 
 class Update : CliktCommand("Update projects")
@@ -45,22 +45,29 @@ class Update : CliktCommand("Update projects")
         }
 
         val updatedProjects: ReceiveChannel<Project> = produce {
-            val oldProjectIds = oldProjects.mapNotNull { it.id[CurseForge.serialName] }
+            val oldProjectIds = oldProjects.asSequence()
+                .flatMap {
+                    it.id.asSequence()
+                }.groupBy({ it.key }, { it.value })
 
-            val newProjects = CurseForge.requestMultipleProjectsWithFiles(
-                pakkuLock.getMcVersions(), pakkuLock.getLoaders(), oldProjectIds
+            val newProjects = Multiplatform.requestMultipleProjectsWithFiles(
+                pakkuLock.getMcVersions(), pakkuLock.getLoaders(), oldProjectIds, numberOfFiles = 1
             )
 
             oldProjects.map { oldProject ->
                 launch {
                     var updatedProject = oldProject.copy(files = mutableSetOf())
 
+                    // Handle versions
+                    val mcVersions = oldProject.files.flatMap { it.mcVersions }
+                    val loaders = oldProject.files.flatMap { it.loaders }
+
                     newProjects.forEach { newProject ->
                         if (newProject isAlmostTheSameAs oldProject)
                         {
                             updatedProject += newProject
 
-                            if (updatedProject != oldProject)
+                            if (updatedProject != oldProject && mcVersions.isNotEmpty() && loaders.isNotEmpty())
                             {
                                 send(updatedProject) // Send updated project to channel
                             }
@@ -98,16 +105,18 @@ class Update : CliktCommand("Update projects")
         }
 
         var updated = false
+        val jobs = mutableListOf<Job>()
 
         for (updatedProject in updatedProjects)
         {
-            launch {
+            jobs += launch {
                 pakkuLock.update(updatedProject)
                 terminal.success("${updatedProject.slug} updated")
 
                 updated = true
             }
         }
+        jobs.joinAll()
 
         if (!updated && updatedProjects.isClosedForReceive && oldProjects.isNotEmpty())
         {