[TEST] jetbrains: idea-oss, pycharm-oss: 2025.1.1.1 -> 2025.3 + fetchurl: builder.sh: handle urls as a Bash array

pkgs/applications/editors/jetbrains/bin/update_bin.py

@@ -13,6 +13,7 @@
 
 updates_url = "https://www.jetbrains.com/updates/updates.xml"
 current_path = pathlib.Path(__file__).parent
+ides_file_path = current_path.joinpath("..").joinpath("ides.json").resolve()
 versions_file_path = current_path.joinpath("versions.json").resolve()
 fromVersions = {}
 toVersions = {}
@@ -72,8 +73,8 @@ def get_url(template, version_or_build_number, version_number):
     return None
 
 
-def update_product(name, product):
-    update_channel = product["update-channel"]
+def update_product(name, ide, version_info):
+    update_channel = ide["updateChannel"]
     logging.info("Updating %s", name)
     channel = channels.get(update_channel)
     if channel is None:
@@ -93,17 +94,17 @@ def update_product(name, product):
             else:
                 version_or_build_number = new_build_number
             version_number = new_version.split(' ')[0]
-            download_url = get_url(product["url-template"], version_or_build_number, version_number)
+            download_url = get_url(version_info["url-template"], version_or_build_number, version_number)
             if not download_url:
                 raise Exception(f"No valid url for {name} version {version_or_build_number}")
-            product["url"] = download_url
-            if "sha256" not in product or product.get("build_number") != new_build_number:
-                fromVersions[name] = product["version"]
+            version_info["url"] = download_url
+            if "sha256" not in version_info or version_info.get("build_number") != new_build_number:
+                fromVersions[name] = version_info["version"]
                 toVersions[name] = new_version
                 logging.info("Found a newer version %s with build number %s.", new_version, new_build_number)
-                product["version"] = new_version
-                product["build_number"] = new_build_number
-                product["sha256"] = download_sha256(download_url)
+                version_info["version"] = new_version
+                version_info["build_number"] = new_build_number
+                version_info["sha256"] = download_sha256(download_url)
             else:
                 logging.info("Already at the latest version %s with build number %s.", new_version, new_build_number)
         except Exception as e:
@@ -112,16 +113,19 @@ def update_product(name, product):
             logging.warning("It may be out-of-date. Fix the error and rerun.")
 
 
-def update_products(products):
-    for name, product in products.items():
-        update_product(name, product)
+def update_products(versioned_products, ides):
+    for name, version_info in versioned_products.items():
+        update_product(name, ides[name], version_info)
 
 
 with open(versions_file_path, "r") as versions_file:
     versions = json.load(versions_file)
 
-for products in versions.values():
-    update_products(products)
+with open(ides_file_path, "r") as ides_file:
+    ides = json.load(ides_file)
+
+for versioned_products in versions.values():
+    update_products(versioned_products, ides)
 
 with open(versions_file_path, "w") as versions_file:
     json.dump(versions, versions_file, indent=2)

pkgs/applications/editors/jetbrains/ides.json

@@ -1,6 +1,7 @@
 {
   "aqua": {
     "product": "Aqua",
+    "updateChannel": "Aqua RELEASE",
     "wmClass": "jetbrains-aqua",
     "meta": {
       "isOpenSource": false,
@@ -12,6 +13,7 @@
   },
   "clion": {
     "product": "CLion",
+    "updateChannel": "CLion RELEASE",
     "wmClass": "jetbrains-clion",
     "meta": {
       "isOpenSource": false,
@@ -23,6 +25,7 @@
   },
   "datagrip": {
     "product": "DataGrip",
+    "updateChannel": "DataGrip RELEASE",
     "wmClass": "jetbrains-datagrip",
     "meta": {
       "isOpenSource": false,
@@ -34,6 +37,7 @@
   },
   "dataspell": {
     "product": "DataSpell",
+    "updateChannel": "DataSpell RELEASE",
     "wmClass": "jetbrains-dataspell",
     "meta": {
       "isOpenSource": false,
@@ -45,6 +49,7 @@
   },
   "gateway": {
     "product": "JetBrains Gateway",
+    "updateChannel": "Gateway RELEASE",
     "productShort": "Gateway",
     "wmClass": "jetbrains-gateway",
     "meta": {
@@ -57,6 +62,7 @@
   },
   "goland": {
     "product": "Goland",
+    "updateChannel": "GoLand RELEASE",
     "wmClass": "jetbrains-goland",
     "meta": {
       "isOpenSource": false,
@@ -68,6 +74,7 @@
   },
   "idea": {
     "product": "IntelliJ IDEA",
+    "updateChannel": "IntelliJ IDEA RELEASE",
     "productShort": "IDEA",
     "wmClass": "jetbrains-idea",
     "meta": {
@@ -80,6 +87,7 @@
   },
   "idea-community": {
     "product": "IntelliJ IDEA CE",
+    "updateChannel": "IntelliJ IDEA RELEASE",
     "productShort": "IDEA",
     "wmClass": "jetbrains-idea-ce",
     "meta": {
@@ -92,6 +100,7 @@
   },
   "idea-oss": {
     "product": "IntelliJ IDEA Open Source",
+    "updateChannel": "IntelliJ IDEA RELEASE",
     "productShort": "IDEA",
     "wmClass": "jetbrains-idea-ce",
     "meta": {
@@ -104,6 +113,7 @@
   },
   "mps": {
     "product": "MPS",
+    "updateChannel": "MPS RELEASE",
     "wmClass": "jetbrains-MPS",
     "meta": {
       "isOpenSource": false,
@@ -115,6 +125,7 @@
   },
   "phpstorm": {
     "product": "PhpStorm",
+    "updateChannel": "PhpStorm RELEASE",
     "wmClass": "jetbrains-phpstorm",
     "meta": {
       "isOpenSource": false,
@@ -127,6 +138,7 @@
   "pycharm": {
     "product": "PyCharm",
     "productShort": "PyCharm",
+    "updateChannel": "PyCharm RELEASE",
     "wmClass": "jetbrains-pycharm",
     "meta": {
       "isOpenSource": false,
@@ -139,6 +151,7 @@
   "pycharm-community": {
     "product": "PyCharm CE",
     "productShort": "PyCharm",
+    "updateChannel": "PyCharm RELEASE",
     "wmClass": "jetbrains-pycharm-ce",
     "meta": {
       "isOpenSource": true,
@@ -151,6 +164,7 @@
   "pycharm-oss": {
     "product": "PyCharm Open Source",
     "productShort": "PyCharm",
+    "updateChannel": "PyCharm RELEASE",
     "wmClass": "jetbrains-pycharm-ce",
     "meta": {
       "isOpenSource": true,
@@ -162,6 +176,7 @@
   },
   "rider": {
     "product": "Rider",
+    "updateChannel": "Rider RELEASE",
     "wmClass": "jetbrains-rider",
     "meta": {
       "isOpenSource": false,
@@ -173,6 +188,7 @@
   },
   "ruby-mine": {
     "product": "RubyMine",
+    "updateChannel": "RubyMine RELEASE",
     "wmClass": "jetbrains-rubymine",
     "meta": {
       "isOpenSource": false,
@@ -184,6 +200,7 @@
   },
   "rust-rover": {
     "product": "RustRover",
+    "updateChannel": "RustRover RELEASE",
     "wmClass": "jetbrains-rustrover",
     "meta": {
       "isOpenSource": false,
@@ -195,6 +212,7 @@
   },
   "webstorm": {
     "product": "WebStorm",
+    "updateChannel": "WebStorm RELEASE",
     "wmClass": "jetbrains-webstorm",
     "meta": {
       "isOpenSource": false,
@@ -206,6 +224,7 @@
   },
   "writerside": {
     "product": "Writerside",
+    "updateChannel": "Writerside EAP",
     "wmClass": "jetbrains-writerside",
     "meta": {
       "isOpenSource": false,

pkgs/applications/editors/jetbrains/patches/bump-jackson-core-in-source.patch

@@ -5,7 +5,7 @@
  )
 
 -private val mavenTelemetryDependencies = listOf("com.fasterxml.jackson.core:jackson-core:2.16.0")
-+private val mavenTelemetryDependencies = listOf("com.fasterxml.jackson.core:jackson-core:2.17.0")
++private val mavenTelemetryDependencies = listOf("com.fasterxml.jackson.core:jackson-core:2.19.0")
 
  object BundledMavenDownloader {
    private val mutex = Mutex()

pkgs/applications/editors/jetbrains/patches/kotlinc-path.patch

@@ -1,7 +1,7 @@
 --- a/src/main/java/org/jetbrains/jpsBootstrap/KotlinCompiler.kt
 +++ b/src/main/java/org/jetbrains/jpsBootstrap/KotlinCompiler.kt
-@@ -14,19 +14,7 @@
-     "https://cache-redirector.jetbrains.com/maven.pkg.jetbrains.space/kotlin/p/kotlin/kotlin-ide-plugin-dependencies"
+@@ -30,25 +30,7 @@
+
 
    fun downloadAndExtractKotlinCompiler(communityRoot: BuildDependenciesCommunityRoot): Path {
 -    // We already have kotlin JPS in the classpath, fetch version from it
@@ -10,12 +10,18 @@
 -    info("Kotlin compiler version is $kotlincVersion")
 -
 -    val kotlincUrl = getUriForMavenArtifact(
--      KOTLIN_IDE_MAVEN_REPOSITORY_URL,
+-      getMavenRepositoryUrl(),
 -      "org.jetbrains.kotlin",
 -      "kotlin-dist-for-ide",
 -      kotlincVersion,
 -      "jar")
--    val kotlincDist = downloadFileToCacheLocation(communityRoot, kotlincUrl)
+-    val kotlincDist = if (shouldUseMavenLocal()) {
+-      val path = kotlincUrl.toPath()
+-      check(path.exists()) { "kotlin-dist-for-ide was not found in the local Maven repository" }
+-      path
+-    } else {
+-      downloadFileToCacheLocation(communityRoot, kotlincUrl)
+-    }
 -    val kotlinc = extractFileToCacheLocation(communityRoot, kotlincDist)
 +    val kotlinc = Path.of("KOTLIN_PATH_HERE")
      verbose("Kotlin compiler is at $kotlinc")

pkgs/applications/editors/jetbrains/patches/no-download.patch

@@ -46,60 +46,58 @@
      return URI.create("${base}/${groupStr}/${artifactId}/${version}/${artifactId}-${version}${classifierStr}.${packaging}")
 --- a/platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/JdkDownloader.kt
 +++ b/platform/build-scripts/downloader/src/org/jetbrains/intellij/build/dependencies/JdkDownloader.kt
-@@ -59,11 +59,7 @@
+@@ -71,9 +71,7 @@
      infoLog: (String) -> Unit,
    ): Path {
      val effectiveVariation = if (isMusl) null else variation
 -    val jdkUrl = getUrl(communityRoot = communityRoot, os = os, arch = arch, isMusl = isMusl, jdkBuildNumber = jdkBuildNumber, variation = effectiveVariation)
--    val jdkArchive = downloadFileToCacheLocation(url = jdkUrl.toString(), communityRoot = communityRoot)
--    val jdkExtracted = BuildDependenciesDownloader.extractFileToCacheLocation(communityRoot = communityRoot,
--                                                                              archiveFile = jdkArchive,
--                                                                              BuildDependenciesExtractOptions.STRIP_ROOT)
+-    val jdkArchive = downloadFileToCacheLocation(url = jdkUrl, communityRoot = communityRoot)
+-    val jdkExtracted = extractFileToCacheLocation(communityRoot = communityRoot, archiveFile = jdkArchive, stripRoot = true)
 +    val jdkExtracted = Path.of("JDK_PATH_HERE")
      val jdkHome = if (os == OS.MACOSX) jdkExtracted.resolve("Contents").resolve("Home") else jdkExtracted
      infoLog("JPS-bootstrap JDK (jdkHome=$jdkHome, executable=${getJavaExecutable(jdkHome)})")
      return jdkHome
 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/LinuxDistributionCustomizer.kt
 +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/LinuxDistributionCustomizer.kt
-@@ -46,7 +46,7 @@
+@@ -62,7 +62,7 @@
    /**
-    * If `true`, a separate *[org.jetbrains.intellij.build.impl.LinuxDistributionBuilder.NO_RUNTIME_SUFFIX].tar.gz artifact without a runtime will be produced.
+    * If `true`, a separate `*[org.jetbrains.intellij.build.impl.LinuxDistributionBuilder.NO_RUNTIME_SUFFIX].tar.gz` artifact without a runtime will be produced.
     */
--  var buildArtifactWithoutRuntime = false
-+  var buildArtifactWithoutRuntime = true
+-  var buildArtifactWithoutRuntime: Boolean = false
++  var buildArtifactWithoutRuntime: Boolean = true
 
    /**
     * Set both properties if a .snap package should be produced.
 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt
 +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/LinuxDistributionBuilder.kt
-@@ -50,7 +50,7 @@
+@@ -76,7 +76,7 @@
          val distBinDir = targetPath.resolve("bin")
          val sourceBinDir = context.paths.communityHomeDir.resolve("bin/linux")
-         addNativeLauncher(distBinDir, targetPath, arch)
+         addNativeLauncher(distBinDir = distBinDir, targetPath = targetPath, arch = arch, context = context)
 -        copyFileToDir(NativeBinaryDownloader.getRestarter(context, OsFamily.LINUX, arch), distBinDir)
 +        copyFileToDir(sourceBinDir.resolve("${arch.dirName}/restarter"), distBinDir)
          copyFileToDir(sourceBinDir.resolve("${arch.dirName}/fsnotifier"), distBinDir)
-         generateBuildTxt(context, targetPath)
-         copyDistFiles(context, targetPath, OsFamily.LINUX, arch)
-@@ -85,6 +85,8 @@
+         generateBuildTxt(context = context, targetDirectory = targetPath)
+         copyDistFiles(context = context, newDir = targetPath, os = OsFamily.LINUX, arch = arch, libcImpl = targetLibcImpl)
+@@ -132,6 +132,8 @@
+           }
          }
        }
-
++      
 +      return@executeStep
-+
-       val runtimeDir = context.bundledRuntime.extract(os = OsFamily.LINUX, arch = arch)
+ 
+       val runtimeDir = context.bundledRuntime.extract(OsFamily.LINUX, arch, targetLibcImpl)
        updateExecutablePermissions(runtimeDir, executableFileMatchers)
-       val tarGzPath = buildTarGz(arch = arch, runtimeDir = runtimeDir, unixDistPath = osAndArchSpecificDistPath, suffix = suffix(arch))
-@@ -383,7 +385,8 @@
-   }
+@@ -422,7 +424,8 @@
+ }
 
-   private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture) {
--    val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch)
-+    val execPath = Path.of("XPLAT_LAUNCHER_PREBUILT_PATH_HERE/Linux-${arch.archName}/xplat-launcher")
-+    val licensePath = Path.of("XPLAT_LAUNCHER_PREBUILT_PATH_HERE/license/xplat-launcher-third-party-licenses.html")
-     copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName))
-     copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html"))
-   }
+ private suspend fun addNativeLauncher(distBinDir: Path, targetPath: Path, arch: JvmArchitecture, context: BuildContext) {
+-  val (execPath, licensePath) = NativeBinaryDownloader.getLauncher(context, OsFamily.LINUX, arch)
++  val execPath = Path.of("XPLAT_LAUNCHER_PREBUILT_PATH_HERE/Linux-${arch.archName}/xplat-launcher")
++  val licensePath = Path.of("XPLAT_LAUNCHER_PREBUILT_PATH_HERE/license/xplat-launcher-third-party-licenses.html")
+   copyFile(execPath, distBinDir.resolve(context.productProperties.baseFileName))
+   copyFile(licensePath, targetPath.resolve("license/launcher-third-party-libraries.html"))
+ }
 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/impl/brokenPlugins.kt
 +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/impl/brokenPlugins.kt
 @@ -9,6 +9,8 @@
@@ -122,20 +120,32 @@
    }
 --- a/platform/build-scripts/src/org/jetbrains/intellij/build/kotlin/KotlinCompilerDependencyDownloader.kt
 +++ b/platform/build-scripts/src/org/jetbrains/intellij/build/kotlin/KotlinCompilerDependencyDownloader.kt
-@@ -23,31 +23,11 @@
+@@ -44,45 +44,11 @@
+
 
- object KotlinCompilerDependencyDownloader {
    fun downloadAndExtractKotlinCompiler(communityRoot: BuildDependenciesCommunityRoot): Path {
 -    val kotlinJpsPluginVersion = getKotlinJpsPluginVersion(communityRoot)
--    val kotlinDistUrl = getUriForMavenArtifact(MAVEN_REPOSITORY_URL, ARTIFACT_GROUP_ID, "kotlin-dist-for-ide", kotlinJpsPluginVersion, "jar")
--    val kotlinDistJar = downloadFileToCacheLocation(communityRoot, kotlinDistUrl)
+-    val kotlinDistUrl = getUriForMavenArtifact(getMavenRepositoryUrl(), ARTIFACT_GROUP_ID, "kotlin-dist-for-ide", kotlinJpsPluginVersion, "jar")
+-    val kotlinDistJar = if (shouldUseMavenLocal()) {
+-      val path = kotlinDistUrl.toPath()
+-      check(path.exists()) { "kotlin-dist-for-ide was not found in the local Maven repository" }
+-      path
+-    } else {
+-      downloadFileToCacheLocation(communityRoot, kotlinDistUrl)
+-    }
 -    return extractFileToCacheLocation(communityRoot, kotlinDistJar)
 +    return Path.of("KOTLIN_PATH_HERE")
    }
 
    suspend fun downloadKotlinJpsPlugin(communityRoot: BuildDependenciesCommunityRoot): Path = withContext(Dispatchers.IO) {
 -    val kotlinJpsPluginVersion = getKotlinJpsPluginVersion(communityRoot)
--    val kotlinJpsPluginUrl = getUriForMavenArtifact(MAVEN_REPOSITORY_URL, ARTIFACT_GROUP_ID, "kotlin-jps-plugin-classpath", kotlinJpsPluginVersion, "jar")
+-    val kotlinJpsPluginUrl = getUriForMavenArtifact(getMavenRepositoryUrl(), ARTIFACT_GROUP_ID, "kotlin-jps-plugin-classpath", kotlinJpsPluginVersion, "jar")
+-
+-    if (shouldUseMavenLocal()) {
+-      val kotlinJpsPluginJar = kotlinJpsPluginUrl.toPath()
+-      check(kotlinJpsPluginJar.exists()) { "kotlin-jps-plugin-classpath was not found in the local Maven repository" }
+-      return@withContext kotlinJpsPluginJar
+-    }
 -
 -    val cacheLocation = getTargetFile(communityRoot, kotlinJpsPluginUrl.toString())
 -    if (cacheLocation.exists()) {
@@ -148,7 +158,9 @@
 -    println(" * Downloading $kotlinJpsPluginUrl")
 -    val tmpLocation = Files.createTempFile(cacheLocation.parent, cacheLocation.name, ".tmp")
 -    retryWithExponentialBackOff {
--      FileUtils.copyURLToFile(kotlinJpsPluginUrl.toURL(), tmpLocation.toFile())
+-      kotlinJpsPluginUrl.toURL().openStream().use {
+-        Files.copy(it, tmpLocation, StandardCopyOption.REPLACE_EXISTING)
+-      }
 -    }
 -    Files.move(tmpLocation, cacheLocation, StandardCopyOption.ATOMIC_MOVE)
 -    return@withContext cacheLocation