[Native][tests] Properly compute "used libraries" to be written to ma…

…nifest

We need to write to `depends=` manifest property in KLIBs produced
in Kotlin/Native tests the same set of dependencies as if a KLIB
would have been compiled by calling the compiler via the command-line
interface.

For more details, see comments in `Fir2IrNativeResultsConverter` and
`IrBackendInput.NativeBackendInput`.

^KT-71333

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt

@@ -15,6 +15,7 @@ import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
 import org.jetbrains.kotlin.ir.backend.js.IrModuleInfo
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.util.KotlinMangler
+import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.test.model.BackendKind
 import org.jetbrains.kotlin.test.model.BackendKinds
 import org.jetbrains.kotlin.test.model.ResultingArtifact
@@ -134,6 +135,12 @@ sealed class IrBackendInput : ResultingArtifact.BackendInput<IrBackendInput>() {
      * Note: For the classic frontend both [firMangler] and [metadataSerializer] are null.
      * The latter is because the Native backend uses
      * [org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer] which serializes a whole module.
+     *
+     * @property usedLibrariesForManifest - The list of dependency libraries that should be written to the produced KLIB
+     *   manifest's `depends=` property. This list includes:
+     *   - direct dependencies (ones that were explicitly specified by `// MODULE` test directives in test data)
+     *   - and "default" dependencies (anything that is implicitly added by the Kotlin/Native compiler, ex: stdlib & platform libraries),
+     *     BUT only if such libraries were actually used during the compilation.
      */
     class NativeBackendInput(
         override val irModuleFragment: IrModuleFragment,
@@ -142,5 +149,6 @@ sealed class IrBackendInput : ResultingArtifact.BackendInput<IrBackendInput>() {
         override val descriptorMangler: KotlinMangler.DescriptorMangler?,
         override val irMangler: KotlinMangler.IrMangler,
         val metadataSerializer: KlibSingleFileMetadataSerializer<*>?,
+        val usedLibrariesForManifest: List<KotlinLibrary>,
     ) : IrBackendInput()
 }

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/configuration/NativeEnvironmentConfigurator.kt

@@ -5,6 +5,8 @@
 
 package org.jetbrains.kotlin.test.services.configuration
 
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.TargetSupportException
 import org.jetbrains.kotlin.test.directives.ConfigurationDirectives
 import org.jetbrains.kotlin.test.model.TestModule
@@ -19,6 +21,9 @@ class NativeEnvironmentConfigurator(testServices: TestServices) : EnvironmentCon
             get() = System.getProperty("kotlin.internal.native.test.nativeHome")
                 ?: error("No nativeHome provided. Are you sure the test are executed within :native:native.tests?")
 
+        fun distributionKlibPath(): File = File(nativeHome, "klib")
+        fun hostTarget(): KonanTarget = HostManager.host
+
         fun getRuntimePathsForModule(module: TestModule, testServices: TestServices): List<String> {
             val result = mutableListOf<String>()
 

native/native.tests/tests/org/jetbrains/kotlin/konan/test/ClassicFrontend2NativeIrConverter.kt

@@ -11,7 +11,6 @@ import org.jetbrains.kotlin.backend.common.linkage.issues.checkNoUnboundSymbols
 import org.jetbrains.kotlin.backend.common.serialization.DescriptorByIdSignatureFinderImpl
 import org.jetbrains.kotlin.backend.konan.serialization.KonanManglerDesc
 import org.jetbrains.kotlin.backend.konan.serialization.KonanManglerIr
-import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.config.messageCollector
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
@@ -22,6 +21,7 @@ import org.jetbrains.kotlin.ir.linkage.partial.partialLinkageConfig
 import org.jetbrains.kotlin.ir.symbols.IrSymbol
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.konan.test.blackbox.support.CastCompatibleKotlinNativeClassLoader
+import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration
@@ -130,14 +130,20 @@ class ClassicFrontend2NativeIrConverter(
             diagnosticReporter = configuration.messageCollector
         )
 
+        // N.B. The list of libraries to be written to manifest `depends=` property is not computed here.
+        // The reason for this is that there are no tests which could produce Kotlin/Native KLIBs with the classic frontend compiler.
+        // And there are no plans to add such tests in the future.
+        val usedLibrariesForManifest = emptyList<KotlinLibrary>()
+
         @OptIn(ObsoleteDescriptorBasedAPI::class)
         return IrBackendInput.NativeBackendInput(
             moduleFragment,
             pluginContext,
             diagnosticReporter = DiagnosticReporterFactory.createReporter(configuration.messageCollector),
             descriptorMangler = (pluginContext.symbolTable as SymbolTable).signaturer!!.mangler,
             irMangler = KonanManglerIr,
-            metadataSerializer = null
+            metadataSerializer = null,
+            usedLibrariesForManifest = usedLibrariesForManifest,
         )
     }
 

native/native.tests/tests/org/jetbrains/kotlin/konan/test/Fir2IrNativeResultsConverter.kt

@@ -10,27 +10,41 @@ import org.jetbrains.kotlin.backend.common.actualizer.IrExtraActualDeclarationEx
 import org.jetbrains.kotlin.backend.common.serialization.metadata.DynamicTypeDeserializer
 import org.jetbrains.kotlin.backend.konan.serialization.KonanManglerIr
 import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns
+import org.jetbrains.kotlin.cli.common.messages.getLogger
 import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
+import org.jetbrains.kotlin.descriptors.isEmpty
 import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
+import org.jetbrains.kotlin.fir.backend.DelicateDeclarationStorageApi
 import org.jetbrains.kotlin.fir.backend.Fir2IrComponents
 import org.jetbrains.kotlin.fir.backend.Fir2IrExtensions
 import org.jetbrains.kotlin.fir.backend.Fir2IrVisibilityConverter
 import org.jetbrains.kotlin.fir.pipeline.Fir2IrActualizedResult
 import org.jetbrains.kotlin.fir.pipeline.Fir2KlibMetadataSerializer
 import org.jetbrains.kotlin.ir.IrBuiltIns
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
+import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment
+import org.jetbrains.kotlin.ir.symbols.IrSymbol
+import org.jetbrains.kotlin.ir.symbols.UnsafeDuringIrConstructionAPI
 import org.jetbrains.kotlin.ir.types.IrTypeSystemContext
 import org.jetbrains.kotlin.ir.types.IrTypeSystemContextImpl
 import org.jetbrains.kotlin.ir.util.KotlinMangler
-import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER
+import org.jetbrains.kotlin.ir.util.getPackageFragment
+import org.jetbrains.kotlin.konan.library.KonanLibraryProperResolver
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.metadata.KlibMetadataFactories
+import org.jetbrains.kotlin.library.metadata.kotlinLibrary
+import org.jetbrains.kotlin.library.metadata.resolver.KotlinResolvedLibrary
+import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
+import org.jetbrains.kotlin.library.metadata.resolver.impl.libraryResolver
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.NativeForwardDeclarationKind
 import org.jetbrains.kotlin.test.backend.ir.IrBackendInput
-import org.jetbrains.kotlin.test.frontend.fir.AbstractFir2IrResultsConverter
-import org.jetbrains.kotlin.test.frontend.fir.FirOutputArtifact
-import org.jetbrains.kotlin.test.frontend.fir.getAllNativeDependenciesPaths
-import org.jetbrains.kotlin.test.frontend.fir.resolveLibraries
+import org.jetbrains.kotlin.test.frontend.fir.*
 import org.jetbrains.kotlin.test.model.TestModule
 import org.jetbrains.kotlin.test.services.TestServices
+import org.jetbrains.kotlin.test.services.compilerConfigurationProvider
+import org.jetbrains.kotlin.test.services.configuration.NativeEnvironmentConfigurator
 
 class Fir2IrNativeResultsConverter(testServices: TestServices) : AbstractFir2IrResultsConverter(testServices) {
 
@@ -43,11 +57,7 @@ class Fir2IrNativeResultsConverter(testServices: TestServices) : AbstractFir2IrR
         { emptyList() }
 
     override fun resolveLibraries(module: TestModule, compilerConfiguration: CompilerConfiguration): List<KotlinLibrary> {
-        return resolveLibraries(
-            compilerConfiguration,
-            getAllNativeDependenciesPaths(module, testServices),
-            knownIrProviders = listOf(KLIB_INTEROP_IR_PROVIDER_IDENTIFIER)
-        ).map { it.library }
+        return resolveKotlinLibrariesWithProperDefaults(module, testServices)
     }
 
     override val klibFactories: KlibMetadataFactories = KlibMetadataFactories(::KonanBuiltIns, DynamicTypeDeserializer)
@@ -60,13 +70,91 @@ class Fir2IrNativeResultsConverter(testServices: TestServices) : AbstractFir2IrR
         fir2IrResult: Fir2IrActualizedResult,
         fir2KlibMetadataSerializer: Fir2KlibMetadataSerializer,
     ): IrBackendInput {
+        val usedPackages: List<FqName> = fir2IrResult.components.computeUsedPackages()
+
+        fun ModuleDescriptorImpl.wasReallyUsed(): Boolean {
+            return usedPackages.any { usedPackage -> !packageFragmentProviderForModuleContentWithoutDependencies.isEmpty(usedPackage) }
+        }
+
+        val usedLibrariesForManifest = fir2IrResult.irModuleFragment.descriptor.allDependencyModules.mapNotNull { moduleDescriptor ->
+            if (moduleDescriptor is ModuleDescriptorImpl) {
+                val library = moduleDescriptor.kotlinLibrary
+                if (!library.isDefault || moduleDescriptor.wasReallyUsed())
+                    return@mapNotNull library
+            }
+            null
+        }
+
         return IrBackendInput.NativeBackendInput(
             fir2IrResult.irModuleFragment,
             fir2IrResult.pluginContext,
             diagnosticReporter = diagnosticReporter,
             descriptorMangler = null,
             irMangler = fir2IrResult.components.irMangler,
-            metadataSerializer = fir2KlibMetadataSerializer
+            metadataSerializer = fir2KlibMetadataSerializer,
+            usedLibrariesForManifest = usedLibrariesForManifest,
         )
     }
+
+    companion object {
+        /**
+         * Unlike [org.jetbrains.kotlin.test.frontend.fir.resolveLibraries], which does not distinguish
+         * "default" (i.e., [KotlinLibrary.isDefault]) from "non-default" libraries, this function does that kind of distinction:
+         * - "default" libraries is anything implicitly added by the Kotlin/Native compiler. For example, stdlib & platform libraries.
+         * - "non-default" libraries are the libraries that are explicitly passed to the compiler via compiler CLI arguments.
+         *   In tests, these are the libraries that were explicitly specified by `// MODULE` test directives in test data.
+         *
+         * This distinction is necessary to treat "default" and "non-default" libraries in a different way to compute the
+         * list of dependencies to be written in manifest's `depends=` property:
+         * - Any "default" library is written to manifest only if it was actually used during the compilation.
+         * - Any "non-default" library is written always unconditionally.
+         */
+        private fun resolveKotlinLibrariesWithProperDefaults(module: TestModule, testServices: TestServices): List<KotlinLibrary> {
+            val directDependencies = getTransitivesAndFriendsPaths(module, testServices)
+            val nativeTarget = NativeEnvironmentConfigurator.hostTarget()
+            val nativeDistributionKlibPath = NativeEnvironmentConfigurator.distributionKlibPath().absolutePath
+            val logger = testServices.compilerConfigurationProvider.getCompilerConfiguration(module).getLogger(treatWarningsAsErrors = true)
+
+            val libraryResolver = KonanLibraryProperResolver(
+                directLibs = directDependencies, // Load all direct dependencies as non-default libraries.
+                target = nativeTarget,
+                distributionKlib = nativeDistributionKlibPath,
+                skipCurrentDir = true,
+                logger = logger
+            ).libraryResolver()
+
+            val resolveResult = libraryResolver.resolveWithDependencies(
+                unresolvedLibraries = emptyList(),
+                noStdLib = false, // Load stdlib as a default library.
+                noDefaultLibs = false, // Load platform libraries for the `nativeTarget` as default libraries.
+                noEndorsedLibs = true
+            )
+
+            val topologicallyOrderedLibraries = resolveResult.getFullResolvedList(TopologicalLibraryOrder)
+
+            return topologicallyOrderedLibraries.map(KotlinResolvedLibrary::library)
+        }
+
+        /**
+         * Collects [FqName]s of all used packages from [Fir2IrComponents].
+         *
+         * This information is further used to filter out "default" dependency libraries which were not used
+         * during the compilation, and thus should not be recorded in KLIB manifest's `depends=` property.
+         */
+        @OptIn(DelicateDeclarationStorageApi::class, UnsafeDuringIrConstructionAPI::class)
+        private fun Fir2IrComponents.computeUsedPackages(): List<FqName> = buildSet {
+            fun addExternalPackage(it: IrSymbol) {
+                // FIXME(KT-64742): Fir2IrDeclarationStorage caches may contain unbound IR symbols, so we filter them out.
+                val p = it.takeIf { it.isBound }?.owner as? IrDeclaration ?: return
+                val fragment = (p.getPackageFragment() as? IrExternalPackageFragment) ?: return
+                add(fragment.packageFqName)
+            }
+            declarationStorage.forEachCachedDeclarationSymbol(::addExternalPackage)
+            classifierStorage.forEachCachedDeclarationSymbol(::addExternalPackage)
+
+            // These packages exist in all platform libraries, but can contain only synthetic declarations.
+            // These declarations are not really located in klib, so we don't need to depend on klib to use them.
+            removeAll(NativeForwardDeclarationKind.entries.map { it.packageFqName }.toSet())
+        }.toList()
+    }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/test/NativeKlibSerializerFacade.kt

@@ -19,7 +19,6 @@ import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
-import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER
 import org.jetbrains.kotlin.konan.library.impl.buildLibrary
 import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.library.*
@@ -53,11 +52,7 @@ abstract class AbstractNativeKlibSerializerFacade(
 
         val configuration = testServices.compilerConfigurationProvider.getCompilerConfiguration(module)
 
-        val dependencyPaths = getAllNativeDependenciesPaths(module, testServices)
-        val dependencies = resolveLibraries(configuration, dependencyPaths, knownIrProviders = listOf(KLIB_INTEROP_IR_PROVIDER_IDENTIFIER))
-            .map { it.library }
-
-        val serializerOutput = serialize(configuration, dependencies, module, inputArtifact)
+        val serializerOutput = serialize(configuration, inputArtifact.usedLibrariesForManifest, module, inputArtifact)
 
         val outputArtifact = BinaryArtifacts.KLib(getKlibArtifactFile(testServices, module.name), inputArtifact.diagnosticReporter)
 
@@ -80,21 +75,20 @@ abstract class AbstractNativeKlibSerializerFacade(
             manifestProperties = null,
         )
 
-        updateTestConfiguration(configuration, dependencyPaths, module, outputArtifact)
+        updateTestConfiguration(configuration, module, outputArtifact)
 
         return outputArtifact
     }
 
     protected abstract fun serialize(
         configuration: CompilerConfiguration,
-        dependencies: List<KotlinLibrary>,
+        usedLibrariesForManifest: List<KotlinLibrary>,
         module: TestModule,
         inputArtifact: IrBackendInput.NativeBackendInput,
     ): SerializerOutput<KotlinLibrary>
 
     protected open fun updateTestConfiguration(
         configuration: CompilerConfiguration,
-        dependencyPaths: List<String>,
         module: TestModule,
         outputArtifact: BinaryArtifacts.KLib
     ) = Unit
@@ -109,7 +103,7 @@ class ClassicNativeKlibSerializerFacade(testServices: TestServices) : AbstractNa
 
     override fun serialize(
         configuration: CompilerConfiguration,
-        dependencies: List<KotlinLibrary>,
+        usedLibrariesForManifest: List<KotlinLibrary>,
         module: TestModule,
         inputArtifact: IrBackendInput.NativeBackendInput,
     ): SerializerOutput<KotlinLibrary> {
@@ -140,18 +134,19 @@ class ClassicNativeKlibSerializerFacade(testServices: TestServices) : AbstractNa
         return SerializerOutput(
             serializedMetadata,
             serializerIr,
-            neededLibraries = dependencies,
+            neededLibraries = usedLibrariesForManifest,
         )
     }
 
     override fun updateTestConfiguration(
         configuration: CompilerConfiguration,
-        dependencyPaths: List<String>,
         module: TestModule,
         outputArtifact: BinaryArtifacts.KLib
     ) {
         val nativeFactories = KlibMetadataFactories(::KonanBuiltIns, NullFlexibleTypeDeserializer)
 
+        val dependencyPaths = getAllNativeDependenciesPaths(module, testServices)
+
         val library = resolveLibraries(
             configuration, dependencyPaths + outputArtifact.outputFile.path,
         ).last().library
@@ -177,7 +172,7 @@ class ClassicNativeKlibSerializerFacade(testServices: TestServices) : AbstractNa
 class FirNativeKlibSerializerFacade(testServices: TestServices) : AbstractNativeKlibSerializerFacade(testServices) {
     override fun serialize(
         configuration: CompilerConfiguration,
-        dependencies: List<KotlinLibrary>,
+        usedLibrariesForManifest: List<KotlinLibrary>,
         module: TestModule,
         inputArtifact: IrBackendInput.NativeBackendInput,
     ) = serializeModuleIntoKlib(
@@ -187,7 +182,7 @@ class FirNativeKlibSerializerFacade(testServices: TestServices) : AbstractNative
         inputArtifact.diagnosticReporter,
         CompatibilityMode.CURRENT,
         cleanFiles = emptyList(),
-        dependencies,
+        usedLibrariesForManifest,
         createModuleSerializer = {
                 irDiagnosticReporter,
                 irBuiltIns,

native/utils/src/org/jetbrains/kotlin/konan/library/SearchPathResolver.kt

@@ -26,7 +26,7 @@ fun defaultResolver(
     logger = logger
 )
 
-internal class KonanLibraryProperResolver(
+class KonanLibraryProperResolver(
     directLibs: List<String>,
     override val target: KonanTarget,
     distributionKlib: String?,