[ICTL-908] Test Compiler for Kotlin (#282)

* Parsing LLM Response for Kotlin tests

* last fixes of merge conflicts

* klint

* build problem fix

* refactoring of packages

* refactoring packages

* minimal buggy version of kotlin compilation cycle

* reduced the code

* reduced the code, fixed the representation bug

* solved imports problem

* retirned back the compiler

* version with kotlinc

* working kotlinc

* changes in compiler

* solved the problem with package

* klint

* Update Run IDE for UI Tests.run.xml

* klint

* refactoring

* klint

* merge

* fixes after the review

* deleted companion object

* merge fixes

* factory for parsers

* interface for compilation

* before adjusting the code to the diagram

* new implementation of TestAssembler

* small refactoring

* deleted todo

* fixed problem with the missing package

* fixing the compilation bug

* fixed problem with missing } in the test class

* added more logging

* reduced the duplication in finding package name

* added some documentation and renamed the packageLine and packageString to the packageName

* klint

* fixing the problem with JavaTestCaseDisplayService

* klint

* fixing the problem with package

* fix: rename Language

* fix: import pattern for Kotlin comment

* fix: renaming of methods in TestClassBuilderHelper

* fix: added displayTestCase

* fix: findJavaCompilerInDirectory

* fix: some left fixes

* done with fixes

* deleted unnecessary line

* fixed java

* last renamint

build.gradle.kts

@@ -157,6 +157,7 @@ dependencies {
 
     // https://mvnrepository.com/artifact/org.mockito/mockito-all
     testImplementation("org.mockito:mockito-all:1.10.19")
+    testImplementation("org.mockito.kotlin:mockito-kotlin:5.1.0")
 
     // https://mvnrepository.com/artifact/net.jqwik/jqwik
     testImplementation("net.jqwik:jqwik:1.6.5")

core/src/main/kotlin/org/jetbrains/research/testspark/core/data/TestGenerationData.kt

@@ -16,7 +16,7 @@ data class TestGenerationData(
 
     // Code required of imports and package for generated tests
     var importsCode: MutableSet<String> = mutableSetOf(),
-    var packageLine: String = "",
+    var packageName: String = "",
     var runWith: String = "",
     var otherInfo: String = "",
 
@@ -37,7 +37,7 @@ data class TestGenerationData(
         resultName = ""
         fileUrl = ""
         importsCode = mutableSetOf()
-        packageLine = ""
+        packageName = ""
         runWith = ""
         otherInfo = ""
         polyDepthReducing = 0

core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/LLMWithFeedbackCycle.kt

@@ -10,7 +10,7 @@ import org.jetbrains.research.testspark.core.generation.llm.prompt.PromptSizeRed
 import org.jetbrains.research.testspark.core.monitor.DefaultErrorMonitor
 import org.jetbrains.research.testspark.core.monitor.ErrorMonitor
 import org.jetbrains.research.testspark.core.progress.CustomProgressIndicator
-import org.jetbrains.research.testspark.core.test.Language
+import org.jetbrains.research.testspark.core.test.SupportedLanguage
 import org.jetbrains.research.testspark.core.test.TestCompiler
 import org.jetbrains.research.testspark.core.test.TestsAssembler
 import org.jetbrains.research.testspark.core.test.TestsPersistentStorage
@@ -45,7 +45,7 @@ data class FeedbackResponse(
 
 class LLMWithFeedbackCycle(
     private val report: Report,
-    private val language: Language,
+    private val language: SupportedLanguage,
     private val initialPromptMessage: String,
     private val promptSizeReductionStrategy: PromptSizeReductionStrategy,
     // filename in which the test suite is saved in result path
@@ -167,13 +167,15 @@ class LLMWithFeedbackCycle(
                 generatedTestSuite.updateTestCases(compilableTestCases.toMutableList())
             } else {
                 for (testCaseIndex in generatedTestSuite.testCases.indices) {
-                    val testCaseFilename =
-                        "${getClassWithTestCaseName(generatedTestSuite.testCases[testCaseIndex].name)}.java"
+                    val testCaseFilename = when (language) {
+                        SupportedLanguage.Java -> "${getClassWithTestCaseName(generatedTestSuite.testCases[testCaseIndex].name)}.java"
+                        SupportedLanguage.Kotlin -> "${getClassWithTestCaseName(generatedTestSuite.testCases[testCaseIndex].name)}.kt"
+                    }
 
                     val testCaseRepresentation = testsPresenter.representTestCase(generatedTestSuite, testCaseIndex)
 
                     val saveFilepath = testStorage.saveGeneratedTest(
-                        generatedTestSuite.packageString,
+                        generatedTestSuite.packageName,
                         testCaseRepresentation,
                         resultPath,
                         testCaseFilename,
@@ -184,7 +186,7 @@ class LLMWithFeedbackCycle(
             }
 
             val generatedTestSuitePath: String = testStorage.saveGeneratedTest(
-                generatedTestSuite.packageString,
+                generatedTestSuite.packageName,
                 testsPresenter.representTestSuite(generatedTestSuite),
                 resultPath,
                 testSuiteFilename,

core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/Utils.kt

@@ -4,13 +4,45 @@ import org.jetbrains.research.testspark.core.generation.llm.network.RequestManag
 import org.jetbrains.research.testspark.core.monitor.DefaultErrorMonitor
 import org.jetbrains.research.testspark.core.monitor.ErrorMonitor
 import org.jetbrains.research.testspark.core.progress.CustomProgressIndicator
-import org.jetbrains.research.testspark.core.test.Language
+import org.jetbrains.research.testspark.core.test.SupportedLanguage
 import org.jetbrains.research.testspark.core.test.TestsAssembler
 import org.jetbrains.research.testspark.core.test.data.TestSuiteGeneratedByLLM
+import org.jetbrains.research.testspark.core.utils.javaPackagePattern
+import org.jetbrains.research.testspark.core.utils.kotlinPackagePattern
 import java.util.Locale
 
 // TODO: find a better place for the below functions
 
+/**
+ * Retrieves the package declaration from the given test suite code for any language.
+ *
+ * @param testSuiteCode The generated code of the test suite.
+ * @return The package name extracted from the test suite code, or an empty string if no package declaration was found.
+ */
+fun getPackageFromTestSuiteCode(testSuiteCode: String?, language: SupportedLanguage): String {
+    testSuiteCode ?: return ""
+    return when (language) {
+        SupportedLanguage.Kotlin -> kotlinPackagePattern.find(testSuiteCode)?.groups?.get(1)?.value.orEmpty()
+        SupportedLanguage.Java -> javaPackagePattern.find(testSuiteCode)?.groups?.get(1)?.value.orEmpty()
+    }
+}
+
+/**
+ * Retrieves the imports code from a given test suite code.
+ *
+ * @param testSuiteCode The test suite code from which to extract the imports code. If null, an empty string is returned.
+ * @param classFQN The fully qualified name of the class to be excluded from the imports code. It will not be included in the result.
+ * @return The imports code extracted from the test suite code. If no imports are found or the result is empty after filtering, an empty string is returned.
+ */
+fun getImportsCodeFromTestSuiteCode(testSuiteCode: String?, classFQN: String): MutableSet<String> {
+    testSuiteCode ?: return mutableSetOf()
+    return testSuiteCode.replace("\r\n", "\n").split("\n").asSequence()
+        .filter { it.contains("^import".toRegex()) }
+        .filterNot { it.contains("evosuite".toRegex()) }
+        .filterNot { it.contains("RunWith".toRegex()) }
+        .filterNot { it.contains(classFQN.toRegex()) }.toMutableSet()
+}
+
 /**
  * Returns the generated class name for a given test case.
  *
@@ -39,7 +71,7 @@ fun getClassWithTestCaseName(testCaseName: String): String {
  * @return instance of TestSuiteGeneratedByLLM if the generated test cases are parsable, otherwise null.
  */
 fun executeTestCaseModificationRequest(
-    language: Language,
+    language: SupportedLanguage,
     testCase: String,
     task: String,
     indicator: CustomProgressIndicator,
@@ -50,15 +82,7 @@ fun executeTestCaseModificationRequest(
     // Update Token information
     val prompt = "For this test:\n ```\n $testCase\n ```\nPerform the following task: $task"
 
-    var packageName = ""
-    testCase.split("\n")[0].let {
-        if (it.startsWith("package")) {
-            packageName = it
-                .removePrefix("package ")
-                .removeSuffix(";")
-                .trim()
-        }
-    }
+    val packageName = getPackageFromTestSuiteCode(testCase, language)
 
     val response = requestManager.request(
         language,

core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/network/RequestManager.kt

@@ -7,7 +7,7 @@ import org.jetbrains.research.testspark.core.data.ChatUserMessage
 import org.jetbrains.research.testspark.core.monitor.DefaultErrorMonitor
 import org.jetbrains.research.testspark.core.monitor.ErrorMonitor
 import org.jetbrains.research.testspark.core.progress.CustomProgressIndicator
-import org.jetbrains.research.testspark.core.test.Language
+import org.jetbrains.research.testspark.core.test.SupportedLanguage
 import org.jetbrains.research.testspark.core.test.TestsAssembler
 
 abstract class RequestManager(var token: String) {
@@ -31,7 +31,7 @@ abstract class RequestManager(var token: String) {
      * @return the generated TestSuite, or null and prompt message
      */
     open fun request(
-        language: Language,
+        language: SupportedLanguage,
         prompt: String,
         indicator: CustomProgressIndicator,
         packageName: String,
@@ -65,7 +65,7 @@ abstract class RequestManager(var token: String) {
     open fun processResponse(
         testsAssembler: TestsAssembler,
         packageName: String,
-        language: Language,
+        language: SupportedLanguage,
     ): LLMResponse {
         // save the full response in the chat history
         val response = testsAssembler.getContent()
@@ -78,7 +78,7 @@ abstract class RequestManager(var token: String) {
             return LLMResponse(ResponseErrorCode.EMPTY_LLM_RESPONSE, null)
         }
 
-        val testSuiteGeneratedByLLM = testsAssembler.assembleTestSuite(packageName, language)
+        val testSuiteGeneratedByLLM = testsAssembler.assembleTestSuite()
 
         return if (testSuiteGeneratedByLLM == null) {
             LLMResponse(ResponseErrorCode.TEST_SUITE_PARSING_FAILURE, null)
@@ -97,7 +97,7 @@ abstract class RequestManager(var token: String) {
     open fun processUserFeedbackResponse(
         testsAssembler: TestsAssembler,
         packageName: String,
-        language: Language,
+        language: SupportedLanguage,
     ): LLMResponse {
         val response = testsAssembler.getContent()
 
@@ -108,7 +108,7 @@ abstract class RequestManager(var token: String) {
             return LLMResponse(ResponseErrorCode.EMPTY_LLM_RESPONSE, null)
         }
 
-        val testSuiteGeneratedByLLM = testsAssembler.assembleTestSuite(packageName, language)
+        val testSuiteGeneratedByLLM = testsAssembler.assembleTestSuite()
 
         return if (testSuiteGeneratedByLLM == null) {
             LLMResponse(ResponseErrorCode.TEST_SUITE_PARSING_FAILURE, null)

core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/prompt/PromptBuilder.kt

@@ -78,7 +78,7 @@ internal class PromptBuilder(private var prompt: String) {
                 fullText += "Here are some information about other methods and classes used by the class under test. Only use them for creating objects, not your own ideas.\n"
             }
             for (interestingClass in interestingClasses) {
-                if (interestingClass.qualifiedName.startsWith("java")) {
+                if (interestingClass.qualifiedName.startsWith("java") || interestingClass.qualifiedName.startsWith("kotlin")) {
                     continue
                 }
 
@@ -88,7 +88,9 @@ internal class PromptBuilder(private var prompt: String) {
                     // Skip java methods
                     // TODO: checks for java methods should be done by a caller to make
                     //       this class as abstract and language agnostic as possible.
-                    if (method.containingClassQualifiedName.startsWith("java")) {
+                    if (method.containingClassQualifiedName.startsWith("java") ||
+                        method.containingClassQualifiedName.startsWith("kotlin")
+                    ) {
                         continue
                     }
 

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/Language.kt → core/src/main/kotlin/org/jetbrains/research/testspark/core/test/SupportedLanguage.kt

@@ -3,6 +3,6 @@ package org.jetbrains.research.testspark.core.test
 /**
  * Language ID string should be the same as the language name in com.intellij.lang.Language
  */
-enum class Language(val languageId: String) {
-    Java("JAVA"), Kotlin("Kotlin")
+enum class SupportedLanguage(val languageId: String) {
+    Java("JAVA"), Kotlin("kotlin")
 }

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestBodyPrinter.kt

@@ -0,0 +1,21 @@
+package org.jetbrains.research.testspark.core.test
+
+import org.jetbrains.research.testspark.core.test.data.TestLine
+
+interface TestBodyPrinter {
+    /**
+     * Generates a test body as a string based on the provided parameters.
+     *
+     * @param testInitiatedText A string containing the upper part of the test case.
+     * @param lines A mutable list of `TestLine` objects representing the lines of the test body.
+     * @param throwsException The exception type that the test function throws, if any.
+     * @param name The name of the test function.
+     * @return A string representing the complete test body.
+     */
+    fun printTestBody(
+        testInitiatedText: String,
+        lines: MutableList<TestLine>,
+        throwsException: String,
+        name: String,
+    ): String
+}

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestCompiler.kt

@@ -1,32 +1,24 @@
 package org.jetbrains.research.testspark.core.test
 
-import io.github.oshai.kotlinlogging.KotlinLogging
 import org.jetbrains.research.testspark.core.test.data.TestCaseGeneratedByLLM
-import org.jetbrains.research.testspark.core.utils.CommandLineRunner
 import org.jetbrains.research.testspark.core.utils.DataFilesUtil
-import java.io.File
 
 data class TestCasesCompilationResult(
     val allTestCasesCompilable: Boolean,
     val compilableTestCases: MutableSet<TestCaseGeneratedByLLM>,
 )
 
-/**
- * TestCompiler is a class that is responsible for compiling generated test cases using the proper javac.
- * It provides methods for compiling test cases and code files.
- */
-open class TestCompiler(
-    private val javaHomeDirectoryPath: String,
+abstract class TestCompiler(
     private val libPaths: List<String>,
     private val junitLibPaths: List<String>,
 ) {
-    private val log = KotlinLogging.logger { this::class.java }
-
     /**
-     * Compiles the generated files with test cases using the proper javac.
+     * Compiles a list of test cases and returns the compilation result.
      *
-     * @return true if all the provided test cases are successfully compiled,
-     *         otherwise returns false.
+     * @param generatedTestCasesPaths A list of file paths where the generated test cases are located.
+     * @param buildPath All the directories where the compiled code of the project under test is saved. This path is used as a classpath to run each test case.
+     * @param testCases A mutable list of `TestCaseGeneratedByLLM` objects representing the test cases to be compiled.
+     * @return A `TestCasesCompilationResult` object containing the overall compilation success status and a set of compilable test cases.
      */
     fun compileTestCases(
         generatedTestCasesPaths: List<String>,
@@ -51,53 +43,19 @@ open class TestCompiler(
      * Compiles the code at the specified path using the provided project build path.
      *
      * @param path The path of the code file to compile.
-     * @param projectBuildPath The project build path to use during compilation.
+     * @param projectBuildPath All the directories where the compiled code of the project under test is saved. This path is used as a classpath to run each test case.
      * @return A pair containing a boolean value indicating whether the compilation was successful (true) or not (false),
      *         and a string message describing any error encountered during compilation.
      */
-    fun compileCode(path: String, projectBuildPath: String): Pair<Boolean, String> {
-        // find the proper javac
-        val javaCompile = File(javaHomeDirectoryPath).walk()
-            .filter {
-                val isCompilerName = if (DataFilesUtil.isWindows()) it.name.equals("javac.exe") else it.name.equals("javac")
-                isCompilerName && it.isFile
-            }
-            .firstOrNull()
-
-        if (javaCompile == null) {
-            val msg = "Cannot find java compiler 'javac' at '$javaHomeDirectoryPath'"
-            log.error { msg }
-            throw RuntimeException(msg)
-        }
-
-        println("javac found at '${javaCompile.absolutePath}'")
-
-        // compile file
-        val errorMsg = CommandLineRunner.run(
-            arrayListOf(
-                javaCompile.absolutePath,
-                "-cp",
-                "\"${getPath(projectBuildPath)}\"",
-                path,
-            ),
-        )
-
-        log.info { "Error message: '$errorMsg'" }
-
-        // create .class file path
-        val classFilePath = path.replace(".java", ".class")
-
-        // check is .class file exists
-        return Pair(File(classFilePath).exists(), errorMsg)
-    }
+    abstract fun compileCode(path: String, projectBuildPath: String): Pair<Boolean, String>
 
     /**
      * Generates the path for the command by concatenating the necessary paths.
      *
      * @param buildPath The path of the build file.
      * @return The generated path as a string.
      */
-    fun getPath(buildPath: String): String {
+    fun getClassPaths(buildPath: String): String {
         // create the path for the command
         val separator = DataFilesUtil.classpathSeparator
         val dependencyLibPath = libPaths.joinToString(separator.toString())

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestSuiteParser.kt

@@ -11,7 +11,7 @@ data class TestCaseParseResult(
 
 interface TestSuiteParser {
     /**
-     * Extracts test cases from raw text and generates a test suite using the given package name.
+     * Extracts test cases from raw text and generates a test suite.
      *
      * @param rawText The raw text provided by the LLM that contains the generated test cases.
      * @return A GeneratedTestSuite instance containing the extracted test cases.

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestsAssembler.kt

@@ -32,10 +32,9 @@ abstract class TestsAssembler {
     }
 
     /**
-     * Extracts test cases from raw text and generates a TestSuite using the given package name.
+     * Extracts test cases from raw text and generates a TestSuite.
      *
-     * @param packageName The package name to be set in the generated TestSuite.
-     * @return A TestSuiteGeneratedByLLM object containing the extracted test cases and package name.
+     * @return A TestSuiteGeneratedByLLM object containing information about the extracted test cases.
      */
-    abstract fun assembleTestSuite(packageName: String, language: Language): TestSuiteGeneratedByLLM?
+    abstract fun assembleTestSuite(): TestSuiteGeneratedByLLM?
 }