Command Line Runner API changes

core/src/main/kotlin/org/jetbrains/research/testspark/core/exception/Exceptions.kt

@@ -29,3 +29,10 @@ class JavaCompilerNotFoundException(message: String) : TestSparkException(messag
  * @param message A descriptive message explaining the specific error that led to this exception.
  */
 class JavaSDKMissingException(message: String) : TestSparkException(message)
+
+/**
+ * Represents an exception thrown when a class file could not be found in the same path after the code compilation.
+ *
+ * @param message A descriptive message explaining the error
+ */
+class ClassFileNotFoundException(message: String) : TestSparkException(message)

core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/LLMWithFeedbackCycle.kt

@@ -281,8 +281,13 @@ class LLMWithFeedbackCycle(
 
                 onWarningCallback?.invoke(WarningType.COMPILATION_ERROR_OCCURRED)
 
-                nextPromptMessage =
-                    "I cannot compile the tests that you provided. The error is:\n${testSuiteCompilationResult.second}\n Fix this issue in the provided tests.\nGenerate public classes and public methods. Response only a code with tests between ```, do not provide any other text."
+                nextPromptMessage = """
+                   I cannot compile the tests that you provided. The error is:
+                   ```
+                   ${testSuiteCompilationResult.executionMessage}
+                   ```
+                   Fix this issue in the provided tests.\nGenerate public classes and public methods. Response only a code with tests between ```, do not provide any other text.
+                """.trimIndent()
                 log.info { nextPromptMessage }
                 continue
             }

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestCompiler.kt

@@ -8,6 +8,13 @@ data class TestCasesCompilationResult(
     val compilableTestCases: MutableSet<TestCaseGeneratedByLLM>,
 )
 
+data class ExecutionResult(
+    val exitCode: Int,
+    val executionMessage: String,
+) {
+    fun isSuccessful(): Boolean = exitCode == 0
+}
+
 abstract class TestCompiler(libPaths: List<String>, junitLibPaths: List<String>) {
     val separator = DataFilesUtil.classpathSeparator
     val dependencyLibPath = libPaths.joinToString(separator.toString())
@@ -27,13 +34,13 @@ abstract class TestCompiler(libPaths: List<String>, junitLibPaths: List<String>)
         generatedTestCasesPaths: List<String>,
         buildPath: String,
         testCases: MutableList<TestCaseGeneratedByLLM>,
-        workingDir: String
+        workingDir: String,
     ): TestCasesCompilationResult {
         var allTestCasesCompilable = true
         val compilableTestCases: MutableSet<TestCaseGeneratedByLLM> = mutableSetOf()
 
         for (index in generatedTestCasesPaths.indices) {
-            val compilable = compileCode(generatedTestCasesPaths[index], buildPath, workingDir).first
+            val compilable = compileCode(generatedTestCasesPaths[index], buildPath, workingDir).isSuccessful()
             allTestCasesCompilable = allTestCasesCompilable && compilable
             if (compilable) {
                 compilableTestCases.add(testCases[index])
@@ -52,7 +59,7 @@ abstract class TestCompiler(libPaths: List<String>, junitLibPaths: List<String>)
      * @return A pair containing a boolean value indicating whether the compilation was successful (true) or not (false),
      *         and a string message describing any error encountered during compilation.
      */
-    abstract fun compileCode(path: String, projectBuildPath: String, workingDir: String): Pair<Boolean, String>
+    abstract fun compileCode(path: String, projectBuildPath: String, workingDir: String): ExecutionResult
 
     /**
      * Generates the path for the command by concatenating the necessary paths.

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt

@@ -1,7 +1,9 @@
 package org.jetbrains.research.testspark.core.test.java
 
 import io.github.oshai.kotlinlogging.KotlinLogging
+import org.jetbrains.research.testspark.core.exception.ClassFileNotFoundException
 import org.jetbrains.research.testspark.core.exception.JavaCompilerNotFoundException
+import org.jetbrains.research.testspark.core.test.ExecutionResult
 import org.jetbrains.research.testspark.core.test.TestCompiler
 import org.jetbrains.research.testspark.core.utils.CommandLineRunner
 import org.jetbrains.research.testspark.core.utils.DataFilesUtil
@@ -37,10 +39,10 @@ class JavaTestCompiler(
         javac = javaCompiler.absolutePath
     }
 
-    override fun compileCode(path: String, projectBuildPath: String, workingDir: String): Pair<Boolean, String> {
+    override fun compileCode(path: String, projectBuildPath: String, workingDir: String): ExecutionResult {
         val classPaths = "\"${getClassPaths(projectBuildPath)}\""
         // compile file
-        val errorMsg = CommandLineRunner.run(
+        val executionResult = CommandLineRunner.run(
             arrayListOf(
                 /**
                  * Filepath may contain spaces, so we need to wrap it in quotes.
@@ -54,13 +56,13 @@ class JavaTestCompiler(
                  */
             ),
         )
+        logger.info { "Exit code: '${executionResult.exitCode}'; Execution message: '${executionResult.executionMessage}'" }
 
-        logger.info { "Error message: '$errorMsg'" }
-        // create .class file path
-        val classFilePath = path.removeSuffix(".java") + ".class"
-
-        // check if .class file exists
-        return Pair(File(classFilePath).exists() && errorMsg.isBlank(), errorMsg)
+        val classFilePath = path.replace(".java", ".class")
+        if (!File(classFilePath).exists()) {
+            throw ClassFileNotFoundException("Expected class file at $classFilePath after the compilation of file $path, but it does not exist.")
+        }
+        return executionResult
     }
 
     override fun getClassPaths(buildPath: String): String {

core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt

@@ -1,7 +1,9 @@
 package org.jetbrains.research.testspark.core.test.kotlin
 
 import io.github.oshai.kotlinlogging.KotlinLogging
+import org.jetbrains.research.testspark.core.exception.ClassFileNotFoundException
 import org.jetbrains.research.testspark.core.exception.KotlinCompilerNotFoundException
+import org.jetbrains.research.testspark.core.test.ExecutionResult
 import org.jetbrains.research.testspark.core.test.TestCompiler
 import org.jetbrains.research.testspark.core.utils.CommandLineRunner
 import org.jetbrains.research.testspark.core.utils.DataFilesUtil
@@ -10,7 +12,7 @@ import java.io.File
 class KotlinTestCompiler(
     libPaths: List<String>,
     junitLibPaths: List<String>,
-    kotlinSDKHomeDirectory: String
+    kotlinSDKHomeDirectory: String,
 ) : TestCompiler(libPaths, junitLibPaths) {
     private val logger = KotlinLogging.logger { this::class.java }
     private val kotlinc: String
@@ -49,13 +51,12 @@ class KotlinTestCompiler(
         kotlinc = kotlinCompiler.absolutePath
     }
 
-    override fun compileCode(path: String, projectBuildPath: String, workingDir: String): Pair<Boolean, String> {
+    override fun compileCode(path: String, projectBuildPath: String, workingDir: String): ExecutionResult {
         logger.info { "[KotlinTestCompiler] Compiling ${path.substringAfterLast('/')}" }
 
         val classPaths = "\"${getClassPaths(projectBuildPath)}\""
         // Compile file
-        // TODO: we treat warnings as errors for now
-        val errorMsg = CommandLineRunner.run(
+        val executionResult = CommandLineRunner.run(
             arrayListOf(
                 /**
                  * Filepath may contain spaces, so we need to wrap it in quotes.
@@ -68,16 +69,16 @@ class KotlinTestCompiler(
                  * Forcing kotlinc to save a classfile in the same place, as '.kt' file
                  */
                 "-d",
-                workingDir
+                workingDir,
             ),
         )
-
-        logger.info { "Error message: '$errorMsg'" }
+        logger.info { "Exit code: '${executionResult.exitCode}'; Execution message: '${executionResult.executionMessage}'" }
 
         val classFilePath = path.removeSuffix(".kt") + ".class"
-
-        // check if .class file exists
-        return Pair(File(classFilePath).exists() && errorMsg.isBlank(), errorMsg)
+        if (!File(classFilePath).exists()) {
+            throw ClassFileNotFoundException("Expected class file at $classFilePath after the compilation of file $path, but it does not exist.")
+        }
+        return executionResult
     }
 
     override fun getClassPaths(buildPath: String): String = commonPath.plus(buildPath)

core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.research.testspark.core.utils
 
 import io.github.oshai.kotlinlogging.KotlinLogging
+import org.jetbrains.research.testspark.core.test.ExecutionResult
 import java.io.BufferedReader
 import java.io.InputStreamReader
 
@@ -9,16 +10,16 @@ class CommandLineRunner {
         protected val log = KotlinLogging.logger {}
 
         /**
-         * Executes a command line process and returns the output as a string.
+         * Executes a command line process
          *
          * @param cmd The command line arguments as an ArrayList of strings.
-         * @return The output of the command line process as a string.
+         * @return A pair containing exit code and a string message containing stdout and stderr of the executed process.
          */
-        fun run(cmd: ArrayList<String>): String {
-            var errorMessage = ""
+        fun run(cmd: ArrayList<String>): ExecutionResult {
+            var executionMsg = ""
 
             /**
-             * Since Windows does not provide bash, use cmd or similar default command line interpreter
+             * Since Windows does not provide bash, use cmd or simila       r default command line interpreter
              */
             val process = if (DataFilesUtil.isWindows()) {
                 ProcessBuilder()
@@ -32,17 +33,16 @@ class CommandLineRunner {
                     .redirectErrorStream(true)
                     .start()
             }
-
             val reader = BufferedReader(InputStreamReader(process.inputStream))
+            val separator = System.lineSeparator()
             var line: String?
 
             while (reader.readLine().also { line = it } != null) {
-                errorMessage += line
+                executionMsg += "$line$separator"
             }
 
             process.waitFor()
-
-            return errorMessage
+            return ExecutionResult(process.exitValue(), executionMsg)
         }
     }
 }

src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt

@@ -64,6 +64,7 @@ class TestProcessor(
      * @param projectBuildPath The build path of the project.
      * @param generatedTestPackage The package where the generated test class is located.
      * @return An empty string if the test execution is successful, otherwise an error message.
+     * TODO refactor method signature to return exit code as well, see `TestCompiler` for example
      */
     fun createXmlFromJacoco(
         className: String,
@@ -99,7 +100,7 @@ class TestProcessor(
             } else {
                 "-javaagent:$jacocoAgentLibraryPath=destfile=$dataFileName.exec,append=false"
             }
-        val testExecutionError = CommandLineRunner.run(
+        val testExecutionResult = CommandLineRunner.run(
             arrayListOf(
                 javaRunner.absolutePath,
                 javaAgentFlag,
@@ -110,7 +111,7 @@ class TestProcessor(
             ),
         )
 
-        log.info("Test execution error message: $testExecutionError")
+        log.info("Exit code: '${testExecutionResult.exitCode}'; Execution message: '${testExecutionResult.executionMessage}'")
 
         // Prepare the command for generating the Jacoco report
         val command = mutableListOf(
@@ -140,7 +141,7 @@ class TestProcessor(
 
         CommandLineRunner.run(command as ArrayList<String>)
 
-        return testExecutionError
+        return if (testExecutionResult.isSuccessful()) "" else testExecutionResult.executionMessage
     }
 
     /**
@@ -179,8 +180,8 @@ class TestProcessor(
 
         // compilation checking
         val compilationResult = testCompiler.compileCode(generatedTestPath, buildPath, resultPath)
-        if (!compilationResult.first) {
-            testsExecutionResultManager.addFailedTest(testId, testCode, compilationResult.second)
+        if (!compilationResult.isSuccessful()) {
+            testsExecutionResultManager.addFailedTest(testId, testCode, compilationResult.executionMessage)
         } else {
             val dataFileName = "$resultPath/jacoco-${fileName.split(".")[0]}"
 

src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt

@@ -84,7 +84,9 @@ class EvoSuiteProcessManager(
             if (ToolUtils.isProcessStopped(errorMonitor, indicator)) return null
 
             val regex = Regex("version \"(.*?)\"")
-            val version = regex.find(CommandLineRunner.run(arrayListOf(evoSuiteSettingsState.javaPath, "-version")))
+            val versionCommandResult = CommandLineRunner.run(arrayListOf(evoSuiteSettingsState.javaPath, "-version"))
+            log.info("Version command result: exit code '${versionCommandResult.exitCode}', message '${versionCommandResult.executionMessage}'")
+            val version = regex.find(versionCommandResult.executionMessage)
                 ?.groupValues
                 ?.get(1)
                 ?.split(".")