diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/exception/Exceptions.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/exception/Exceptions.kt index 3d2bccd6a..c1b1f337f 100644 --- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/exception/Exceptions.kt +++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/exception/Exceptions.kt @@ -29,3 +29,10 @@ class JavaCompilerNotFoundException(message: String) : TestSparkException(messag * @param message A descriptive message explaining the specific error that led to this exception. */ class JavaSDKMissingException(message: String) : TestSparkException(message) + +/** + * Represents an exception thrown when a class file could not be found in the same path after the code compilation. + * + * @param message A descriptive message explaining the error + */ +class ClassFileNotFoundException(message: String) : TestSparkException(message) diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/LLMWithFeedbackCycle.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/LLMWithFeedbackCycle.kt index 7daaf8ff9..31fcde547 100644 --- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/LLMWithFeedbackCycle.kt +++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/LLMWithFeedbackCycle.kt @@ -281,8 +281,13 @@ class LLMWithFeedbackCycle( onWarningCallback?.invoke(WarningType.COMPILATION_ERROR_OCCURRED) - nextPromptMessage = - "I cannot compile the tests that you provided. The error is:\n${testSuiteCompilationResult.second}\n Fix this issue in the provided tests.\nGenerate public classes and public methods. Response only a code with tests between ```, do not provide any other text." + nextPromptMessage = """ + I cannot compile the tests that you provided. The error is: + ``` + ${testSuiteCompilationResult.executionMessage} + ``` + Fix this issue in the provided tests.\nGenerate public classes and public methods. Response only a code with tests between ```, do not provide any other text. + """.trimIndent() log.info { nextPromptMessage } continue } diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestCompiler.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestCompiler.kt index 9e78092f4..3d85f15c1 100644 --- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestCompiler.kt +++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/TestCompiler.kt @@ -8,6 +8,13 @@ data class TestCasesCompilationResult( val compilableTestCases: MutableSet, ) +data class ExecutionResult( + val exitCode: Int, + val executionMessage: String, +) { + fun isSuccessful(): Boolean = exitCode == 0 +} + abstract class TestCompiler(libPaths: List, junitLibPaths: List) { val separator = DataFilesUtil.classpathSeparator val dependencyLibPath = libPaths.joinToString(separator.toString()) @@ -27,13 +34,13 @@ abstract class TestCompiler(libPaths: List, junitLibPaths: List) generatedTestCasesPaths: List, buildPath: String, testCases: MutableList, - workingDir: String + workingDir: String, ): TestCasesCompilationResult { var allTestCasesCompilable = true val compilableTestCases: MutableSet = mutableSetOf() for (index in generatedTestCasesPaths.indices) { - val compilable = compileCode(generatedTestCasesPaths[index], buildPath, workingDir).first + val compilable = compileCode(generatedTestCasesPaths[index], buildPath, workingDir).isSuccessful() allTestCasesCompilable = allTestCasesCompilable && compilable if (compilable) { compilableTestCases.add(testCases[index]) @@ -52,7 +59,7 @@ abstract class TestCompiler(libPaths: List, junitLibPaths: List) * @return A pair containing a boolean value indicating whether the compilation was successful (true) or not (false), * and a string message describing any error encountered during compilation. */ - abstract fun compileCode(path: String, projectBuildPath: String, workingDir: String): Pair + abstract fun compileCode(path: String, projectBuildPath: String, workingDir: String): ExecutionResult /** * Generates the path for the command by concatenating the necessary paths. diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt index a13e844d9..4486eac52 100644 --- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt +++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt @@ -1,7 +1,9 @@ package org.jetbrains.research.testspark.core.test.java import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.research.testspark.core.exception.ClassFileNotFoundException import org.jetbrains.research.testspark.core.exception.JavaCompilerNotFoundException +import org.jetbrains.research.testspark.core.test.ExecutionResult import org.jetbrains.research.testspark.core.test.TestCompiler import org.jetbrains.research.testspark.core.utils.CommandLineRunner import org.jetbrains.research.testspark.core.utils.DataFilesUtil @@ -37,10 +39,10 @@ class JavaTestCompiler( javac = javaCompiler.absolutePath } - override fun compileCode(path: String, projectBuildPath: String, workingDir: String): Pair { + override fun compileCode(path: String, projectBuildPath: String, workingDir: String): ExecutionResult { val classPaths = "\"${getClassPaths(projectBuildPath)}\"" // compile file - val errorMsg = CommandLineRunner.run( + val executionResult = CommandLineRunner.run( arrayListOf( /** * Filepath may contain spaces, so we need to wrap it in quotes. @@ -54,13 +56,13 @@ class JavaTestCompiler( */ ), ) + logger.info { "Exit code: '${executionResult.exitCode}'; Execution message: '${executionResult.executionMessage}'" } - logger.info { "Error message: '$errorMsg'" } - // create .class file path - val classFilePath = path.removeSuffix(".java") + ".class" - - // check if .class file exists - return Pair(File(classFilePath).exists() && errorMsg.isBlank(), errorMsg) + val classFilePath = path.replace(".java", ".class") + if (!File(classFilePath).exists()) { + throw ClassFileNotFoundException("Expected class file at $classFilePath after the compilation of file $path, but it does not exist.") + } + return executionResult } override fun getClassPaths(buildPath: String): String { diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt index da10a5c28..e1487ebba 100644 --- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt +++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt @@ -1,7 +1,9 @@ package org.jetbrains.research.testspark.core.test.kotlin import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.research.testspark.core.exception.ClassFileNotFoundException import org.jetbrains.research.testspark.core.exception.KotlinCompilerNotFoundException +import org.jetbrains.research.testspark.core.test.ExecutionResult import org.jetbrains.research.testspark.core.test.TestCompiler import org.jetbrains.research.testspark.core.utils.CommandLineRunner import org.jetbrains.research.testspark.core.utils.DataFilesUtil @@ -10,7 +12,7 @@ import java.io.File class KotlinTestCompiler( libPaths: List, junitLibPaths: List, - kotlinSDKHomeDirectory: String + kotlinSDKHomeDirectory: String, ) : TestCompiler(libPaths, junitLibPaths) { private val logger = KotlinLogging.logger { this::class.java } private val kotlinc: String @@ -49,13 +51,12 @@ class KotlinTestCompiler( kotlinc = kotlinCompiler.absolutePath } - override fun compileCode(path: String, projectBuildPath: String, workingDir: String): Pair { + override fun compileCode(path: String, projectBuildPath: String, workingDir: String): ExecutionResult { logger.info { "[KotlinTestCompiler] Compiling ${path.substringAfterLast('/')}" } val classPaths = "\"${getClassPaths(projectBuildPath)}\"" // Compile file - // TODO: we treat warnings as errors for now - val errorMsg = CommandLineRunner.run( + val executionResult = CommandLineRunner.run( arrayListOf( /** * Filepath may contain spaces, so we need to wrap it in quotes. @@ -68,16 +69,16 @@ class KotlinTestCompiler( * Forcing kotlinc to save a classfile in the same place, as '.kt' file */ "-d", - workingDir + workingDir, ), ) - - logger.info { "Error message: '$errorMsg'" } + logger.info { "Exit code: '${executionResult.exitCode}'; Execution message: '${executionResult.executionMessage}'" } val classFilePath = path.removeSuffix(".kt") + ".class" - - // check if .class file exists - return Pair(File(classFilePath).exists() && errorMsg.isBlank(), errorMsg) + if (!File(classFilePath).exists()) { + throw ClassFileNotFoundException("Expected class file at $classFilePath after the compilation of file $path, but it does not exist.") + } + return executionResult } override fun getClassPaths(buildPath: String): String = commonPath.plus(buildPath) diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt index 97e870bae..a9d0343c9 100644 --- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt +++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt @@ -1,6 +1,7 @@ package org.jetbrains.research.testspark.core.utils import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.research.testspark.core.test.ExecutionResult import java.io.BufferedReader import java.io.InputStreamReader @@ -9,16 +10,16 @@ class CommandLineRunner { protected val log = KotlinLogging.logger {} /** - * Executes a command line process and returns the output as a string. + * Executes a command line process * * @param cmd The command line arguments as an ArrayList of strings. - * @return The output of the command line process as a string. + * @return A pair containing exit code and a string message containing stdout and stderr of the executed process. */ - fun run(cmd: ArrayList): String { - var errorMessage = "" + fun run(cmd: ArrayList): ExecutionResult { + var executionMsg = "" /** - * Since Windows does not provide bash, use cmd or similar default command line interpreter + * Since Windows does not provide bash, use cmd or simila r default command line interpreter */ val process = if (DataFilesUtil.isWindows()) { ProcessBuilder() @@ -32,17 +33,16 @@ class CommandLineRunner { .redirectErrorStream(true) .start() } - val reader = BufferedReader(InputStreamReader(process.inputStream)) + val separator = System.lineSeparator() var line: String? while (reader.readLine().also { line = it } != null) { - errorMessage += line + executionMsg += "$line$separator" } process.waitFor() - - return errorMessage + return ExecutionResult(process.exitValue(), executionMsg) } } } diff --git a/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt b/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt index f304c965e..8b5de49db 100644 --- a/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt +++ b/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt @@ -64,6 +64,7 @@ class TestProcessor( * @param projectBuildPath The build path of the project. * @param generatedTestPackage The package where the generated test class is located. * @return An empty string if the test execution is successful, otherwise an error message. + * TODO refactor method signature to return exit code as well, see `TestCompiler` for example */ fun createXmlFromJacoco( className: String, @@ -99,7 +100,7 @@ class TestProcessor( } else { "-javaagent:$jacocoAgentLibraryPath=destfile=$dataFileName.exec,append=false" } - val testExecutionError = CommandLineRunner.run( + val testExecutionResult = CommandLineRunner.run( arrayListOf( javaRunner.absolutePath, javaAgentFlag, @@ -110,7 +111,7 @@ class TestProcessor( ), ) - log.info("Test execution error message: $testExecutionError") + log.info("Exit code: '${testExecutionResult.exitCode}'; Execution message: '${testExecutionResult.executionMessage}'") // Prepare the command for generating the Jacoco report val command = mutableListOf( @@ -140,7 +141,7 @@ class TestProcessor( CommandLineRunner.run(command as ArrayList) - return testExecutionError + return if (testExecutionResult.isSuccessful()) "" else testExecutionResult.executionMessage } /** @@ -179,8 +180,8 @@ class TestProcessor( // compilation checking val compilationResult = testCompiler.compileCode(generatedTestPath, buildPath, resultPath) - if (!compilationResult.first) { - testsExecutionResultManager.addFailedTest(testId, testCode, compilationResult.second) + if (!compilationResult.isSuccessful()) { + testsExecutionResultManager.addFailedTest(testId, testCode, compilationResult.executionMessage) } else { val dataFileName = "$resultPath/jacoco-${fileName.split(".")[0]}" diff --git a/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt b/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt index 3ae554a8e..d5a26810f 100644 --- a/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt +++ b/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt @@ -84,7 +84,9 @@ class EvoSuiteProcessManager( if (ToolUtils.isProcessStopped(errorMonitor, indicator)) return null val regex = Regex("version \"(.*?)\"") - val version = regex.find(CommandLineRunner.run(arrayListOf(evoSuiteSettingsState.javaPath, "-version"))) + val versionCommandResult = CommandLineRunner.run(arrayListOf(evoSuiteSettingsState.javaPath, "-version")) + log.info("Version command result: exit code '${versionCommandResult.exitCode}', message '${versionCommandResult.executionMessage}'") + val version = regex.find(versionCommandResult.executionMessage) ?.groupValues ?.get(1) ?.split(".")