diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt
index 4553f5702..c0e69fff0 100644
--- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt
+++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/java/JavaTestCompiler.kt
@@ -34,7 +34,7 @@ class JavaTestCompiler(
         println("javac found at '${javaCompile.absolutePath}'")
 
         // compile file
-        val errorMsg = CommandLineRunner.run(
+        val executionResult = CommandLineRunner.run(
             arrayListOf(
                 javaCompile.absolutePath,
                 "-cp",
@@ -42,13 +42,14 @@ class JavaTestCompiler(
                 path,
             ),
         )
+        val executionMsg = executionResult.second
 
-        log.info { "Error message: '$errorMsg'" }
+        log.info { "Execution result: '${executionMsg}'" }
         // create .class file path
         val classFilePath = path.replace(".java", ".class")
 
         // check is .class file exists
-        return Pair(File(classFilePath).exists(), errorMsg)
+        return Pair(File(classFilePath).exists() && (executionResult.first == 0), executionMsg)
     }
 
     override fun getClassPaths(buildPath: String): String {
diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt
index fe74d036a..14fac0d86 100644
--- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt
+++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/test/kotlin/KotlinTestCompiler.kt
@@ -15,7 +15,7 @@ class KotlinTestCompiler(libPaths: List<String>, junitLibPaths: List<String>) :
         // TODO find the kotlinc if it is not in PATH
         val classPaths = "\"${getClassPaths(projectBuildPath)}\""
         // Compile file
-        val errorMsg = CommandLineRunner.run(
+        val executionResult = CommandLineRunner.run(
             arrayListOf(
                 "kotlinc",
                 "-cp",
@@ -23,16 +23,18 @@ class KotlinTestCompiler(libPaths: List<String>, junitLibPaths: List<String>) :
                 path,
             ),
         )
+        val executionMsg = executionResult.second
+        val execSuccessful = executionResult.first == 0
 
-        if (errorMsg.isNotEmpty()) {
-            log.info { "Error message: '$errorMsg'" }
-            if (errorMsg.contains("kotlinc: command not found'")) {
-                throw RuntimeException(errorMsg)
+        if (!execSuccessful) {
+            log.info { "Error message: '$executionMsg'" }
+            if (executionMsg.contains("kotlinc: command not found'")) {
+                throw RuntimeException(executionMsg)
             }
         }
 
-        // No need to save the .class file for kotlin, so checking the error message is enough
-        return Pair(errorMsg.isBlank(), errorMsg)
+        // TODO `.class` files are not saving for Kotlin
+        return Pair(execSuccessful, executionMsg)
     }
 
     override fun getClassPaths(buildPath: String): String = commonPath.plus(buildPath)
diff --git a/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt b/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt
index 97e870bae..5d2d79eb0 100644
--- a/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt
+++ b/core/src/main/kotlin/org/jetbrains/research/testspark/core/utils/CommandLineRunner.kt
@@ -9,13 +9,13 @@ class CommandLineRunner {
         protected val log = KotlinLogging.logger {}
 
         /**
-         * Executes a command line process and returns the output as a string.
+         * Executes a command line process
          *
          * @param cmd The command line arguments as an ArrayList of strings.
-         * @return The output of the command line process as a string.
+         * @return A pair containing exit value and a string message containing execution results
          */
-        fun run(cmd: ArrayList<String>): String {
-            var errorMessage = ""
+        fun run(cmd: ArrayList<String>): Pair<Int, String> {
+            var executionMsg = ""
 
             /**
              * Since Windows does not provide bash, use cmd or similar default command line interpreter
@@ -37,12 +37,11 @@ class CommandLineRunner {
             var line: String?
 
             while (reader.readLine().also { line = it } != null) {
-                errorMessage += line
+                executionMsg += line
             }
 
             process.waitFor()
-
-            return errorMessage
+            return Pair(process.exitValue(), executionMsg)
         }
     }
 }
diff --git a/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt b/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt
index 1bf6acb2a..ae3a63ce5 100644
--- a/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt
+++ b/src/main/kotlin/org/jetbrains/research/testspark/tools/TestProcessor.kt
@@ -99,7 +99,7 @@ class TestProcessor(
             } else {
                 "-javaagent:$jacocoAgentLibraryPath=destfile=$dataFileName.exec,append=false"
             }
-        val testExecutionError = CommandLineRunner.run(
+        val testExecutionResult = CommandLineRunner.run(
             arrayListOf(
                 javaRunner.absolutePath,
                 javaAgentFlag,
@@ -110,7 +110,7 @@ class TestProcessor(
             ),
         )
 
-        log.info("Test execution error message: $testExecutionError")
+        log.info("Test execution message: ${testExecutionResult.second}")
 
         // Prepare the command for generating the Jacoco report
         val command = mutableListOf(
@@ -140,7 +140,7 @@ class TestProcessor(
 
         CommandLineRunner.run(command as ArrayList<String>)
 
-        return testExecutionError
+        return if (testExecutionResult.first == 0) "" else testExecutionResult.second
     }
 
     /**
diff --git a/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt b/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt
index 3ae554a8e..04071cf6c 100644
--- a/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt
+++ b/src/main/kotlin/org/jetbrains/research/testspark/tools/evosuite/generation/EvoSuiteProcessManager.kt
@@ -84,7 +84,7 @@ class EvoSuiteProcessManager(
             if (ToolUtils.isProcessStopped(errorMonitor, indicator)) return null
 
             val regex = Regex("version \"(.*?)\"")
-            val version = regex.find(CommandLineRunner.run(arrayListOf(evoSuiteSettingsState.javaPath, "-version")))
+            val version = regex.find(CommandLineRunner.run(arrayListOf(evoSuiteSettingsState.javaPath, "-version")).second)
                 ?.groupValues
                 ?.get(1)
                 ?.split(".")