From 2d9002c068d44e4aac8ea282e39788f015418c82 Mon Sep 17 00:00:00 2001 From: Tristan Kretzer Date: Tue, 5 Nov 2024 22:28:44 +0100 Subject: [PATCH 1/3] Add function to run shell commands in a non-blocking way Add function `runCommandNonBlocking()` to run shell commands in a non-blocking way and refactor `runCommand()` and `runCommandWithExitCode()` to use the newly created function. Additionally, add parameter `timestamp` to function `runCommand()`. Breaking changes: * Renamed parameter `nonBlocking` to `discardOutput` in `runCommandWithExitCode()`. --- .../sumatra/ConfigureInverseSearchAction.kt | 4 +- .../run/sumatra/SumatraConversation.kt | 2 +- .../texifyidea/util/CommandRunner.kt | 127 ++++++++++++++++++ .../texifyidea/util/SystemEnvironment.kt | 98 +------------- 4 files changed, 131 insertions(+), 100 deletions(-) create mode 100644 src/nl/hannahsten/texifyidea/util/CommandRunner.kt diff --git a/src/nl/hannahsten/texifyidea/action/sumatra/ConfigureInverseSearchAction.kt b/src/nl/hannahsten/texifyidea/action/sumatra/ConfigureInverseSearchAction.kt index 69d8820d5..b227056e1 100644 --- a/src/nl/hannahsten/texifyidea/action/sumatra/ConfigureInverseSearchAction.kt +++ b/src/nl/hannahsten/texifyidea/action/sumatra/ConfigureInverseSearchAction.kt @@ -52,10 +52,10 @@ open class ConfigureInverseSearchAction : AnAction() { // We will assume that since the user is using a 64-bit IDEA that name64 exists, this is at least true for idea64.exe and pycharm64.exe on Windows name += "64" // We also remove an extra "" because it opens an empty IDEA instance when present - runCommandWithExitCode("cmd.exe", "/C", "start", "SumatraPDF", "-inverse-search", "\"$path\\$name.exe\" --line %l \"%f\"", workingDirectory = sumatraWorkingDir, nonBlocking = true) + runCommandWithExitCode("cmd.exe", "/C", "start", "SumatraPDF", "-inverse-search", "\"$path\\$name.exe\" --line %l \"%f\"", workingDirectory = sumatraWorkingDir, discardOutput = true) } else { - runCommandWithExitCode("cmd.exe", "/C", "start", "SumatraPDF", "-inverse-search", "\"$path\\$name.exe\" \"\" --line %l \"%f\"", workingDirectory = sumatraWorkingDir, nonBlocking = true) + runCommandWithExitCode("cmd.exe", "/C", "start", "SumatraPDF", "-inverse-search", "\"$path\\$name.exe\" \"\" --line %l \"%f\"", workingDirectory = sumatraWorkingDir, discardOutput = true) } dialogWrapper.close(0) diff --git a/src/nl/hannahsten/texifyidea/run/sumatra/SumatraConversation.kt b/src/nl/hannahsten/texifyidea/run/sumatra/SumatraConversation.kt index 5999cf44a..2da78811a 100644 --- a/src/nl/hannahsten/texifyidea/run/sumatra/SumatraConversation.kt +++ b/src/nl/hannahsten/texifyidea/run/sumatra/SumatraConversation.kt @@ -41,7 +41,7 @@ object SumatraConversation : ViewerConversation() { catch (e: TeXception) { // Make sure Windows popup error doesn't appear and we will still open Sumatra if (SumatraAvailabilityChecker.isSumatraAvailable) { - runCommandWithExitCode("cmd.exe", "/C", "start", "SumatraPDF", "-reuse-instance", pdfFilePath, workingDirectory = SumatraAvailabilityChecker.sumatraDirectory, nonBlocking = true) + runCommandWithExitCode("cmd.exe", "/C", "start", "SumatraPDF", "-reuse-instance", pdfFilePath, workingDirectory = SumatraAvailabilityChecker.sumatraDirectory, discardOutput = true) } } } diff --git a/src/nl/hannahsten/texifyidea/util/CommandRunner.kt b/src/nl/hannahsten/texifyidea/util/CommandRunner.kt new file mode 100644 index 000000000..80e364bcb --- /dev/null +++ b/src/nl/hannahsten/texifyidea/util/CommandRunner.kt @@ -0,0 +1,127 @@ +package nl.hannahsten.texifyidea.util + +import com.intellij.execution.ExecutionException +import com.intellij.execution.configurations.GeneralCommandLine +import com.intellij.util.io.awaitExit +import kotlinx.coroutines.* +import java.io.File +import java.io.IOException +import javax.swing.SwingUtilities + +data class CommandResult( + val exitCode: Int, + val standardOutput: String?, + val errorOutput: String? +) { + + val output: String? + get() = if (standardOutput != null || errorOutput != null) (standardOutput ?: "") + (errorOutput ?: "") else null +} + +/** + * Run a command in the terminal in a non-blocking way. + * + * @param workingDirectory If provided, the process' working directory. + * @param input If provided, this will be written to the process' input pipe. + * @param discardOutput Whether to discard all command outputs (stdout, stderr) and only return its exit code. + * @param returnExceptionMessageAsErrorOutput Whether to return exception messages as error output if exceptions are thrown. + * @param timeout The timeout for execution. Does not stop reading the process' output as long as it is available. + */ +suspend fun runCommandNonBlocking( + vararg commands: String, + workingDirectory: File? = null, + input: String? = null, + discardOutput: Boolean = false, + returnExceptionMessageAsErrorOutput: Boolean = false, + timeout: Long = 3 +): CommandResult = withContext(Dispatchers.IO) { + try { + Log.debug("isEDT=${SwingUtilities.isEventDispatchThread()} Executing in ${workingDirectory ?: "current working directory"} ${GeneralCommandLine(*commands).commandLineString}") + + val processBuilder = GeneralCommandLine(*commands) + .withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE) + .withWorkDirectory(workingDirectory) + .toProcessBuilder() + + if (discardOutput) { + processBuilder.redirectOutput(ProcessBuilder.Redirect.DISCARD) + processBuilder.redirectError(ProcessBuilder.Redirect.DISCARD) + } + + val process = processBuilder.start() + + process.outputWriter().use { if (input != null) it.write(input) } + val output = if (!discardOutput) async { process.inputReader().use { it.readText() } } else null + val error = if (!discardOutput) async { process.errorReader().use { it.readText() } } else null + + withTimeoutOrNull(1_000 * timeout) { + process.awaitExit() + } ?: run { + process.destroy() + Log.debug("${commands.firstOrNull()} destroyed after timeout $timeout seconds") + } + + val result = CommandResult(process.awaitExit(), output?.await()?.trim(), error?.await()?.trim()) + Log.debug("${commands.firstOrNull()} exited with ${result.exitCode} ${result.standardOutput?.take(100)} ${result.errorOutput?.take(100)}") + + return@withContext result + } + catch (e: IOException) { + Log.debug(e.message ?: "Unknown IOException occurred") + + return@withContext CommandResult( + -1, + null, + if (returnExceptionMessageAsErrorOutput) e.message else null + ) + } + catch (e: ExecutionException) { + Log.debug(e.message ?: "Unknown ExecutionException occurred") + + return@withContext CommandResult( + -1, + null, + if (returnExceptionMessageAsErrorOutput) e.message else null + ) + } +} + +/** + * Run a command in the terminal. + * + * @return The output of the command or null if an exception was thrown. + */ +fun runCommand(vararg commands: String, workingDirectory: File? = null, timeout: Long = 3): String? = + runBlocking { + runCommandNonBlocking(*commands, workingDirectory = workingDirectory, timeout = timeout).output + } + +/** + * See [runCommandNonBlocking]. + * + * @param returnExceptionMessage Whether to return exception messages as output if exceptions are thrown. + * @param inputString If provided, this will be written to the process' input pipe. + * @return Pair of output (stdout + stderr) to exit code. + */ +fun runCommandWithExitCode( + vararg commands: String, + workingDirectory: File? = null, + timeout: Long = 3, + returnExceptionMessage: Boolean = false, + discardOutput: Boolean = false, + inputString: String = "" +): Pair = + runBlocking { + with( + runCommandNonBlocking( + *commands, + workingDirectory = workingDirectory, + timeout = timeout, + returnExceptionMessageAsErrorOutput = returnExceptionMessage, + discardOutput = discardOutput, + input = inputString + ) + ) { + Pair(output, exitCode) + } + } \ No newline at end of file diff --git a/src/nl/hannahsten/texifyidea/util/SystemEnvironment.kt b/src/nl/hannahsten/texifyidea/util/SystemEnvironment.kt index cd55d0bfd..d9ae2ce0b 100644 --- a/src/nl/hannahsten/texifyidea/util/SystemEnvironment.kt +++ b/src/nl/hannahsten/texifyidea/util/SystemEnvironment.kt @@ -1,21 +1,16 @@ package nl.hannahsten.texifyidea.util import com.intellij.execution.RunManager -import com.intellij.execution.configurations.GeneralCommandLine import com.intellij.execution.impl.RunManagerImpl -import com.intellij.execution.process.ProcessNotCreatedException import com.intellij.openapi.project.Project import com.intellij.openapi.project.guessProjectDir import com.intellij.openapi.util.SystemInfo import com.intellij.openapi.vfs.LocalFileSystem import com.intellij.openapi.vfs.VirtualFile import nl.hannahsten.texifyidea.run.latex.LatexRunConfiguration -import nl.hannahsten.texifyidea.util.files.* +import nl.hannahsten.texifyidea.util.files.allChildDirectories import org.apache.maven.artifact.versioning.DefaultArtifactVersion import java.io.File -import java.io.IOException -import java.util.concurrent.TimeUnit -import javax.swing.SwingUtilities /** * Information about the system other than the LatexDistribution or the OS. @@ -68,97 +63,6 @@ class SystemEnvironment { } } -/** - * Run a command in the terminal. - * - * @return The output of the command or null if an exception was thrown. - */ -fun runCommand(vararg commands: String, workingDirectory: File? = null): String? { - return runCommandWithExitCode(*commands, workingDirectory = workingDirectory).first -} - -/** - * See [runCommand], but also returns exit code. - * - * @param returnExceptionMessage Whether to return exception messages if exceptions are thrown. - * @param nonBlocking If true, the function will not block waiting for output - * @param inputString If provided, this will be written to the process outputStream before starting the process. - */ -fun runCommandWithExitCode(vararg commands: String, workingDirectory: File? = null, timeout: Long = 3, returnExceptionMessage: Boolean = false, nonBlocking: Boolean = false, inputString: String = ""): Pair { - Log.debug("isEDT=${SwingUtilities.isEventDispatchThread()} Executing in ${workingDirectory ?: "current working directory"} ${GeneralCommandLine(*commands).commandLineString}") - return try { - val proc = GeneralCommandLine(*commands) - .withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE) - .withWorkDirectory(workingDirectory) - .createProcess() - - if (inputString.isNotBlank()) { - proc.outputStream.bufferedWriter().apply { - write(inputString) - close() - } - } - - if (proc.waitFor(timeout, TimeUnit.SECONDS)) { - val output = readInputStream(nonBlocking, proc) - Log.debug("${commands.firstOrNull()} exited with ${proc.exitValue()} ${output.take(100)}") - return Pair(output, proc.exitValue()) - } - else { - var output = "" - // If the program has timed out, something is stuck so we are not going to wait until it prints its stdout/stderr, we just check if ready and otherwise are out of luck - if (proc.inputStream.bufferedReader().ready()) { - output += proc.inputStream.bufferedReader().readText().trim() - } - if (proc.errorStream.bufferedReader().ready()) { - output += proc.errorStream.bufferedReader().readText().trim() - } - proc.destroy() - proc.waitFor() - Log.debug("${commands.firstOrNull()} exited ${proc.exitValue()} with timeout") - Pair(output, proc.exitValue()) - } - } - catch (e: IOException) { - Log.debug(e.message ?: "Unknown IOException occurred") - if (!returnExceptionMessage) { - Pair(null, -1) // Don't print the stacktrace because that is confusing. - } - else { - Pair(e.message, -1) - } - } - catch (e: ProcessNotCreatedException) { - Log.debug(e.message ?: "Unknown ProcessNotCreatedException occurred") - // e.g. if the command is just trying if a program can be run or not, and it's not the case - if (!returnExceptionMessage) { - Pair(null, -1) - } - else { - Pair(e.message, -1) - } - } -} - -/** - * Read input and error streams. If non-blocking, we will skip reading the streams if they are not ready. - */ -private fun readInputStream(nonBlocking: Boolean, proc: Process): String { - var output = "" - if (nonBlocking) { - if (proc.inputStream.bufferedReader().ready()) { - output += proc.inputStream.bufferedReader().readText().trim() - } - if (proc.errorStream.bufferedReader().ready()) { - output += proc.errorStream.bufferedReader().readText().trim() - } - } - else { - output = proc.inputStream.bufferedReader().readText().trim() + proc.errorStream.bufferedReader().readText().trim() - } - return output -} - /** * Collect texinputs from various places * From 139c9145b4e5b353a5ae61605db91147f037cb23 Mon Sep 17 00:00:00 2001 From: Tristan Kretzer Date: Tue, 5 Nov 2024 22:46:16 +0100 Subject: [PATCH 2/3] Fix empty LatexPackageLocationCache on Windows using Tex Live Full Increase timeout for kpsewhich path expansion to prevent LatexPackageLocationCache being empty. Additionally, make `fillCacheWithKpsewhich()` a suspend function, use non-blocking shell command execution and add logging to analyze number of paths loaded into LatexPackageLocationCache. See !3653 for details and measurements. --- .../util/files/LatexPackageLocationCache.kt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nl/hannahsten/texifyidea/util/files/LatexPackageLocationCache.kt b/src/nl/hannahsten/texifyidea/util/files/LatexPackageLocationCache.kt index 4dd7e73c2..de6ac8e31 100644 --- a/src/nl/hannahsten/texifyidea/util/files/LatexPackageLocationCache.kt +++ b/src/nl/hannahsten/texifyidea/util/files/LatexPackageLocationCache.kt @@ -2,9 +2,11 @@ package nl.hannahsten.texifyidea.util.files import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.LocalFileSystem +import kotlinx.coroutines.runBlocking import nl.hannahsten.texifyidea.settings.sdk.LatexSdkUtil import nl.hannahsten.texifyidea.settings.sdk.TectonicSdk -import nl.hannahsten.texifyidea.util.runCommand +import nl.hannahsten.texifyidea.util.Log +import nl.hannahsten.texifyidea.util.runCommandNonBlocking import java.io.File /** @@ -21,19 +23,22 @@ object LatexPackageLocationCache { * Fill cache with all paths of all files in the LaTeX installation. * Note: this can take a long time. */ - fun fillCacheWithKpsewhich(project: Project) { + suspend fun fillCacheWithKpsewhich(project: Project) { // We will get all search paths that kpsewhich has, expand them and find all files // Source: https://www.tug.org/texinfohtml/kpathsea.html#Casefolding-search // We cannot just fill the cache on the fly, because then we will also run kpsewhich when the user is still typing a package name, so we will run it once for every letter typed and this is already too expensive. // We cannot rely on ls-R databases because they are not always populated, and running mktexlsr may run into permission issues. val executableName = LatexSdkUtil.getExecutableName("kpsewhich", project) - val searchPaths = (runCommand(executableName, "-show-path=tex") ?: ".") + File.pathSeparator + (runCommand(executableName, "-show-path=bib") ?: ".") - cache = runCommand(executableName, "-expand-path", searchPaths)?.split(File.pathSeparator) + val searchPaths = (runCommandNonBlocking(executableName, "-show-path=tex").standardOutput ?: ".") + File.pathSeparator + (runCommandNonBlocking(executableName, "-show-path=bib").standardOutput ?: ".") + + cache = runCommandNonBlocking(executableName, "-expand-path", searchPaths, timeout = 10).standardOutput?.split(File.pathSeparator) ?.flatMap { LocalFileSystem.getInstance().findFileByPath(it)?.children?.toList() ?: emptyList() } ?.filter { !it.isDirectory } ?.toSet() ?.associate { it.name to it.path } ?.toMutableMap() ?: mutableMapOf() + + Log.debug("Latex package location cache generated with ${cache?.size} paths") } /** @@ -46,7 +51,7 @@ object LatexPackageLocationCache { */ fun getPackageLocation(name: String, project: Project): String? { if (cache == null) { - fillCacheWithKpsewhich(project) + runBlocking { fillCacheWithKpsewhich(project) } } // Tectonic does not have kpsewhich, but works a little differently From c956459534516930043f1c241a02f5c523f9ce76 Mon Sep 17 00:00:00 2001 From: Tristan Kretzer Date: Sat, 9 Nov 2024 22:16:45 +0100 Subject: [PATCH 3/3] Add extension function `String.runCommandNonBlocking()` Additionally, update usages of `String.runCommand()` in suspending functions to use `String.runCommandNonBlocking()`. --- .../texifyidea/startup/TexLivePackageListInitializer.kt | 4 ++-- src/nl/hannahsten/texifyidea/util/Strings.kt | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/nl/hannahsten/texifyidea/startup/TexLivePackageListInitializer.kt b/src/nl/hannahsten/texifyidea/startup/TexLivePackageListInitializer.kt index 0f9784ac9..b23a97a24 100644 --- a/src/nl/hannahsten/texifyidea/startup/TexLivePackageListInitializer.kt +++ b/src/nl/hannahsten/texifyidea/startup/TexLivePackageListInitializer.kt @@ -4,13 +4,13 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.startup.ProjectActivity import nl.hannahsten.texifyidea.settings.sdk.TexliveSdk import nl.hannahsten.texifyidea.util.TexLivePackages -import nl.hannahsten.texifyidea.util.runCommand +import nl.hannahsten.texifyidea.util.runCommandNonBlocking class TexLivePackageListInitializer : ProjectActivity { override suspend fun execute(project: Project) { if (TexliveSdk.Cache.isAvailable) { - val result = "tlmgr list --only-installed".runCommand() ?: return + val result = "tlmgr list --only-installed".runCommandNonBlocking().output ?: return TexLivePackages.packageList = Regex("i\\s(.*):").findAll(result) .map { it.groupValues.last() }.toMutableList() } diff --git a/src/nl/hannahsten/texifyidea/util/Strings.kt b/src/nl/hannahsten/texifyidea/util/Strings.kt index ac5703393..e4665ad9a 100644 --- a/src/nl/hannahsten/texifyidea/util/Strings.kt +++ b/src/nl/hannahsten/texifyidea/util/Strings.kt @@ -209,6 +209,12 @@ fun String.removeHtmlTags() = this.replace(PatternMagic.htmlTag.toRegex(), "") fun String.runCommand(workingDirectory: File? = null) = runCommand(*(this.split("\\s".toRegex())).toTypedArray(), workingDirectory = workingDirectory) +/** + * See [String.runCommand] but implemented in a non-blocking way. + */ +suspend fun String.runCommandNonBlocking(workingDirectory: File? = null) = + runCommandNonBlocking(*(this.split("\\s".toRegex())).toTypedArray(), workingDirectory = workingDirectory) + fun String.runCommandWithExitCode(workingDirectory: File? = null) = runCommandWithExitCode(*(this.split("\\s".toRegex())).toTypedArray(), workingDirectory = workingDirectory)