From 4db94c59bd43d4c49e4a2cffb0e0a4daad414df0 Mon Sep 17 00:00:00 2001 From: Peter Trifanov Date: Mon, 19 Sep 2022 12:02:01 +0300 Subject: [PATCH 1/3] [skip ci] [WIP] save-agent requires authentication * NetworkPolicy to allow egress only to orchestrator, kube-dns and the Internet from save-agent pods * Don't automount ServiceAccount token into save-agent pods * Field `Agent.isAuthenticated` --- .../templates/save-agent-network-policy.yaml | 31 +++++++++++++++++++ .../com/saveourtool/save/entities/Agent.kt | 5 ++- .../kubernetes/KubernetesManager.kt | 2 ++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 save-cloud-charts/save-cloud/templates/save-agent-network-policy.yaml diff --git a/save-cloud-charts/save-cloud/templates/save-agent-network-policy.yaml b/save-cloud-charts/save-cloud/templates/save-agent-network-policy.yaml new file mode 100644 index 0000000000..7f428e1236 --- /dev/null +++ b/save-cloud-charts/save-cloud/templates/save-agent-network-policy.yaml @@ -0,0 +1,31 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + namespace: save-agent-network-policy +spec: + podSelector: + matchLabels: + io.kompose.service: save-agent + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + io.kompose.service: orchestrator + - to: + # https://stackoverflow.com/q/73049535 + - ipBlock: + cidr: 0.0.0.0/0 + # Forbid private IP ranges effectively allowing only egress to the Internet + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: "kube-system" + - podSelector: + matchLabels: + k8s-app: "kube-dns" \ No newline at end of file diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Agent.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Agent.kt index 892a9fa65c..b5352ac06c 100644 --- a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Agent.kt +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Agent.kt @@ -8,7 +8,8 @@ import javax.persistence.ManyToOne /** * @property containerId id of the container, inside which the agent is running * @property execution id of the execution, which the agent is serving - * @property version + * @property version version of the agent binary + * @property isAuthenticated whether this agent has already received a token from orchestrator */ @Entity class Agent( @@ -19,4 +20,6 @@ class Agent( var execution: Execution, var version: String? = null, + + var isAuthenticated: Boolean, ) : BaseEntity() diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt index adfb84f2cd..7a823d706c 100644 --- a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt @@ -74,6 +74,8 @@ class KubernetesManager( } // If agent fails, we should handle it manually (update statuses, attempt restart etc.) restartPolicy = "Never" + // save-agent pods shouldn't have access to valid cluster tokens + automountServiceAccountToken = false containers = listOf( agentContainerSpec(baseImageTag, agentRunCmd, workingDir, configuration.env) ) From d81ba231fef770d3f3b20dc51c914cf3d5a32b61 Mon Sep 17 00:00:00 2001 From: Peter Trifanov Date: Mon, 19 Sep 2022 17:53:18 +0300 Subject: [PATCH 2/3] [skip ci] [WIP] A dedicated user for running the tested tool inside the save-agent container --- save-deploy/base-images/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/save-deploy/base-images/Dockerfile b/save-deploy/base-images/Dockerfile index 2ce71c75a8..827d8896b1 100644 --- a/save-deploy/base-images/Dockerfile +++ b/save-deploy/base-images/Dockerfile @@ -14,9 +14,13 @@ RUN if [ "$BASE_IMAGE_NAME" = "python" ]; then \ fi RUN groupadd --gid 1100 save-agent && \ + groupadd --gid 1200 save-executor && \ useradd --uid 1100 --gid 1100 --create-home --shell /bin/sh save-agent && \ + useradd --uid 1200 --gid 1100 --create-home --shell /bin/sh save-agent && \ + usermod -aG save-executor save-agent && \ # `WORKDIR` directive creates the directory as `root` user unless the directory already exists mkdir /home/save-agent/save-execution && \ - chown -R 1100:1100 /home/save-agent/save-execution + chown -R 1100:1100 /home/save-agent/save-execution && \ + echo 'save-agent ALL = (save-executor) ALL' >> /etc/sudoers USER save-agent WORKDIR /home/save-agent/save-execution \ No newline at end of file From bcae1774accde2b7786cab007748663c06524257 Mon Sep 17 00:00:00 2001 From: sanyavertolet Date: Thu, 6 Apr 2023 18:31:50 +0300 Subject: [PATCH 3/3] [skip ci] Authenticated agent ### What's done: * Made save-agent run tested tool under save-executor user * Made save-demo-agent run demo tool under save-executor user * Added parentUserName and childUserName to agents configuration * Added token key protection if parentUserName is provided --- .../save/agent/AgentConfiguration.kt | 7 ++++ .../kotlin/com/saveourtool/save/agent/Main.kt | 7 ++-- .../com/saveourtool/save/agent/SaveAgent.kt | 4 ++ .../save-cloud/templates/demo-configmap.yaml | 2 + .../templates/orchestrator-configmap.yaml | 2 + .../templates/sandbox-configmap.yaml | 2 + save-cloud-charts/save-cloud/values.yaml | 2 + .../saveourtool/save/agent/AgentEnvName.kt | 2 + .../saveourtool/save/demo/DemoAgentConfig.kt | 4 ++ .../com/saveourtool/save/utils/Constants.kt | 2 + .../com/saveourtool/save/utils/FileUtils.kt | 24 ++++++++++- .../com/saveourtool/save/utils/HttpUtils.kt | 5 ++- .../com/saveourtool/save/utils/FileUtils.kt | 5 +++ .../com/saveourtool/save/utils/FileUtils.kt | 16 +++++++- .../com/saveourtool/save/utils/FileUtils.kt | 17 ++++++++ .../com/saveourtool/save/demo/agent/Server.kt | 41 ++++++++++++------- .../save/demo/agent/SimpleRunner.kt | 30 +++++++++----- .../save/demo/agent/utils/SetupUtils.kt | 23 +++++++++-- .../save/demo/config/ConfigProperties.kt | 6 ++- .../save/demo/service/KubernetesService.kt | 2 + save-deploy/base-images/Dockerfile | 3 +- .../saveourtool/save/orchestrator/Utils.kt | 3 ++ .../orchestrator/config/ConfigProperties.kt | 4 ++ .../kubernetes/KubernetesManager.kt | 8 ++-- 24 files changed, 179 insertions(+), 42 deletions(-) diff --git a/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt b/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt index 1d8f6bd506..1594bd7fd1 100644 --- a/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt +++ b/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt @@ -9,6 +9,7 @@ import com.saveourtool.save.agent.utils.TEST_SUITES_DIR_NAME import com.saveourtool.save.core.config.LogType import com.saveourtool.save.core.config.OutputStreamType import com.saveourtool.save.core.config.ReportType +import com.saveourtool.save.utils.optionalEnv import com.saveourtool.save.utils.requiredEnv import generated.SAVE_CLOUD_VERSION @@ -25,6 +26,8 @@ import kotlinx.serialization.Serializable * @property debug whether debug logging should be enabled * @property testSuitesDir directory where tests and additional files need to be stored into * @property logFilePath path to logs of save-cli execution + * @property parentUserName name of a parent process user, needed for token isolation + * @property childUserName name of a child process user, needed for token isolation * @property save additional configuration for save-cli */ @Serializable @@ -37,6 +40,8 @@ data class AgentConfiguration( val debug: Boolean = false, val testSuitesDir: String = TEST_SUITES_DIR_NAME, val logFilePath: String = "logs.txt", + val parentUserName: String? = null, + val childUserName: String? = null, val save: SaveCliConfig = SaveCliConfig(), ) { companion object { @@ -52,6 +57,8 @@ data class AgentConfiguration( heartbeat = HeartbeatConfig( url = requiredEnv(AgentEnvName.HEARTBEAT_URL.name), ), + parentUserName = optionalEnv(AgentEnvName.PARENT_PROCESS_USERNAME.name), + childUserName = optionalEnv(AgentEnvName.CHILD_PROCESS_USERNAME.name), ) } } diff --git a/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/Main.kt b/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/Main.kt index 8e68a588bb..1a02a4749a 100644 --- a/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/Main.kt +++ b/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/Main.kt @@ -9,9 +9,7 @@ import com.saveourtool.save.agent.utils.ktorLogger import com.saveourtool.save.core.config.LogType import com.saveourtool.save.core.logging.describe import com.saveourtool.save.core.logging.logType -import com.saveourtool.save.utils.KubernetesServiceAccountAuthHeaderPlugin -import com.saveourtool.save.utils.fs -import com.saveourtool.save.utils.parseConfig +import com.saveourtool.save.utils.* import io.ktor.client.HttpClient import io.ktor.client.plugins.* @@ -51,9 +49,10 @@ fun main() { .updateFromEnv() logType.set(if (config.debug) LogType.ALL else LogType.WARN) logDebugCustom("Instantiating save-agent version ${config.info.version} with config $config") - handleSigterm() + config.parentUserName?.let { protectAuthToken(it, it) } + val httpClient = configureHttpClient(config) runBlocking { diff --git a/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/SaveAgent.kt b/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/SaveAgent.kt index 9da3cf2421..85215c0794 100644 --- a/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/SaveAgent.kt +++ b/save-agent/src/commonMain/kotlin/com/saveourtool/save/agent/SaveAgent.kt @@ -100,6 +100,7 @@ class SaveAgent( ?.let { fileName -> val targetFile = targetDirectory / fileName logDebugCustom("Additionally setup of evaluated tool by $targetFile") + // todo: protect me after ProcessBuilder is updated (https://github.com/saveourtool/save-cli/issues/521) val setupResult = ProcessBuilder(true, fs) .exec( "./$targetFile", @@ -287,6 +288,9 @@ class SaveAgent( private fun runSave(cliArgs: String): ExecutionResult { val fullCliCommand = buildString { + config.childUserName?.let { userName -> + append("sudo -u $userName ") + } append(config.cliCommand) append(" ${config.testSuitesDir}") append(" $cliArgs") diff --git a/save-cloud-charts/save-cloud/templates/demo-configmap.yaml b/save-cloud-charts/save-cloud/templates/demo-configmap.yaml index 5a197dd2d0..327dfcafab 100644 --- a/save-cloud-charts/save-cloud/templates/demo-configmap.yaml +++ b/save-cloud-charts/save-cloud/templates/demo-configmap.yaml @@ -22,3 +22,5 @@ data: demo.backend-url=http://backend/internal demo.agent-config.demo-url=http://demo + demo.agent-config.parent-user-name={{ .Values.agentParentUserName }} + demo.agent-config.child-user-name={{ .Values.agentChildUserName }} diff --git a/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml b/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml index c90e2c9bbb..e6fcbe973f 100644 --- a/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml +++ b/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml @@ -16,6 +16,8 @@ data: management.server.port={{ .Values.orchestrator.managementPort }} orchestrator.agent-settings.heartbeat-url=http://{{ .Values.orchestrator.name }}/heartbeat orchestrator.agent-settings.debug=true + orchestrator.agent-settings.parent-user-name={{ .Values.agentParentUserName }} + orchestrator.agent-settings.child-user-name={{ .Values.agentChildUserName }} logging.level.com.saveourtool.save.orchestrator.kubernetes=DEBUG logging.level.org.springframework=DEBUG diff --git a/save-cloud-charts/save-cloud/templates/sandbox-configmap.yaml b/save-cloud-charts/save-cloud/templates/sandbox-configmap.yaml index 3a304a1c94..fd61a7c384 100644 --- a/save-cloud-charts/save-cloud/templates/sandbox-configmap.yaml +++ b/save-cloud-charts/save-cloud/templates/sandbox-configmap.yaml @@ -17,6 +17,8 @@ data: sandbox.agent-settings.sandbox-url=http://{{ .Values.sandbox.name }} orchestrator.agent-settings.heartbeat-url=http://{{ .Values.sandbox.name }}/heartbeat orchestrator.agent-settings.debug=true + orchestrator.agent-settings.parent-user-name={{ .Values.agentParentUserName }} + orchestrator.agent-settings.child-user-name={{ .Values.agentChildUserName }} logging.level.com.saveourtool.save.orchestrator.kubernetes=DEBUG diff --git a/save-cloud-charts/save-cloud/values.yaml b/save-cloud-charts/save-cloud/values.yaml index 6648242358..64f4464404 100644 --- a/save-cloud-charts/save-cloud/values.yaml +++ b/save-cloud-charts/save-cloud/values.yaml @@ -221,3 +221,5 @@ demo: agentPort: 23456 agentNamespace: save-agent +agentParentUserName: save-agent +agentChildUserName: save-executor diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/agent/AgentEnvName.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/agent/AgentEnvName.kt index 611f16632f..5d4b6f3992 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/agent/AgentEnvName.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/agent/AgentEnvName.kt @@ -4,12 +4,14 @@ package com.saveourtool.save.agent * Env names which agent supports and expects */ enum class AgentEnvName { + CHILD_PROCESS_USERNAME, CLI_COMMAND, CONTAINER_ID, CONTAINER_NAME, DEBUG, EXECUTION_ID, HEARTBEAT_URL, + PARENT_PROCESS_USERNAME, TEST_SUITES_DIR, ; } diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/demo/DemoAgentConfig.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/demo/DemoAgentConfig.kt index 6b5d5c4494..01435b316b 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/demo/DemoAgentConfig.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/demo/DemoAgentConfig.kt @@ -13,6 +13,8 @@ import kotlinx.serialization.Serializable * @property demoConfiguration all the information about current demo e.g. maintainer and version * @property runConfiguration all the required information to run demo * @property demoUrl url of save-demo + * @property parentUserName name of a parent process user, needed for token isolation + * @property childUserName name of a child process user, needed for token isolation * @property setupShTimeoutMillis amount of milliseconds to run setup.sh if it is present, [DEFAULT_SETUP_SH_TIMEOUT_MILLIS] by default */ @Serializable @@ -20,6 +22,8 @@ data class DemoAgentConfig( val demoUrl: String, val demoConfiguration: DemoConfiguration, val runConfiguration: RunConfiguration, + val parentUserName: String?, + val childUserName: String?, val setupShTimeoutMillis: Long = DEFAULT_SETUP_SH_TIMEOUT_MILLIS, ) { companion object { diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/Constants.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/Constants.kt index 658978451b..602840a7d6 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/Constants.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/Constants.kt @@ -53,3 +53,5 @@ const val CONTENT_LENGTH_CUSTOM = "Content-Length-Custom" * Default time to execute setup.sh */ const val DEFAULT_SETUP_SH_TIMEOUT_MILLIS = 60_000L + +const val DEFAULT_KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/FileUtils.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/FileUtils.kt index 3af5f23e24..aed66a478c 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/FileUtils.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/FileUtils.kt @@ -15,7 +15,15 @@ import okio.Path.Companion.toPath expect val fs: FileSystem /** - * Mark [this] file as executable. Sets permissions to rwxr--r-- + * Sets permissions to [this] file to r--|---|--- + * + * @param ownerName name of owner user + * @param groupName name of group + */ +expect fun Path.permitReadingOnlyForOwner(ownerName: String, groupName: String) + +/** + * Mark [this] file as executable. Sets permissions to rwx|r--|r-- */ expect fun Path.markAsExecutable() @@ -98,3 +106,17 @@ inline fun parseConfigOrDefault( logInfo("Config file $configName not found, falling back to default config.") defaultConfig } + +/** + * Allow reading file with path [tokenPathString] to owner only + * + * @param ownerName name of an owner to permit reading + * @param groupName name of an owner's group to permit reading + * @param tokenPathString path to token file + * @return Unit + */ +fun protectAuthToken( + ownerName: String, + groupName: String, + tokenPathString: String = DEFAULT_KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH +) = tokenPathString.toPath().permitReadingOnlyForOwner(ownerName, groupName) diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/HttpUtils.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/HttpUtils.kt index 92c62ae5b5..fa712f86bb 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/HttpUtils.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/utils/HttpUtils.kt @@ -34,8 +34,9 @@ val KubernetesServiceAccountAuthHeaderPlugin = createClientPlugin( val token = ExpiringValueWrapper(pluginConfig.expirationTime) { fs.read(pluginConfig.tokenPath.toPath()) { readUtf8() } } + val headerName = pluginConfig.headerName onRequest { request, _ -> - request.headers.append(SA_HEADER_NAME, token.getValue()) + request.headers.append(headerName, token.getValue()) } } @@ -47,7 +48,7 @@ class KubernetesServiceAccountAuthHeaderPluginConfig { /** * Kubernetes service account token path configuration */ - var tokenPath: String = "/var/run/secrets/kubernetes.io/serviceaccount/token" + var tokenPath: String = DEFAULT_KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH /** * Token expiration [Duration] configuration diff --git a/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/FileUtils.kt b/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/FileUtils.kt index 0c16c141a8..cb6c7d2515 100644 --- a/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/FileUtils.kt +++ b/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/FileUtils.kt @@ -11,6 +11,11 @@ const val NOT_IMPLEMENTED_ON_JS = "Cannot be used in js." @Suppress("IMPLICIT_NOTHING_TYPE_ARGUMENT_IN_RETURN_POSITION") actual val fs: FileSystem by lazy { throw NotImplementedError(NOT_IMPLEMENTED_ON_JS) } + +actual fun Path.permitReadingOnlyForOwner(ownerName: String, groupName: String) { + throw NotImplementedError(NOT_IMPLEMENTED_ON_JS) +} + actual fun Path.markAsExecutable() { throw NotImplementedError(NOT_IMPLEMENTED_ON_JS) } diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt index 6eb6b7cda7..aeb571003e 100644 --- a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt @@ -22,7 +22,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption import java.nio.file.StandardOpenOption -import java.nio.file.attribute.PosixFilePermission +import java.nio.file.attribute.* import java.util.* import java.util.stream.Collectors @@ -130,6 +130,20 @@ fun Path.requireIsAbsolute(): Path = apply { } } +actual fun okio.Path.permitReadingOnlyForOwner(ownerName: String, groupName: String) { + val file = toFile().toPath() + val fileAttributeView = Files.getFileAttributeView(file, PosixFileAttributeView::class.java) + val lookupService = file.fileSystem.userPrincipalLookupService + + val owner = lookupService.lookupPrincipalByName(ownerName) + val group = lookupService.lookupPrincipalByGroupName(groupName) + + fileAttributeView.owner = owner + fileAttributeView.setGroup(group) + + Files.setPosixFilePermissions(file, EnumSet.of(PosixFilePermission.OWNER_READ)) +} + actual fun okio.Path.markAsExecutable() { val file = this.toFile().toPath() Files.setPosixFilePermissions(file, Files.getPosixFilePermissions(file) + EnumSet.of( diff --git a/save-cloud-common/src/nativeMain/kotlin/com/saveourtool/save/utils/FileUtils.kt b/save-cloud-common/src/nativeMain/kotlin/com/saveourtool/save/utils/FileUtils.kt index 324c4344b9..3b83850ecc 100644 --- a/save-cloud-common/src/nativeMain/kotlin/com/saveourtool/save/utils/FileUtils.kt +++ b/save-cloud-common/src/nativeMain/kotlin/com/saveourtool/save/utils/FileUtils.kt @@ -12,10 +12,27 @@ import platform.posix.* import kotlinx.cinterop.UnsafeNumber import kotlinx.cinterop.convert +import kotlinx.cinterop.pointed import kotlinx.serialization.serializer actual val fs: FileSystem = FileSystem.SYSTEM +@OptIn(UnsafeNumber::class) +@Suppress("TooGenericExceptionThrown") +actual fun Path.permitReadingOnlyForOwner(ownerName: String, groupName: String) { + val owner = requireNotNull(getpwnam(ownerName)) { "Could not find user with name $ownerName" } + val group = requireNotNull(getgrnam(groupName)) { "Could not find group with name $groupName" } + + if (chown(toString(), owner.pointed.pw_uid, group.pointed.gr_gid) != 0) { + throw RuntimeException("Could not change file owner or group") + } + + val mode: mode_t = S_IRUSR.convert() + if (chmod(toString(), mode) != 0) { + throw RuntimeException("Could not change file permissions") + } +} + @OptIn(UnsafeNumber::class) actual fun Path.markAsExecutable() { val mode: mode_t = (S_IRUSR or S_IWUSR or S_IXUSR or S_IRGRP or S_IROTH).convert() diff --git a/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/Server.kt b/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/Server.kt index b71281ca14..f4f3f55d2d 100644 --- a/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/Server.kt +++ b/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/Server.kt @@ -12,6 +12,7 @@ import com.saveourtool.save.demo.DemoRunRequest import com.saveourtool.save.demo.ServerConfiguration import com.saveourtool.save.demo.agent.utils.getConfiguration import com.saveourtool.save.demo.agent.utils.setupEnvironment +import com.saveourtool.save.utils.protectAuthToken import com.saveourtool.save.utils.retry import io.ktor.http.* import io.ktor.serialization.kotlinx.json.* @@ -48,10 +49,11 @@ private fun Application.getConfigurationOnStartup( logDebug("Configuration successfully fetched.") config } + ?.also { config -> config.parentUserName?.let { userName -> protectAuthToken(userName, userName) } } ?.also(updateConfig) ?.let { config -> logTrace("Configuration successfully updated.") - setupEnvironment(config.demoUrl, config.setupShTimeoutMillis, config.demoConfiguration) + setupEnvironment(config.demoUrl, config.setupShTimeoutMillis, config.childUserName, config.demoConfiguration) } ?: run { logWarn("Could not prepare save-demo-agent, expecting /setup call.") } } @@ -65,20 +67,31 @@ private fun Routing.alive(configuration: CompletableDeferred) = }) } -private fun Routing.configure(updateConfig: (DemoAgentConfig) -> Unit) = post("/setup") { - val config = call.receive().also(updateConfig) - logInfo("Agent has received configuration.") - try { - setupEnvironment(config.demoUrl, config.setupShTimeoutMillis, config.demoConfiguration) - call.respondText( - "Agent is set up.", - status = HttpStatusCode.OK, - ) - } catch (exception: IllegalStateException) { +private fun Routing.configure( + deferredConfig: CompletableDeferred, + updateConfig: (DemoAgentConfig) -> Unit, +) = post("/setup") { + if (deferredConfig.isCompleted) { call.respondText( - exception.message ?: "Internal agent error.", - status = HttpStatusCode.InternalServerError, + "save-demo-agent is already configured.", + status = HttpStatusCode.Conflict, ) + } else { + val config = call.receive().also(updateConfig) + logInfo("Agent has received configuration.") + config.parentUserName?.let { parentUserName -> protectAuthToken(parentUserName, parentUserName) } + try { + setupEnvironment(config.demoUrl, config.setupShTimeoutMillis, config.childUserName, config.demoConfiguration) + call.respondText( + "Agent is set up.", + status = HttpStatusCode.OK, + ) + } catch (exception: IllegalStateException) { + call.respondText( + exception.message ?: "Internal agent error.", + status = HttpStatusCode.InternalServerError, + ) + } } } @@ -110,7 +123,7 @@ fun server(serverConfiguration: ServerConfiguration, skipStartupConfiguration: B install(ContentNegotiation) { json() } routing { alive(deferredConfig) - configure { deferredConfig.complete(it) } + configure(deferredConfig) { deferredConfig.complete(it) } run(deferredConfig) } } diff --git a/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/SimpleRunner.kt b/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/SimpleRunner.kt index d37e589846..b6853c11c5 100644 --- a/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/SimpleRunner.kt +++ b/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/SimpleRunner.kt @@ -6,10 +6,8 @@ package com.saveourtool.save.demo.agent import com.saveourtool.save.core.files.readLines import com.saveourtool.save.core.utils.ProcessBuilder -import com.saveourtool.save.demo.DemoAgentConfig -import com.saveourtool.save.demo.DemoResult -import com.saveourtool.save.demo.DemoRunRequest -import com.saveourtool.save.demo.RunConfiguration +import com.saveourtool.save.demo.* +import com.saveourtool.save.demo.agent.utils.wrapCommandForUser import com.saveourtool.save.utils.createAndWrite import com.saveourtool.save.utils.createAndWriteIfNeeded import com.saveourtool.save.utils.fs @@ -31,13 +29,11 @@ private const val PROCESS_BUILDER_TIMEOUT_MILLIS = 20_000L fun runDemo(demoRunRequest: DemoRunRequest, deferredConfig: CompletableDeferred): DemoResult { require(demoRunRequest.mode.isNotBlank()) { "Demo mode should not be blank." } - val config = deferredConfig.getCompleted().runConfiguration - val runCommand = requireNotNull(config.runCommands[demoRunRequest.mode]) { - "Could not find run command for mode ${demoRunRequest.mode}." - } + val config = deferredConfig.getCompleted() + val runCommand = getRunCommand(config.runConfiguration.runCommands, demoRunRequest.mode, config.childUserName) - val (inputFile, configFile) = createRequiredFiles(demoRunRequest, config) - val outputFile = config.outputFileName?.toPath() + val (inputFile, configFile) = createRequiredFiles(demoRunRequest, config.runConfiguration) + val outputFile = config.runConfiguration.outputFileName?.toPath() return try { run(runCommand, inputFile, outputFile) @@ -46,6 +42,20 @@ fun runDemo(demoRunRequest: DemoRunRequest, deferredConfig: CompletableDeferred< } } +/** + * Get run command and prepend + * + * `sudo -u childProcessUserName` + * + * if [childProcessUserName] is not null + */ +private fun getRunCommand( + runCommands: RunCommandMap, + mode: String, + childProcessUserName: String?, +) = requireNotNull(runCommands[mode]) { "Could not find run command for mode $mode." } + .wrapCommandForUser(childProcessUserName) + private fun createRequiredFiles( demoRunRequest: DemoRunRequest, config: RunConfiguration, diff --git a/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/utils/SetupUtils.kt b/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/utils/SetupUtils.kt index aeb231d14d..d0ff663d51 100644 --- a/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/utils/SetupUtils.kt +++ b/save-demo-agent/src/nativeMain/kotlin/com/saveourtool/save/demo/agent/utils/SetupUtils.kt @@ -19,15 +19,26 @@ import okio.Path.Companion.toPath private const val SETUP_SH_LOGS_FILENAME = "setup.logs" private const val CWD = "." +/** + * @param childUserName name of child process user + */ +fun String.wrapCommandForUser(childUserName: String?) = "${childUserName?.let { " sudo -u $it " }.orEmpty()}$this" + /** * Download all the required files from save-demo * * @param demoUrl url to save-demo * @param setupShTimeoutMillis amount of milliseconds to run setup.sh if it is present + * @param childUserName name of user that the child process should be run by * @param demoConfiguration all the information required for tool download * @throws IllegalStateException when it was caught from [downloadDemoFiles] */ -suspend fun setupEnvironment(demoUrl: String, setupShTimeoutMillis: Long, demoConfiguration: DemoConfiguration) { +suspend fun setupEnvironment( + demoUrl: String, + setupShTimeoutMillis: Long, + childUserName: String?, + demoConfiguration: DemoConfiguration, +) { logInfo("Setting up the environment...") try { @@ -39,7 +50,7 @@ suspend fun setupEnvironment(demoUrl: String, setupShTimeoutMillis: Long, demoCo logDebug("All files successfully downloaded.") - val executionResult = executeSetupSh(setupShTimeoutMillis) + val executionResult = executeSetupSh(setupShTimeoutMillis, childUserName) executionResult?.let { if (executionResult.code != 0) { logError("Setup script has finished with ${executionResult.code} code.") @@ -51,12 +62,16 @@ suspend fun setupEnvironment(demoUrl: String, setupShTimeoutMillis: Long, demoCo logInfo("The environment is successfully set up.") } -private fun executeSetupSh(setupShTimeoutMillis: Long, setupShName: String = "setup.sh"): ExecutionResult? = setupShName.takeIf { +private fun executeSetupSh( + setupShTimeoutMillis: Long, + childUserName: String?, + setupShName: String = "setup.sh", +): ExecutionResult? = setupShName.takeIf { fs.exists(it.toPath()) } ?.let { setupSh -> ProcessBuilder(true, fs).exec( - "./$setupSh", + "./$setupSh".wrapCommandForUser(childUserName), CWD, SETUP_SH_LOGS_FILENAME.toPath(), setupShTimeoutMillis, diff --git a/save-demo/src/main/kotlin/com/saveourtool/save/demo/config/ConfigProperties.kt b/save-demo/src/main/kotlin/com/saveourtool/save/demo/config/ConfigProperties.kt index ed71d4901e..3fa4678f09 100644 --- a/save-demo/src/main/kotlin/com/saveourtool/save/demo/config/ConfigProperties.kt +++ b/save-demo/src/main/kotlin/com/saveourtool/save/demo/config/ConfigProperties.kt @@ -24,9 +24,13 @@ data class ConfigProperties( ) : S3OperationsProperties.Provider { /** * @property demoUrl url of save-demo + * @property parentUserName name of a parent process user, needed for token isolation + * @property childUserName name of a child process user, needed for token isolation */ data class AgentConfig( - val demoUrl: String + val demoUrl: String, + val parentUserName: String? = null, + val childUserName: String? = null, ) } diff --git a/save-demo/src/main/kotlin/com/saveourtool/save/demo/service/KubernetesService.kt b/save-demo/src/main/kotlin/com/saveourtool/save/demo/service/KubernetesService.kt index d768a4c67a..d03b04b285 100644 --- a/save-demo/src/main/kotlin/com/saveourtool/save/demo/service/KubernetesService.kt +++ b/save-demo/src/main/kotlin/com/saveourtool/save/demo/service/KubernetesService.kt @@ -111,6 +111,8 @@ class KubernetesService( agentConfig.demoUrl, demo.toDemoConfiguration(version), demo.toRunConfiguration(), + agentConfig.parentUserName, + agentConfig.childUserName, ) private fun createConfiguredJob(demo: Demo, downloadAgentUrl: String) { diff --git a/save-deploy/base-images/Dockerfile b/save-deploy/base-images/Dockerfile index 0ea2e40b8c..d52131c81d 100644 --- a/save-deploy/base-images/Dockerfile +++ b/save-deploy/base-images/Dockerfile @@ -16,11 +16,12 @@ RUN if [ "$BASE_IMAGE_NAME" = "python" ]; then \ RUN groupadd --gid 1100 save-agent && \ groupadd --gid 1200 save-executor && \ useradd --uid 1100 --gid 1100 --create-home --shell /bin/sh save-agent && \ - useradd --uid 1200 --gid 1100 --create-home --shell /bin/sh save-agent && \ + useradd --uid 1200 --gid 1100 --create-home --shell /bin/sh save-executor && \ usermod -aG save-executor save-agent && \ # `WORKDIR` directive creates the directory as `root` user unless the directory already exists mkdir /home/save-agent/save-execution && \ chown -R 1100:1100 /home/save-agent/save-execution && \ echo 'save-agent ALL = (save-executor) ALL' >> /etc/sudoers + USER save-agent WORKDIR /home/save-agent/save-execution diff --git a/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt b/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt index c4f6f73d4d..30bd4e48ce 100644 --- a/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt +++ b/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt @@ -98,6 +98,9 @@ internal fun fillAgentPropertiesFromConfiguration( with(agentSettings) { put(AgentEnvName.HEARTBEAT_URL, heartbeatUrl) debug?.let { put(AgentEnvName.DEBUG, it.toString()) } + + parentUserName?.let { put(AgentEnvName.PARENT_PROCESS_USERNAME, parentUserName) } + childUserName?.let { put(AgentEnvName.CHILD_PROCESS_USERNAME, childUserName) } } } diff --git a/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/config/ConfigProperties.kt b/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/config/ConfigProperties.kt index 79829f6c81..1d1a80f787 100644 --- a/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/config/ConfigProperties.kt +++ b/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/config/ConfigProperties.kt @@ -95,10 +95,14 @@ data class ConfigProperties( /** * @property heartbeatUrl url that will be used by save-agent to post heartbeats * @property debug whether debug logging should be enabled or not + * @property parentUserName name of a parent process user, needed for token isolation + * @property childUserName name of a child process user, needed for token isolation */ data class AgentSettings( val heartbeatUrl: String, val debug: Boolean? = null, + val parentUserName: String? = null, + val childUserName: String? = null, ) /** diff --git a/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt b/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt index 927f73c953..15234de43b 100644 --- a/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt +++ b/save-orchestrator-common/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt @@ -37,9 +37,10 @@ class KubernetesManager( "NestedBlockDepth", "ComplexMethod", ) - override fun createAndStart(executionId: Long, - configuration: ContainerService.RunConfiguration, - replicas: Int, + override fun createAndStart( + executionId: Long, + configuration: ContainerService.RunConfiguration, + replicas: Int, ) { val baseImageTag = configuration.imageTag val agentRunCmd = configuration.runCmd @@ -79,7 +80,6 @@ class KubernetesManager( // If agent fails, we should handle it manually (update statuses, attempt restart etc.) restartPolicy = "Never" // save-agent pods shouldn't have access to valid cluster tokens - automountServiceAccountToken = false containers = listOf( agentContainerSpec(baseImageTag, agentRunCmd, workingDir, configuration.env) )