From 65839ced31969affe23ecc483c95872684e7dd0b Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Wed, 5 Jun 2024 23:13:03 -0400 Subject: [PATCH] feat(backend): mount EmptyDir volumes for launcher write locations (#10857) Launcher writes input artifacts to root paths /gcs, /minio, and /s3. These paths are not accessible by non-root users by default, which is problematic in locked-down Kubernetes installations and/or OpenShift. /gcs is currently a contract for KFP v2 python component wrappers, so the path cannot be changed. Mount an EmptyDir scratch volume to these paths to work around this. Additionally, /.local and /.cache are written to by pip, so add EmptyDir mounts for those too. Fixes: #5673 Fixes: #7345 Signed-off-by: Humair Khan Co-authored-by: Greg Sheremeta --- .../src/v2/compiler/argocompiler/container.go | 114 +++++++++++++++--- .../create_mount_delete_dynamic_pvc.yaml | 24 ++++ .../testdata/create_pod_metadata.yaml | 32 ++++- .../argocompiler/testdata/hello_world.yaml | 24 ++++ 4 files changed, 172 insertions(+), 22 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 72b2f8350b7..0dbb38a1117 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -26,11 +26,23 @@ import ( ) const ( - volumeNameKFPLauncher = "kfp-launcher" - DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f" - LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" - DefaultDriverImage = "gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec" - DriverImageEnvVar = "V2_DRIVER_IMAGE" + volumeNameKFPLauncher = "kfp-launcher" + DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f" + LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" + DefaultDriverImage = "gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec" + DriverImageEnvVar = "V2_DRIVER_IMAGE" + gcsScratchLocation = "/gcs" + gcsScratchName = "gcs-scratch" + s3ScratchLocation = "/s3" + s3ScratchName = "s3-scratch" + minioScratchLocation = "/minio" + minioScratchName = "minio-scratch" + dotLocalScratchLocation = "/.local" + dotLocalScratchName = "dot-local-scratch" + dotCacheScratchLocation = "/.cache" + dotCacheScratchName = "dot-cache-scratch" + dotConfigScratchLocation = "/.config" + dotConfigScratchName = "dot-config-scratch" ) func (c *workflowCompiler) Container(name string, component *pipelinespec.ComponentSpec, container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec) error { @@ -241,21 +253,61 @@ func (c *workflowCompiler) addContainerExecutorTemplate(refName string) string { // args come from. It is treated as a strategic merge patch on // top of the Pod spec. PodSpecPatch: inputValue(paramPodSpecPatch), - Volumes: []k8score.Volume{{ - Name: volumeNameKFPLauncher, - VolumeSource: k8score.VolumeSource{ - EmptyDir: &k8score.EmptyDirVolumeSource{}, + Volumes: []k8score.Volume{ + { + Name: volumeNameKFPLauncher, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, }, - }}, + { + Name: gcsScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: s3ScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: minioScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: dotLocalScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: dotCacheScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + { + Name: dotConfigScratchName, + VolumeSource: k8score.VolumeSource{ + EmptyDir: &k8score.EmptyDirVolumeSource{}, + }, + }, + }, InitContainers: []wfapi.UserContainer{{ Container: k8score.Container{ Name: "kfp-launcher", Image: GetLauncherImage(), Command: []string{"launcher-v2", "--copy", component.KFPLauncherPath}, - VolumeMounts: []k8score.VolumeMount{{ - Name: volumeNameKFPLauncher, - MountPath: component.VolumePathKFPLauncher, - }}, + VolumeMounts: []k8score.VolumeMount{ + { + Name: volumeNameKFPLauncher, + MountPath: component.VolumePathKFPLauncher, + }, + }, Resources: launcherResources, }, }}, @@ -268,10 +320,36 @@ func (c *workflowCompiler) addContainerExecutorTemplate(refName string) string { // These are added to pass argo workflows linting. Image: "gcr.io/ml-pipeline/should-be-overridden-during-runtime", Command: []string{"should-be-overridden-during-runtime"}, - VolumeMounts: []k8score.VolumeMount{{ - Name: volumeNameKFPLauncher, - MountPath: component.VolumePathKFPLauncher, - }}, + VolumeMounts: []k8score.VolumeMount{ + { + Name: volumeNameKFPLauncher, + MountPath: component.VolumePathKFPLauncher, + }, + { + Name: gcsScratchName, + MountPath: gcsScratchLocation, + }, + { + Name: s3ScratchName, + MountPath: s3ScratchLocation, + }, + { + Name: minioScratchName, + MountPath: minioScratchLocation, + }, + { + Name: dotLocalScratchName, + MountPath: dotLocalScratchLocation, + }, + { + Name: dotCacheScratchName, + MountPath: dotCacheScratchLocation, + }, + { + Name: dotConfigScratchName, + MountPath: dotConfigScratchLocation, + }, + }, EnvFrom: []k8score.EnvFromSource{metadataEnvFrom}, Env: commonEnvs, }, diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml index 7402e9e16d8..e3b427d2455 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml @@ -142,6 +142,18 @@ spec: volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch initContainers: - command: - launcher-v2 @@ -168,6 +180,18 @@ spec: volumes: - emptyDir: {} name: kfp-launcher + - emptyDir: { } + name: gcs-scratch + - emptyDir: { } + name: s3-scratch + - emptyDir: { } + name: minio-scratch + - emptyDir: { } + name: dot-local-scratch + - emptyDir: { } + name: dot-cache-scratch + - emptyDir: { } + name: dot-config-scratch - dag: tasks: - arguments: diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml index 8b623b87e6a..450aed6aeca 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml @@ -124,8 +124,20 @@ spec: name: "" resources: {} volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch initContainers: - command: - launcher-v2 @@ -156,8 +168,20 @@ spec: outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' volumes: - - emptyDir: {} - name: kfp-launcher + - emptyDir: {} + name: kfp-launcher + - emptyDir: { } + name: gcs-scratch + - emptyDir: { } + name: s3-scratch + - emptyDir: { } + name: minio-scratch + - emptyDir: { } + name: dot-local-scratch + - emptyDir: { } + name: dot-cache-scratch + - emptyDir: { } + name: dot-config-scratch - dag: tasks: - arguments: diff --git a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml index 7d44562a534..5685ece5de5 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml @@ -125,6 +125,18 @@ spec: volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch initContainers: - command: - launcher-v2 @@ -151,6 +163,18 @@ spec: volumes: - emptyDir: {} name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch - dag: tasks: - arguments: