diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 5a2616193738..348aeca4c089 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -17,6 +17,7 @@ package argocompiler import ( "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "strings" @@ -285,19 +286,17 @@ func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) str log.Debug("executorName: ", executorName) // If one of them matches the executorLabel we extracted earlier... if executorName == executorLabel { - // Get the corresponding list of commands. - commandList := executorValue.GetContainer().GetCommand() - // And the list of args. - argsList := executorValue.GetContainer().GetArgs() - if !(commandList == nil && argsList == nil) { // Some components have neither command nor args. - // Convert the command list into a hash string. - stringToHash := strings.Join(commandList, " ") - if argsList != nil { - // Convert the args list into a hash string and append it to the command list hash string. - stringToHash += strings.Join(argsList, " ") - - return hashString(stringToHash) + // Get the corresponding container. + container := executorValue.GetContainer() + if container != nil { + containerHash, err := hashValue(container) + if err != nil { + // Do not bubble up since this is not a breaking error + // and we can just return the componentName in full. + log.Debug("Error hashing container: ", err) } + + return containerHash } } } @@ -306,11 +305,16 @@ func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) str return componentName } -func hashString(s string) string { +// hashValue serializes and hashes a provided value. +func hashValue(value interface{}) (string, error) { + bytes, err := json.Marshal(value) + if err != nil { + return "", err + } h := sha256.New() - h.Write([]byte(s)) + h.Write([]byte(bytes)) - return hex.EncodeToString(h.Sum(nil)) + return hex.EncodeToString(h.Sum(nil)), nil } const ( diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml index d51c50da07a0..7a5565595ed3 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-comp value: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1 + - name: components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080 value: '{"executorLabel":"exec-comp"}' - - name: implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1 + - name: implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080 value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 @@ -21,13 +21,13 @@ spec: *\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}' - name: kubernetes-comp-comp-2 value: '{"pvcMount":[{"mountPath":"/reused_data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}' - - name: components-comp-createpvc + - name: components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"parameterType":"LIST"},"annotations":{"isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"isOptional":true,"parameterType":"STRING"},"size":{"parameterType":"STRING"},"storage_class_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"volume_name":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}' - - name: implementations-comp-createpvc + - name: implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767 value: '{"image":"argostub/createpvc"}' - - name: components-comp-deletepvc + - name: components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725 value: '{"executorLabel":"exec-deletepvc","inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}' - - name: implementations-comp-deletepvc + - name: implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725 value: '{"image":"argostub/deletepvc"}' - name: components-root value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}},"comp-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}}}}' @@ -197,11 +197,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config @@ -222,11 +222,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' - name: container - value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}' + value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config @@ -247,11 +247,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-createpvc}}' + value: '{{workflow.parameters.components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}' - name: container - value: '{{workflow.parameters.implementations-comp-createpvc}}' + value: '{{workflow.parameters.implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: createpvc @@ -259,11 +259,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-comp-deletepvc}}' + value: '{{workflow.parameters.components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}' - name: container - value: '{{workflow.parameters.implementations-comp-deletepvc}}' + value: '{{workflow.parameters.implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' depends: comp-2.Succeeded && createpvc.Succeeded diff --git a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml index 7e70001c9ded..67308ba33dda 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml @@ -8,9 +8,9 @@ spec: parameters: - name: kubernetes-comp-hello-world value: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}' - - name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser @@ -191,11 +191,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config diff --git a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml index b710aa48a0b4..6bc59366d8d9 100644 --- a/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml @@ -6,9 +6,9 @@ metadata: spec: arguments: parameters: - - name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b + - name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390 value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser @@ -183,11 +183,11 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: task value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - name: container - value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}' + value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' name: hello-world-driver