Skip to content

Commit 293792f

Browse files
rimoliverimolive
committed
Backport fixes in #11075
Introduced back the functions to convert k8s size values to float, but moved to kfp.dsl.utils Signed-off-by: Ricardo M. Oliveira <rmartine@redhat.com>
1 parent 467f30c commit 293792f

File tree

4 files changed

+114
-20
lines changed

4 files changed

+114
-20
lines changed

‎sdk/python/kfp/compiler/compiler_test.py

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3435,31 +3435,31 @@ def simple_pipeline():
34353435
['exec-return-1']['container'])
34363436

34373437
self.assertEqual(
3438-
'5', dict_format['deploymentSpec']['executors']['exec-return-1-2']
3439-
['container']['resources']['resourceCpuLimit'])
3438+
5.0, dict_format['deploymentSpec']['executors']['exec-return-1-2']
3439+
['container']['resources']['cpuLimit'])
34403440
self.assertNotIn(
34413441
'memoryLimit', dict_format['deploymentSpec']['executors']
34423442
['exec-return-1-2']['container']['resources'])
34433443

34443444
self.assertEqual(
3445-
'50G', dict_format['deploymentSpec']['executors']['exec-return-1-3']
3446-
['container']['resources']['resourceMemoryLimit'])
3445+
50.0, dict_format['deploymentSpec']['executors']['exec-return-1-3']
3446+
['container']['resources']['memoryLimit'])
34473447
self.assertNotIn(
34483448
'cpuLimit', dict_format['deploymentSpec']['executors']
34493449
['exec-return-1-3']['container']['resources'])
34503450

34513451
self.assertEqual(
3452-
'2', dict_format['deploymentSpec']['executors']['exec-return-1-4']
3453-
['container']['resources']['resourceCpuRequest'])
3452+
2.0, dict_format['deploymentSpec']['executors']['exec-return-1-4']
3453+
['container']['resources']['cpuRequest'])
34543454
self.assertEqual(
3455-
'5', dict_format['deploymentSpec']['executors']['exec-return-1-4']
3456-
['container']['resources']['resourceCpuLimit'])
3455+
5.0, dict_format['deploymentSpec']['executors']['exec-return-1-4']
3456+
['container']['resources']['cpuLimit'])
34573457
self.assertEqual(
3458-
'4G', dict_format['deploymentSpec']['executors']['exec-return-1-4']
3459-
['container']['resources']['resourceMemoryRequest'])
3458+
4.0, dict_format['deploymentSpec']['executors']['exec-return-1-4']
3459+
['container']['resources']['memoryRequest'])
34603460
self.assertEqual(
3461-
'50G', dict_format['deploymentSpec']['executors']['exec-return-1-4']
3462-
['container']['resources']['resourceMemoryLimit'])
3461+
50.0, dict_format['deploymentSpec']['executors']['exec-return-1-4']
3462+
['container']['resources']['memoryLimit'])
34633463

34643464

34653465
class TestPlatformConfig(unittest.TestCase):

‎sdk/python/kfp/compiler/compiler_utils.py

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@
1515

1616
import collections
1717
import copy
18+
import re
1819
from typing import DefaultDict, Dict, List, Mapping, Set, Tuple, Union
1920

2021
from kfp import dsl
22+
from kfp.dsl import constants
2123
from kfp.dsl import for_loop
2224
from kfp.dsl import pipeline_channel
2325
from kfp.dsl import pipeline_context
@@ -803,3 +805,79 @@ def recursive_replace_placeholders(data: Union[Dict, List], old_value: str,
803805
if isinstance(data, pipeline_channel.PipelineChannel):
804806
data = str(data)
805807
return new_value if data == old_value else data
808+
809+
810+
def validate_cpu_request_limit_to_float(cpu: str) -> float:
811+
"""Validates cpu request/limit string and converts to its numeric float
812+
value.
813+
814+
Args:
815+
cpu: CPU requests or limits. This string should be a number or a
816+
number followed by an "m" to indicate millicores (1/1000). For
817+
more information, see `Specify a CPU Request and a CPU Limit
818+
<https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#specify-a-cpu-request-and-a-cpu-limit>`_.
819+
820+
Raises:
821+
ValueError if the cpu request/limit string value is invalid.
822+
823+
Returns:
824+
The numeric value (float) of the cpu request/limit.
825+
"""
826+
if re.match(r'([0-9]*[.])?[0-9]+m?$', cpu) is None:
827+
raise ValueError(
828+
'Invalid cpu string. Should be float or integer, or integer'
829+
' followed by "m".')
830+
831+
return float(cpu[:-1]) / 1000 if cpu.endswith('m') else float(cpu)
832+
833+
834+
def validate_memory_request_limit_to_float(memory: str) -> float:
835+
"""Validates memory request/limit string and converts to its numeric value.
836+
837+
Args:
838+
memory: Memory requests or limits. This string should be a number or
839+
a number followed by one of "E", "Ei", "P", "Pi", "T", "Ti", "G",
840+
"Gi", "M", "Mi", "K", or "Ki".
841+
842+
Raises:
843+
ValueError if the memory request/limit string value is invalid.
844+
845+
Returns:
846+
The numeric value (float) of the memory request/limit.
847+
"""
848+
if re.match(r'^[0-9]+(E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki){0,1}$',
849+
memory) is None:
850+
raise ValueError(
851+
'Invalid memory string. Should be a number or a number '
852+
'followed by one of "E", "Ei", "P", "Pi", "T", "Ti", "G", '
853+
'"Gi", "M", "Mi", "K", "Ki".')
854+
855+
if memory.endswith('E'):
856+
memory = float(memory[:-1]) * constants._E / constants._G
857+
elif memory.endswith('Ei'):
858+
memory = float(memory[:-2]) * constants._EI / constants._G
859+
elif memory.endswith('P'):
860+
memory = float(memory[:-1]) * constants._P / constants._G
861+
elif memory.endswith('Pi'):
862+
memory = float(memory[:-2]) * constants._PI / constants._G
863+
elif memory.endswith('T'):
864+
memory = float(memory[:-1]) * constants._T / constants._G
865+
elif memory.endswith('Ti'):
866+
memory = float(memory[:-2]) * constants._TI / constants._G
867+
elif memory.endswith('G'):
868+
memory = float(memory[:-1])
869+
elif memory.endswith('Gi'):
870+
memory = float(memory[:-2]) * constants._GI / constants._G
871+
elif memory.endswith('M'):
872+
memory = float(memory[:-1]) * constants._M / constants._G
873+
elif memory.endswith('Mi'):
874+
memory = float(memory[:-2]) * constants._MI / constants._G
875+
elif memory.endswith('K'):
876+
memory = float(memory[:-1]) * constants._K / constants._G
877+
elif memory.endswith('Ki'):
878+
memory = float(memory[:-2]) * constants._KI / constants._G
879+
else:
880+
# By default interpret as a plain integer, in the unit of Bytes.
881+
memory = float(memory) / constants._G
882+
883+
return memory

‎sdk/python/kfp/compiler/pipeline_spec_builder.py

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -648,29 +648,41 @@ def convert_to_placeholder(input_value: str) -> str:
648648
container_spec.resources.resource_cpu_request = (
649649
convert_to_placeholder(
650650
task.container_spec.resources.cpu_request))
651+
container_spec.resources.cpu_request = compiler_utils.validate_cpu_request_limit_to_float(
652+
cpu=convert_to_placeholder(
653+
task.container_spec.resources.cpu_request))
651654
if task.container_spec.resources.cpu_limit is not None:
652655
container_spec.resources.resource_cpu_limit = (
653656
convert_to_placeholder(task.container_spec.resources.cpu_limit))
657+
container_spec.resources.cpu_limit = compiler_utils.validate_cpu_request_limit_to_float(
658+
cpu=convert_to_placeholder(
659+
task.container_spec.resources.cpu_limit))
654660
if task.container_spec.resources.memory_request is not None:
655661
container_spec.resources.resource_memory_request = (
656662
convert_to_placeholder(
657663
task.container_spec.resources.memory_request))
664+
container_spec.resources.memory_request = compiler_utils.validate_memory_request_limit_to_float(
665+
memory=convert_to_placeholder(
666+
task.container_spec.resources.memory_request))
658667
if task.container_spec.resources.memory_limit is not None:
659668
container_spec.resources.resource_memory_limit = (
660669
convert_to_placeholder(
661670
task.container_spec.resources.memory_limit))
671+
container_spec.resources.memory_limit = compiler_utils.validate_memory_request_limit_to_float(
672+
memory=convert_to_placeholder(
673+
task.container_spec.resources.memory_limit))
662674
if task.container_spec.resources.accelerator_count is not None:
663675
container_spec.resources.accelerator.CopyFrom(
664676
pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec
665677
.ResourceSpec.AcceleratorConfig(
666-
resource_type=convert_to_placeholder(
667-
task.container_spec.resources.accelerator_type),
668-
resource_count=convert_to_placeholder(
669-
task.container_spec.resources.accelerator_count),
670678
type=convert_to_placeholder(
671679
task.container_spec.resources.accelerator_type),
672680
count=convert_to_placeholder(
673681
int(task.container_spec.resources.accelerator_count)),
682+
resource_type=convert_to_placeholder(
683+
task.container_spec.resources.accelerator_type),
684+
resource_count=convert_to_placeholder(
685+
task.container_spec.resources.accelerator_count),
674686
))
675687

676688
return container_spec

‎sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,17 @@ deploymentSpec:
6262
resources:
6363
accelerator:
6464
count: '1'
65-
type: tpu-v3
65+
type: 'tpu-v3'
6666
resourceCount: '1'
67-
resourceType: tpu-v3
67+
resourceType: 'tpu-v3'
68+
cpuLimit: 4.0
69+
cpuRequest: 2.0
70+
memoryLimit: 15.032385536
71+
memoryRequest: 4.294967296
6872
resourceCpuLimit: '4'
6973
resourceCpuRequest: '2'
70-
resourceMemoryLimit: 14Gi
71-
resourceMemoryRequest: 4Gi
74+
resourceMemoryLimit: '14Gi'
75+
resourceMemoryRequest: '4Gi'
7276
pipelineInfo:
7377
description: A linear two-step pipeline with resource specification.
7478
name: two-step-pipeline-with-resource-spec

0 commit comments

Comments
 (0)