Skip to content

Commit fffb630

Browse files
committed
Signed-off-by: Constantin M Adam <cmadam@us.ibm.com>
1 parent 528457c commit fffb630

File tree

1 file changed

+18
-10
lines changed

1 file changed

+18
-10
lines changed

transforms/universal/fdedup/kfp_ray/fdedup_wf.py

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -279,10 +279,12 @@ def fuzzydedup(
279279
prefix="scdata",
280280
)
281281
ComponentUtils.add_settings_to_component(execute_signature_calc_job, ONE_WEEK_SEC)
282-
ComponentUtils.set_s3_env_vars_to_component(execute_signature_calc_job, data_s3_access_secret)
283-
ComponentUtils.set_s3_env_vars_to_component(
284-
execute_signature_calc_job, scdata_s3_access_secret, prefix="scdata"
285-
)
282+
# FIXME: see https://github.com/kubeflow/pipelines/issues/10914
283+
if os.getenv("KFPv2", "0") == "1":
284+
ComponentUtils.set_s3_env_vars_to_component(execute_signature_calc_job, data_s3_access_secret)
285+
ComponentUtils.set_s3_env_vars_to_component(
286+
execute_signature_calc_job, scdata_s3_access_secret, prefix="scdata"
287+
)
286288
execute_signature_calc_job.after(compute_signature_calc_exec_params)
287289

288290
# Get the parameters for the cluster analysis job
@@ -311,7 +313,9 @@ def fuzzydedup(
311313
server_url=server_url,
312314
)
313315
ComponentUtils.add_settings_to_component(execute_cluster_analysis_job, ONE_WEEK_SEC)
314-
ComponentUtils.set_s3_env_vars_to_component(execute_cluster_analysis_job, data_s3_access_secret)
316+
# FIXME: see https://github.com/kubeflow/pipelines/issues/10914
317+
if os.getenv("KFPv2", "0") == "1":
318+
ComponentUtils.set_s3_env_vars_to_component(execute_cluster_analysis_job, data_s3_access_secret)
315319
execute_cluster_analysis_job.after(compute_cluster_analysis_exec_params)
316320

317321
compute_get_duplicate_list_exec_params = compute_get_duplicate_list_exec_params_op(
@@ -338,7 +342,9 @@ def fuzzydedup(
338342
server_url=server_url,
339343
)
340344
ComponentUtils.add_settings_to_component(execute_get_duplicate_list_job, ONE_WEEK_SEC)
341-
ComponentUtils.set_s3_env_vars_to_component(execute_get_duplicate_list_job, data_s3_access_secret)
345+
# FIXME: see https://github.com/kubeflow/pipelines/issues/10914
346+
if os.getenv("KFPv2", "0") == "1":
347+
ComponentUtils.set_s3_env_vars_to_component(execute_get_duplicate_list_job, data_s3_access_secret)
342348
execute_get_duplicate_list_job.after(compute_get_duplicate_list_exec_params)
343349

344350
compute_data_cleaning_exec_params = compute_data_cleaning_exec_params_op(
@@ -368,10 +374,12 @@ def fuzzydedup(
368374
prefix="dcdata",
369375
)
370376
ComponentUtils.add_settings_to_component(execute_data_cleaning_job, ONE_WEEK_SEC)
371-
ComponentUtils.set_s3_env_vars_to_component(execute_data_cleaning_job, data_s3_access_secret)
372-
ComponentUtils.set_s3_env_vars_to_component(
373-
execute_data_cleaning_job, dcdata_s3_access_secret, prefix="dcdata"
374-
)
377+
# FIXME: see https://github.com/kubeflow/pipelines/issues/10914
378+
if os.getenv("KFPv2", "0") == "1":
379+
ComponentUtils.set_s3_env_vars_to_component(execute_data_cleaning_job, data_s3_access_secret)
380+
ComponentUtils.set_s3_env_vars_to_component(
381+
execute_data_cleaning_job, dcdata_s3_access_secret, prefix="dcdata"
382+
)
375383
execute_data_cleaning_job.after(compute_data_cleaning_exec_params)
376384

377385

0 commit comments

Comments
 (0)