-
Notifications
You must be signed in to change notification settings - Fork 4
/
workflow.py
34 lines (30 loc) · 1.78 KB
/
workflow.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
from kfp import dsl
from mlrun.platforms import auto_mount
import os
import sys
import mlrun
#each pipe creates 20 jobs that runs one by one change the pipe number to select how much pipelines you want to run in parllel
#for example if you set pipe_num=3, it will run 3 pipelines in parller and each one of them will run 20 jobs and in total it will run 3*20=60 jobs
pipe_num = 5
def kfpipeline():
with dsl.ParallelFor([i for i in range(pipe_num)]) as item:
step_1 = mlrun.run_function('normal-job-sec')
step_2 = mlrun.run_function('artifact-job').after(step_1)
step_3 = mlrun.run_function('normal-job-sec').after(step_2)
step_4 = mlrun.run_function('normal-job-sec').after(step_3)
step_5 = mlrun.run_function('normal-job-sec').after(step_4)
step_6 = mlrun.run_function('normal-job-sec').after(step_5)
step_7 = mlrun.run_function('artifact-job').after(step_6)
step_8 = mlrun.run_function('normal-job-sec').after(step_7)
step_9 = mlrun.run_function('artifact-job').after(step_8)
step_10 = mlrun.run_function('normal-job-sec').after(step_9)
step_11 = mlrun.run_function('normal-job-sec').after(step_10)
step_12 = mlrun.run_function('artifact-job').after(step_11)
step_13 = mlrun.run_function('normal-job-sec').after(step_12)
step_14 = mlrun.run_function('normal-job-sec').after(step_13)
step_15 = mlrun.run_function('normal-job-sec').after(step_14)
step_16 = mlrun.run_function('artifact-job').after(step_15)
step_17 = mlrun.run_function('normal-job-sec').after(step_16)
step_18 = mlrun.run_function('normal-job-sec').after(step_17)
step_19 = mlrun.run_function('artifact-job').after(step_18)
step_20 = mlrun.run_function('normal-job-sec').after(step_19)