-
Notifications
You must be signed in to change notification settings - Fork 1
/
job_creation.py
151 lines (132 loc) · 4.57 KB
/
job_creation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
#!/usr/bin/python3
import pprint
from pprint import pprint as P
from job import *
import logging
logger = logging.getLogger('remote-jobs')
def map_name(from_, to_, *dicts_):
""" maps from name to name in a list of dictionaries """
for d in dicts_:
if from_ in d:
d[to_]=d[from_]
def get_job_info(default_dict, user_data_dict):
"""Merges info from defaults and user entry"""
defaults={
'type' : None,
'rsync_flag' : ['-avh'],
'files' : None,
'collections' : None,
'luser' : None,
'ruser' : None,
'lhome' : '/home/{user}',
'rhome' : '/home/{user}',
'lhost' : None,
'rhost' : None,
'local' : False,
'glob' : False
}
result=dict()
my_map_name=lambda from_, to_: map_name(from_, to_, default_dict, user_data_dict)
my_map_name('home','lhome')
my_map_name('host','rhost')
#merge
for item in ['type', 'home', 'rsync_flags', 'local', 'glob',
'lhost', 'rhost', 'luser', 'ruser', 'lhome', 'rhome']:
if item in default_dict:
result[item]=default_dict[item]
if item in user_data_dict:
result[item]=user_data_dict[item]
if item not in result:
result[item]=defaults[item]
for item in ['files', 'collections']:
if item in default_dict:
result[item]=default_dict[item]
if item in user_data_dict:
if item not in result:
result[item]=user_data_dict[item]
else:
result[item]+=user_data_dict[item]
if item not in result:
result[item]=defaults[item]
#Translate Strings
for key, val in result.items():
if val in ['true', 'True', '1']:
result[key] = True
if val in ['false', 'False', '0']:
result[key] = False
return result
def build_rsync_jobs(host, user, job_dict):
"""Creates job instances"""
logger.debug("job_dict:\n" + pprint.pformat(job_dict))
jobs=[]
for file_pair in job_dict['files']:
if job_dict['luser']:
luser = job_dict['luser']
else:
luser = user
jobs.append(
RemoteJobRsync(
luser = luser, ruser = user,
lhost = job_dict['lhost'], rhost = host,
lhome = job_dict['lhome'], rhome = job_dict['rhome'],
local = job_dict['local'],
src = file_pair[0], dest = file_pair[1],
glob = job_dict['glob'], direction = "local2remote",
flags = job_dict['rsync_flags']
)
)
return jobs
def build_jobs(host_list, user_list, job_dict, collections):
jobs=[]
if 'type' not in job_dict:
print("no operation type give!!")
return
## expand collections
files=[]
if job_dict['files']:
files = job_dict['files']
if job_dict['collections']:
for collection in job_dict['collections']:
files+=collections[collection]
if len(files) > 0:
job_dict['files'] = files
## select creation function depending on type
fun=None
if job_dict['type'] == 'rsync':
fun=build_rsync_jobs
else:
logger.error("no valid operation type")
## run creation function for ever host user pair
for host in host_list:
for user in user_list:
rv = fun(host, user, job_dict)
jobs += rv
return jobs
def create_jobs(dataMap):
"""Create Job list form yaml map"""
## get collections
collections=None
if 'collections' in dataMap:
collections=dataMap['collections']
else:
logger.info("no collections defined")
jobs=[]
if 'hosts' not in dataMap:
logger.error("no hosts defined!")
else:
for hosts_key, hosts_val in dataMap['hosts'].items():
host_list=hosts_key.split(',')
default=None
if 'default' in hosts_val:
default=hosts_val['default']
else:
logger.info("defaults missing")
if 'users' in hosts_val:
for users_key,users_val in hosts_val['users'].items():
user_list=users_key.split(',')
job_dict = get_job_info(default, users_val)
job_list=build_jobs(host_list, user_list, job_dict, collections)
jobs += job_list
else:
logger.error("no users given")
return jobs