forked from CityofSantaMonica/mds-provider-services
-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
201 lines (159 loc) · 5.55 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
"""
Command-line interface implementing various MDS Provider data analytics, including:
- calculate average availability over a period of time
"""
import argparse
from datetime import datetime, timedelta
import dateutil
import mds.providers
from measure import DeviceCounter
import pandas
import query
from statistics import mean
import time
def setup_cli():
"""
Create the cli argument interface, and parses incoming args.
Returns a tuple:
- the argument parser
- the parsed args
"""
parser = argparse.ArgumentParser()
parser.add_argument(
"--availability",
action="store_true",
help="Run the availability calculation."
)
parser.add_argument(
"--debug",
action="store_true",
help="Print debug messages."
)
parser.add_argument(
"--duration",
type=int,
help="Number of seconds; with --start_time or --end_time, defines a time query range."
)
parser.add_argument(
"--end",
type=str,
help="The end of the time query range for this request.\
Should be either int Unix seconds or ISO-8601 datetime format.\
At least one of end or start is required."
)
parser.add_argument(
"--local",
action="store_true",
help="Input and query times are local."
)
parser.add_argument(
"--output",
action="store_true",
help="Write results to csv."
)
parser.add_argument(
"--query",
action="append",
type=lambda kv: kv.split("=", 1),
dest="queries",
help="A series of PROVIDER=VEHICLE pairs; each pair will be analyzed separately."
)
parser.add_argument(
"--start",
type=str,
help="The beginning of the time query range for this request.\
Should be either int Unix seconds or ISO-8601 datetime format\
At least one of end or start is required."
)
return parser, parser.parse_args()
def parse_time_range(args):
"""
Returns a valid range tuple (start, end) given an object with some mix of:
- start
- end
- duration
If both start and end are present, use those. Otherwise, compute from duration.
"""
def _to_datetime(data):
"""
Helper to parse different textual representations into datetime
"""
try:
return datetime.fromtimestamp(int(data), timezone.utc)
except:
return dateutil.parser.parse(data)
if args.start is not None and args.end is not None:
return _to_datetime(args.start), _to_datetime(args.end)
duration = int(args.duration)
if args.start is not None:
start = _to_datetime(args.start)
return start, start + timedelta(seconds=duration)
if args.end is not None:
end = _to_datetime(args.end)
return end - timedelta(seconds=duration), end
def log(args, msg):
"""
Prints the message if debugging is turned on.
"""
def __now():
return datetime.utcnow().isoformat()
if args.debug:
print(f"[{__now()}] {msg}")
def availability(provider_name, vehicle_type, args):
"""
Runs the availability calculation
"""
oneday = timedelta(days=1)
log(args, f"""Starting availability calculation:
- time range: {args.start} to {args.end}
- provider: {provider_name}
- vehicle type: {vehicle_type}""")
devices = DeviceCounter(args.start, args.end, local=args.local, debug=args.debug)
q = query.Availability(args.start, args.end,
vehicle_types=vehicle_type,
table="csm_availability",
local=args.local, debug=True)
results = {}
log(args, f"Starting calculation for {provider_name}")
data = q.get(provider_name=provider_name)
partition = devices.count(data).partition()
log(args, partition.describe())
overall_avg = devices.average()
log(args, f"Overall average: {overall_avg}")
counts = {}
start = args.start
while start < args.end:
end = start + oneday
log(args, f"Counting {start.strftime('%Y-%m-%d')} to {end.strftime('%Y-%m-%d')}")
_q = query.Availability(start, end,
vehicle_types=vehicle_type,
table="csm_availability",
local=args.local, debug=args.debug)
_data = _q.get(provider_name=provider_name)
log(args, f"{len(_data)} availability records in time period")
_devices = DeviceCounter(start, end, local=args.local, debug=args.debug)
counts[start] = _devices.count(_data)
start = end
if args.debug:
for date, count in counts.items():
print(f"{provider_name},{vehicle_type},{date.strftime('%Y-%m-%d')},{count.average()},{overall_avg}")
return overall_avg, counts
if __name__ == "__main__":
arg_parser, args = setup_cli()
# assert the time range parameters and parse a valid range
if args.start is None and args.end is None:
arg_parser.print_help()
exit(1)
if (args.start is None or args.end is None) and args.duration is None:
arg_parser.print_help()
exit(1)
args.start, args.end = parse_time_range(args)
if args.availability:
for provider_name, vehicle_type in dict(args.queries).items():
overall_avg, counts = availability(provider_name, vehicle_type, args)
if args.output:
for date, count in counts.items():
print(f"{provider_name},{vehicle_type},{date.strftime('%Y-%m-%d')},{count.average()},{overall_avg}")
else:
arg_parser.print_help()
exit(0)