Skip to content

Commit 282a9e0

Browse files
Louis-HeLouis-He
committed
improve the code and the whole system
1 parent 8ada312 commit 282a9e0

File tree

5 files changed

+310
-85
lines changed

5 files changed

+310
-85
lines changed

‎main.py

+253-33
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,74 @@
11
from datetime import datetime
22
import time
33
import urllib.request
4-
from urllib.request import urlretrieve,urlopen
4+
from urllib.request import urlretrieve
5+
from apscheduler.schedulers.blocking import BlockingScheduler
6+
#from apscheduler.schedulers.background import BackgroundScheduler
57
import os
8+
#import progressbar
69

710
utc = 0
811

12+
#decide the target init forecast time
13+
def targetinittime():
14+
global downloadhour
15+
timechange = -1 * utc
16+
nowtime = time.time() + timechange * 60 * 60 # struct time of UTC
17+
result = time.strftime("%H", time.localtime(nowtime)) # hour of UTC time
18+
if int(result) >= 21:
19+
hourresult = '18'
20+
elif int(result) < 9:
21+
hourresult = '00'
22+
elif int(result) < 15:
23+
hourresult = '06'
24+
elif int(result) < 21:
25+
hourresult = '12'
26+
27+
returnlist = []
28+
isdownload = []
29+
for i in range(0, len(downloadhour)):
30+
returnlist.append(hourresult)
31+
isdownload.append(False)
32+
33+
return returnlist, isdownload
34+
35+
# return the URL of the latest GFS model (3 hours after the initial)
36+
# return [0]:the URL of target file; [1]:the filename
37+
def decideURL2(forecasthour, isexist):
38+
global utc
39+
hourresult = ''
40+
41+
timechange = -1 * utc
42+
nowtime = time.time() + timechange * 60 * 60 # struct time of UTC
43+
result = time.strftime("%H", time.localtime(nowtime)) # hour of UTC time
44+
#print(result)
45+
46+
if int(result) >= 18:
47+
hourresult = '18'
48+
elif int(result) < 6:
49+
hourresult = '00'
50+
elif int(result) < 12:
51+
hourresult = '06'
52+
elif int(result) < 18:
53+
hourresult = '12'
54+
'''
55+
elif int(result) < 4:
56+
hourresult = '18'
57+
nowtime = nowtime - 24 * 60 * 60
58+
'''
59+
assumetime = time.strftime("%Y%m%d", time.localtime(nowtime)) + hourresult + '0000' # hour of UTC time
60+
assumetimestamp = time.mktime(time.strptime(assumetime, "%Y%m%d%H%M%S"))
61+
if not isexist:
62+
assumetimestamp = assumetimestamp - 6 * 60 * 60
63+
64+
#http://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p25.pl?file=gfs.t12z.pgrb2.0p25.f000&all_var=on&leftlon=0&rightlon=360&toplat=90&bottomlat=-90&dir=%2Fgfs.2017102712
65+
66+
result = time.strftime("%Y%m%d%H", time.localtime(assumetimestamp)) # final hour of UTC time
67+
filename = 'GFS' + result + '.f' + forecasthour
68+
#print(filename)
69+
result = 'http://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p25.pl?file=gfs.t'+ hourresult +'z.pgrb2.0p25.f' + forecasthour + '&all_var=on&leftlon=0&rightlon=360&toplat=90&bottomlat=-90&dir=%2Fgfs.' + result
70+
return [result, 'gfs.' + filename]
71+
972
# return the URL of the latest GFS model (4 hours after the initial)
1073
def decideURL(forecasthour):
1174
global utc
@@ -14,7 +77,7 @@ def decideURL(forecasthour):
1477
timechange = -1 * utc
1578
nowtime = time.time() + timechange * 60 * 60 # struct time of UTC
1679
result = time.strftime("%H", time.localtime(nowtime)) # hour of UTC time
17-
print(result)
80+
#print(result)
1881

1982
if int(result) >= 22:
2083
hourresult = '18'
@@ -32,11 +95,11 @@ def decideURL(forecasthour):
3295

3396
result = time.strftime("%Y%m%d", time.localtime(nowtime)) + hourresult # hour of UTC time
3497
filename = 'GFS' + result + '.f' + forecasthour
35-
print(filename)
98+
#print(filename)
3699
result = 'http://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p25.pl?file=gfs.t'+ hourresult +'z.pgrb2.0p25.f' + forecasthour + '&all_var=on&leftlon=0&rightlon=360&toplat=90&bottomlat=-90&dir=%2Fgfs.' + result
37100
return [result, 'gfs.' + filename]
38101

39-
#定义下载函数downLoadPicFromURL（本地文件夹，网页URL）
102+
#downLoadPicFromURL(local path，source URL)
40103
def downLoadPicFromURL(dest_dir,URL):
41104
try:
42105
urllib.request.urlopen(URL , dest_dir)
@@ -50,45 +113,202 @@ def initialize():
50113
os.system('mkdir rawfile')
51114
print('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + 'Create rawfile folder')
52115
print('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + 'Start downloading file...')
53-
f = open('/root/GFS/sysreport/downloadreport.txt', 'w+')
116+
f = open('sysreport/downloadreport.txt', 'w+')
54117
f.close()
55118
print('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + 'Create system download report file...')
119+
f = open('sysreport/sysrealreport.txt', 'w+')
120+
f.close()
121+
print('[' + time.strftime('%Y-%m-%d %H:%M:%S',
122+
time.localtime(time.time() + utc * 60 * 60)) + ']' + 'Create system realtime file...')
123+
f = open('sysreport/ongingmission.sh', 'w+')
124+
f.close()
125+
print('[' + time.strftime('%Y-%m-%d %H:%M:%S',
126+
time.localtime(time.time() + utc * 60 * 60)) + ']' + 'Create ongingmission script...')
127+
f = open('sysreport/waitlistmission.sh', 'w+')
128+
f.close()
129+
print('[' + time.strftime('%Y-%m-%d %H:%M:%S',
130+
time.localtime(time.time() + utc * 60 * 60)) + ']' + 'Create waitlistmission script...')
56131

57-
58-
#initialize the program
59-
initialize()
60-
61-
62-
#set the forecast hour of the file from GFS
63-
downloadhour = ['000','006', '012', '018', '024','030', '036', '042','048', '054', '060', '066','072', '078', '084',
64-
'090','096','102', '108', '114', '120', '126', '132', '138', '144', '150', '156', '162', '168', '174',
65-
'180', '186', '192', '198', '204', '210', '216', '222', '228', '234', '240']
66-
67-
#reqiured download file
68-
total = len(downloadhour)
69-
count = 0
70-
for i in downloadhour:
71-
count += 1
72-
downloadinfo = decideURL(i)
73-
print('' + str((count-1) / total * 100) + '%[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' + downloadinfo[0])
132+
def downloadfile(forecasthour,isexist):
133+
#global bar
134+
downloadinfo = decideURL2(forecasthour,isexist)
135+
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' +downloadinfo[0])
74136
path = 'rawfile/' + downloadinfo[1]
75137
try:
76138
urlretrieve(downloadinfo[0], path)
139+
print('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' + downloadinfo[1] + 'SUCCESS')
77140
f = open('/root/GFS/sysreport/downloadreport.txt', 'a+')
78-
f.write('['+time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60))+']' + '\t' + downloadinfo[1] + ' DOWNLOAD SUCCESS\n')
79-
f.close()
80-
f = open('/root/GFS/sysreport/running.txt', 'a+')
81141
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
82142
downloadinfo[1] + ' DOWNLOAD SUCCESS\n')
83143
f.close()
84-
except:
85-
print('(UNEXPECTED ERR)[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(
86-
time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' + downloadinfo[1] + 'ERROR')
87-
f = open('/root/GFS/sysreport/downloadreport.txt', 'a+')
88-
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
89-
downloadinfo[1] + ' DOWNLOAD FAILED\n')
90-
f.close()
91144
f = open('/root/GFS/sysreport/running.txt', 'a+')
92145
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
93-
downloadinfo[1] + ' DOWNLOAD FAILED\n')
146+
downloadinfo[1] + ' DOWNLOAD SUCCESS\n')
94147
f.close()
148+
addmission(downloadinfo[1])
149+
return True
150+
except:
151+
if isexist:
152+
print('(FILE DO NOT EXIST)[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' + downloadinfo[1] + 'ERROR')
153+
else:
154+
print('(UNEXPECTED ERR)[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' + downloadinfo[1] + 'ERROR')
155+
f = open('/root/GFS/sysreport/downloadreport.txt', 'a+')
156+
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
157+
downloadinfo[1] + ' DOWNLOAD FAILED\n')
158+
f.close()
159+
f = open('/root/GFS/sysreport/running.txt', 'a+')
160+
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
161+
downloadinfo[1] + ' DOWNLOAD FAILED\n')
162+
f.close()
163+
return False
164+
165+
def istruelist(list):
166+
result = True
167+
for i in list:
168+
if not i:
169+
result = False
170+
171+
return result
172+
173+
# main method
174+
def mainmethod():
175+
#global bar
176+
global successdownload
177+
178+
# initialize progressbar
179+
'''
180+
bar = progressbar.ProgressBar(widgets=[
181+
' [', progressbar.Timer(), '] ',
182+
progressbar.Bar(),
183+
], redirect_stdout=True)
184+
'''
185+
186+
global downloadhour
187+
inittime = time.time() + (-1) * utc * 60 * 60
188+
189+
#initialize the program
190+
initialize()
191+
192+
#set the forecast hour of the file from GFS
193+
downloadhour = ['000','006', '012', '018', '024','030', '036', '042','048', '054', '060', '066','072', '078', '084',
194+
'090','096','102', '108', '114', '120', '126', '132', '138', '144', '150', '156', '162', '168', '174',
195+
'180', '186', '192', '198', '204', '210', '216', '222', '228', '234', '240']
196+
successdownload = 0
197+
# set progressbar
198+
#bar.max_value = len(downloadhour)
199+
#bar.min_value = 0
200+
#bar.update(0)
201+
#scheduler = BackgroundScheduler()
202+
#scheduler.add_job(updateprogessbar, 'interval', seconds = 1) # 24hr
203+
#scheduler.start()
204+
205+
initialprocess = targetinittime()
206+
isdownload = initialprocess[1] # initial all false, true: isdownload; false: not download yet
207+
208+
'''
209+
#reqiured download file
210+
total = len(downloadhour)
211+
count = 0
212+
for i in downloadhour:
213+
count += 1
214+
downloadinfo = decideURL(i)
215+
print('' + str((count-1) / total * 100) + '%[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' + downloadinfo[0])
216+
path = 'rawfile/' + downloadinfo[1]
217+
try:
218+
urlretrieve(downloadinfo[0], path)
219+
f = open('/root/GFS/sysreport/downloadreport.txt', 'a+')
220+
f.write('['+time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60))+']' + '\t' + downloadinfo[1] + ' DOWNLOAD SUCCESS\n')
221+
f.close()
222+
f = open('/root/GFS/sysreport/running.txt', 'a+')
223+
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
224+
downloadinfo[1] + ' DOWNLOAD SUCCESS\n')
225+
f.close()
226+
except:
227+
print('(UNEXPECTED ERR)[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(
228+
time.time() + utc * 60 * 60)) + ']Dowanloading file... from URL: ' + downloadinfo[1] + 'ERROR')
229+
f = open('/root/GFS/sysreport/downloadreport.txt', 'a+')
230+
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
231+
downloadinfo[1] + ' DOWNLOAD FAILED\n')
232+
f.close()
233+
f = open('/root/GFS/sysreport/running.txt', 'a+')
234+
f.write('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time() + utc * 60 * 60)) + ']' + '\t' +
235+
downloadinfo[1] + ' DOWNLOAD FAILED\n')
236+
f.close()
237+
'''
238+
239+
while not istruelist(isdownload):
240+
#reqiured download file
241+
print('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(
242+
time.time() + utc * 60 * 60)) + ']Start downloading cycle.')
243+
count = 0
244+
tmpbool = True
245+
for i in downloadhour:
246+
if not isdownload[count] and tmpbool:
247+
tmpbool = downloadfile(downloadhour[count], True)
248+
if tmpbool:
249+
isdownload[count] = True
250+
successdownload += 1
251+
count += 1
252+
# print(isdownload)
253+
print('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(
254+
time.time() + utc * 60 * 60)) + ']No more new file. Start sleeping cycle...[5 min]')
255+
time.sleep(120) # sleep 2 mins for another try
256+
257+
finishtime = time.time() + (-1) * utc * 60 * 60
258+
259+
print('*--------------------------------------------*')
260+
print('[' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(
261+
time.time() + utc * 60 * 60)) + ']Download cycle ends.')
262+
print('TOTAL RUNNING TIME:' + str(finishtime-inittime) + 's')
263+
264+
# decide First start time
265+
def startmain():
266+
nowtime = time.time()
267+
result = int(time.strftime("%H%M", time.localtime(nowtime))) # hour of UTC time
268+
print('nowtime: (HHMM)' + str(result))
269+
'''
270+
forecast1 = 325
271+
forecast2 = 925
272+
forecast3 = 1525
273+
forecast4 = 2125
274+
'''
275+
276+
if result > 325 and result < 330:
277+
return True
278+
elif result > 925 and result < 930:
279+
return True
280+
elif result > 1525 and result < 1530:
281+
return True
282+
elif result > 2125 and result < 2130:
283+
return True
284+
return False
285+
286+
# add onging mission list
287+
def addmission(filename):
288+
path = 'rawfile/'
289+
files = os.listdir(path)
290+
for i in files:
291+
if filename[-4:] == i[-4:]:
292+
os.system('rm product/WTP/' + i)
293+
os.system('rm product/WGP/' + i)
294+
os.system('rm product/WTPrain/' + i)
295+
print('delete same product in the previous hour')
296+
f = open('sysreport/waitlistmission.sh', 'a+')
297+
f.write('python3 plotWGP.py --path ' + filename + ' --area CN')
298+
f.write('python3 plotWTP.py --path ' + filename + ' --area CN')
299+
f.write('python3 plotrain.py --path ' + filename + ' --area CN')
300+
f.close()
301+
302+
print('[Please Wait]System First Start: Wait for next closest GFS files download time window...')
303+
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
304+
while not startmain():
305+
time.sleep(60)
306+
# main operation
307+
scheduler = BlockingScheduler()
308+
scheduler.add_job(mainmethod, 'interval', seconds = 6 * 60 * 60)
309+
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
310+
mainmethod()
311+
try:
312+
scheduler.start()
313+
except (KeyboardInterrupt, SystemExit):
314+
pass

0 commit comments

Comments
 (0)