forked from detorto/tulen
-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.py
executable file
·97 lines (78 loc) · 2.08 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
import json
import time
import multiprocessing
import sys, traceback
def load_json(filename):
try:
data = json.load(open(filename))
except:
return None
return data
def pretty_dump(data):
return json.dumps(data, indent=4, separators=(',', ': '),ensure_ascii=False).encode('utf8')
import time
import threading
from functools import wraps
import sys
import os
import multiprocessing
from multiprocessing.pool import ThreadPool
threads = ThreadPool(16)
import time
import threading
from vk.exceptions import VkAPIError
import psutil
#@RateLimited(2) # 2 per second at most
import time
import threading
from functools import wraps
lock = multiprocessing.Lock()
q= multiprocessing.Queue()
def RateLimited(maxPerSecond):
minInterval = 1.0 / float(maxPerSecond)
def decorate(func):
lastTimeCalled = [0.0]
def rateLimitedFunction(*args,**kargs):
elapsed = time.clock() - lastTimeCalled[0]
leftToWait = minInterval - elapsed
if leftToWait>0:
time.sleep(leftToWait)
ret = func(*args,**kargs)
lastTimeCalled[0] = time.clock()
return ret
return rateLimitedFunction
return decorate
@RateLimited(1)
def set_event():
try:
lock.release()
except ValueError:
pass
def time_manager():
set_event()
while True:
q.get()
set_event();
p = multiprocessing.Process(target=time_manager)
p.start()
cerror = 0
def rated_operation(operation, args):
global cerror
lock.acquire()
q.put("Can unlock")
ret = None
while True:
try:
process = psutil.Process(os.getpid())
print time.time(), operation._method_name, process.memory_info().rss, process.memory_info().rss/1024.0/1024.0, cerror
return operation(**args)
except VkAPIError as e:
if e.code == 6:
print "------- To many req for sec, throttling", e
time.sleep(5)
cerror+=1
continue
else:
raise
except:
raise