7
7
import time
8
8
from datetime import date
9
9
10
+
10
11
def download (uf , cache = True , checkmemory = True ):
11
12
"""
12
13
Download ESUS data by UF
@@ -19,7 +20,7 @@ def download(uf, cache=True, checkmemory=True):
19
20
pwd = 'Za4qNXdyQNSa9YaA'
20
21
today = date .today ()
21
22
dt = today .strftime ("_%d_%m_%Y" )
22
- base = f'desc-notificacoes-esusve-{ uf } ' #desc-notificacoes-esusve-
23
+ base = f'desc-notificacoes-esusve-{ uf } ' # desc-notificacoes-esusve-
23
24
url = f'https://{ user } :{ pwd } @elasticsearch-saps.saude.gov.br'
24
25
out = f'ESUS_{ uf } _{ dt } .parquet'
25
26
@@ -33,7 +34,7 @@ def download(uf, cache=True, checkmemory=True):
33
34
fname = fetch (base , uf , url )
34
35
size = os .stat (fname ).st_size
35
36
if size > 50e6 and checkmemory :
36
- print (f"Downloaded data is to large:{ size / 1e6 } MB compressed." )
37
+ print (f"Downloaded data is to large:{ size / 1e6 } MB compressed." )
37
38
print ("Only loading the first 1000 rows. If your computer has enough memory, set 'checkmemory' to False" )
38
39
print (f"The full data is in { fname } " )
39
40
df = pd .read_csv (fname , chunksize = 1000 )
@@ -47,7 +48,6 @@ def download(uf, cache=True, checkmemory=True):
47
48
return df
48
49
49
50
50
-
51
51
def fetch (base , uf , url ):
52
52
UF = uf .upper ()
53
53
print (f"Reading ESUS data for { UF } " )
@@ -61,16 +61,14 @@ def fetch(base, uf, url):
61
61
tempfile = os .path .join (CACHEPATH , f'ESUS_temp_{ UF } .csv.gz' )
62
62
for ch in chunker :
63
63
df = pd .DataFrame .from_dict (ch )
64
- df .sintomas = df ['sintomas' ].str .replace (';' , '' ,) ## remove os ;
64
+ df .sintomas = df ['sintomas' ].str .replace (';' , '' , ) ## remove os ;
65
65
if h :
66
66
df .to_csv (tempfile )
67
67
h = 0
68
68
else :
69
69
df .to_csv (tempfile , mode = 'a' , header = False )
70
70
# df = pd.read_csv('temp.csv.gz')
71
71
72
-
73
-
74
72
return tempfile
75
73
76
74
0 commit comments