Skip to content

Commit

Permalink
Merge pull request #7 from NSSAC/socrata_api-changes
Browse files Browse the repository at this point in the history 
Socrata api changes
  • Loading branch information
@bdklahn
bdklahn authored Oct 6, 2023
2 parents 0c04641 + 1566d52 commit 4b6b944
Show file tree
Hide file tree
Showing 2 changed files with 132 additions and 18 deletions.
108 changes: 98 additions & 10 deletions webdesign/polls/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
from django.http import HttpResponse
from django.core.files.storage import FileSystemStorage
from django.contrib import messages
from django.forms import URLField
from django.core.exceptions import ValidationError

import os
import pandas as pd
from os.path import exists
Expand All @@ -26,6 +29,8 @@
csvtime=''
context={}
csvtype='Singletime'
datamindate=''
datamaxdate=''

media_path = Path(__file__).resolve().parent.parent / "media"
# make sure output paths are created
Expand All @@ -41,14 +46,45 @@
zip_path = media_path / "zip"
zip_path.mkdir(exist_ok=True, parents=True)


def index(request):
cat_ts = pd.Series
trend_ts = pd.Series
bin_bounds = np.ndarray

global file_path, input_ts, pp_ts, csvname, freq, csvtime, context, csvtype, media_path, analytical_summary_path, categorize_output_path, zip_path
global file_path, input_ts, pp_ts, csvname, freq, csvtime, context, csvtype, media_path, analytical_summary_path, categorize_output_path, zip_path, datamindate, datamaxdate
context={}

if request.method == 'POST' and 'uploadapibutton' in request.POST:
dataurl = request.POST.get('apiurltext')
if validate_url(dataurl):
if 'W' in request.POST.getlist('csvfrequency'):
freq='W'
else:
freq='D'
if 'Multitime' in request.POST.getlist('uploadtype'):
csvtype='Multitime'
else:
csvtype='Singletime'

readapi(dataurl)

csvtime=datetime.utcnow()
csvname='API URL'
datamindate=input_ts.index.min()
datamaxdate=input_ts.index.max()
context= {'csvname': csvname}
context.update({'csvtype': csvtype} )
context.update({'csvtime': csvtime} )
context.update( {'csvdata':input_ts.reset_index().to_dict('records')})
context.update({'datamindate': datamindate} )
context.update({'datamaxdate': datamaxdate} )
headers=[]
for col in input_ts.reset_index().columns:
headers.append({'column': col})
context.update( {'csvdataheaders':headers})
else:
messages.warning(request, 'Please enter a valid url')

if request.method == 'POST' and 'uploadbutton' in request.POST:
uploaded_file = request.FILES['document']
Expand All @@ -60,7 +96,7 @@ def index(request):
messages.info(request, 'File upload Success!')
loc = csvname.rfind("_")
csvname = csvname[0:len(csvname)-12]+'.csv'
csvtime = os.path.getmtime(file_path)
csvtime = datetime.utcfromtimestamp(os.path.getmtime(file_path))
if 'W' in request.POST.getlist('csvfrequency'):
freq='W'
else:
Expand All @@ -73,9 +109,17 @@ def index(request):
readfile(file_path)
context= {'csvname': csvname}
context.update({'csvtype': csvtype} )
context.update({'csvtime': datetime.utcfromtimestamp(csvtime)} )
context.update({'csvtime': csvtime} )
context.update( {'csvdata':input_ts.reset_index().to_dict('records')})
if 'Multitime' in csvtype:
datamindate=input_ts.index.min()
datamaxdate=input_ts.index.max()
elif 'Singletime' in csvtype:
datamindate=input_ts['date'].min()
datamaxdate=input_ts['date'].max()

context.update({'datamindate': datamindate} )
context.update({'datamaxdate': datamaxdate} )
headers=[]
for col in input_ts.reset_index().columns:
headers.append({'column': col})
Expand All @@ -91,17 +135,20 @@ def index(request):
messages.warning(request, 'Upload failed. Please try again!')

if request.method == 'POST' and 'runbutton' in request.POST:
if exists(file_path):
if not input_ts.empty:
formdata = {}
# formdata['csvfrequency'] = freq

csvtime= os.path.getmtime(file_path)
methods=request.POST.getlist('datapreprocess')
smoothing_window = request.POST.get('smoothingwindow')
cat_method=request.POST.getlist('categorizetype')
num_bins = request.POST.get('binsize')
win_size = request.POST.get('trendsize')
fill_method=request.POST.getlist('fillmethod')
datamindate=request.POST.getlist('min_date')
datamaxdate=request.POST.getlist('max_date')

datamindate = datetime.strptime(datamindate[0], '%Y-%m-%d')
datamaxdate = datetime.strptime(datamaxdate[0], '%Y-%m-%d')

if(not smoothing_window):
smoothing_window = 7
Expand Down Expand Up @@ -131,7 +178,7 @@ def index(request):


context= {'csvname': csvname}
context.update({'csvtime': datetime.utcfromtimestamp(csvtime)} )
context.update({'csvtime': csvtime} )

formdata['smoothingwindow'] = smoothing_window
formdata['binsize'] = num_bins
Expand All @@ -142,6 +189,7 @@ def index(request):
headers=[]
if(csvtype=='Singletime'):
workflow_type='single'
input_ts = input_ts.loc[(input_ts['date'] >= datamindate ) & (input_ts['date'] <= datamaxdate)]
input_df = input_ts
name, cat_ts, df ,formdata = single_ts_workflow(input_df,request, fill_method, formdata, methods, freq, cat_method, workflow_type, smoothing_window, win_size, num_bins, custom_range)

Expand Down Expand Up @@ -172,6 +220,11 @@ def index(request):
elif(csvtype=='Multitime'):
workflow_type='multi-signal'
cat_df = pd.DataFrame()
if input_ts[datamindate:datamaxdate].empty:
input_ts=input_ts[datamaxdate:datamindate]
else:
input_ts=input_ts[datamindate:datamaxdate]

for column in input_ts.columns:
signal_ts = input_ts[[column]]
signal_ts.columns = ['value']
Expand Down Expand Up @@ -214,14 +267,41 @@ def index(request):
fig = px.line(input_ts.reset_index(), x='date', y=input_ts.columns.to_list())
graph_plotly = plot(fig, output_type="div")
# context.update( {'graph_plotly':graph_plotly})


context.update({'datamindate': datamindate} )
context.update({'datamaxdate': datamaxdate} )
context.update( {'formdata': json.dumps(formdata)} )

else:
messages.warning(request, 'Please upload a .csv file first!')
messages.warning(request, 'No data in the memory')
context.update( {'currentpath':'index'})
return render(request, 'pages/index.html',context)

def readapi(dataurl):
global input_ts
dateflag=False
mainurl=dataurl[0:dataurl.index('.json')+5]
suffix="?$select=count(*)"
maxlimit=pd.read_json(mainurl+suffix)['count'][0]

if('$query' in dataurl):
suffix="%20LIMIT%20"+str(maxlimit)
else:
suffix="?$limit="+str(maxlimit)+"&$offset=0"
dataurl=dataurl+suffix

results_df = pd.read_json(dataurl)
for col in results_df.columns:
if "date" in col.lower() and not dateflag:
results_df[col] = pd.to_datetime(results_df[col])
results_df = results_df.rename(columns={col: 'date'})
dateflag=True
elif not results_df[col].dtype==np.int64 or not results_df[col].dtype==np.float64 and not dateflag:
results_df = results_df.drop(col, axis=1)

results_df=results_df.drop_duplicates(subset=['date'])
results_df=results_df.set_index('date')
input_ts = results_df

def readfile(filename):
global input_ts
Expand Down Expand Up @@ -300,4 +380,12 @@ def zipfiles(filenames, tag, path):
fpath=path+i['name']
z.write(fpath)
z.close()
return name
return name

def validate_url(url):
url_form_field = URLField()
try:
url = url_form_field.clean(url)
except ValidationError:
return False
return True
42 changes: 34 additions & 8 deletions webdesign/templates/includes/sidebar.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,24 +59,37 @@ <h4>PEpiTA</h4>
</li>
</ul>
</ul>
<div style="background-color: #ededed; border: 0.5px solid gray;">
<label class="text-black up bold mb-0 p-0 w-100">Upload csv file:</label>
<div class="row content">
<div class="col">
<ul>
<li style="list-style-type: none;margin-left: -30px;">
<input type="file" class="btn btn-sm shadow-sm bg-white p-0 m-50" name="document"
style="max-width:200px" id="document" required="required">
</div>
<div class="col">
style="max-width:195px" id="document" >
</li>
<li style="list-style-type: none;">
<button id="submit" name="uploadbutton" class="shadow-sm bg-white rounded ">Click to upload</a>
</div>
</div>
</li>
</ul>
<div style="background-color:gray;height:1px;max-width:200px"></div>
<label class="text-black up bold mb-0 p-0 w-100">Or API URL:</label>
<ul>
<li style="list-style-type: none;margin-left: -30px;">
<input type="text" id="apiurltext" name="apiurltext" style="height:28px;width:98%">
</li>
<li style="list-style-type: none;">
<button id="submit" name="uploadapibutton" style="margin-top: 15px;" class="shadow-sm bg-white rounded ">Get Data</a>
</li>
</ul>
</div>

</form>
</div>
</div>
</div>
{% if csvname %}
<div>
<code>File in memory: {{csvname}}</code>
<code>Last Updated: {{csvtime}}</code>
<code>Last Updated(UTC): {{csvtime}}</code>
</div>
{% elif messages %}
<div class="container" style="color: firebrick; margin-top: 20px">
Expand All @@ -91,6 +104,19 @@ <h4>PEpiTA</h4>
<form action="" method="POST" enctype="multipart/form-data">
{% csrf_token %}
<div class="card bg-light" id="sidenavCard2">
<div class="card-body text-start p-0 w-100">
<label>Date Filter:</label>
<ul>
<li style="list-style-type: none;"> <label for="min_date">Min Date:</label>
<input type="date" id="min_date" name="min_date" value="{{datamindate|date:'Y-m-d'}}" style="font-size:13.5px;width:55%;height:20px">
</li>
<li style="list-style-type: none;">
<label for="max_date">Max Date:</label>
<input type="date" id="max_date" name="max_date" value="{{datamaxdate|date:'Y-m-d'}}" style="font-size:13.5px;width:55%;height:20px">

</li>
</ul>
</div>
<div class="card-body text-start p-0 w-100">
<label>Data Preprocess methods:</label>
<div class="hover-text"><i class="fa fa-info-circle text-secondary" aria-hidden="true"></i>
Expand Down

0 comments on commit 4b6b944

Please sign in to comment.