Skip to content

Commit d265ddd

Browse files
mantvydasdeltuvamantvydasdeltuva
committed
MDE/PKFE-16 back-end implementation
1 parent bfd8981 commit d265ddd

File tree

6 files changed

+3395
-1
lines changed

6 files changed

+3395
-1
lines changed

‎app/back-end/src/constants.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
WORKSPACE_CREATE_ROUTE = "/workspace/create"
3333
WORKSPACE_RENAME_ROUTE = "/workspace/rename"
3434
WORKSPACE_DELETE_ROUTE = "/workspace/delete"
35+
WORKSPACE_AGGREGATE_ROUTE = "/workspace/aggregate"
3536

3637
# Events
3738
CONSOLE_FEEDBACK_EVENT = "console_feedback"

‎app/back-end/src/routes/workspace_aggregate_route.py

Lines changed: 359 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,359 @@
1+
# pylint: disable=import-error
2+
3+
import os
4+
import csv
5+
from flask import Blueprint, request, jsonify
6+
from ast import literal_eval
7+
8+
from src.setup.extensions import logger
9+
from src.utils.helpers import socketio_emit_to_user_session, is_number
10+
from src.utils.exceptions import UnexpectedError
11+
from src.constants import WORKSPACE_AGGREGATE_ROUTE, CONSOLE_FEEDBACK_EVENT, WORKSPACE_DIR
12+
13+
14+
workspace_aggregate_route_bp = Blueprint("workspace_aggregate_route", __name__)
15+
16+
17+
@workspace_aggregate_route_bp.route(
18+
f"{WORKSPACE_AGGREGATE_ROUTE}/all/<path:relative_path>", methods=["GET"]
19+
)
20+
def get_workspace_aggregate_all(relative_path):
21+
uuid = request.headers.get("uuid")
22+
sid = request.headers.get("sid")
23+
24+
# Ensure the uuid header is present
25+
if not uuid:
26+
return jsonify({"error": "UUID header is missing"}), 400
27+
28+
# Ensure the sid header is present
29+
if not sid:
30+
return jsonify({"error": "SID header is missing"}), 400
31+
32+
# Emit a feedback to the user's console
33+
socketio_emit_to_user_session(
34+
CONSOLE_FEEDBACK_EVENT,
35+
{"type": "info", "message": f"Calculating all file at '{relative_path}'..."},
36+
uuid,
37+
sid,
38+
)
39+
40+
user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid)
41+
file_path = os.path.join(user_workspace_dir, relative_path)
42+
43+
columns_aggregation = request.args.get("columnsAggregation")
44+
columns_aggregation = literal_eval(columns_aggregation)
45+
header_actions = {
46+
field: columns_aggregation[field]["action"] for field in columns_aggregation.keys()
47+
}
48+
header_values = {
49+
field: (
50+
float("inf")
51+
if columns_aggregation[field]["action"] == "min"
52+
else float("-inf") if columns_aggregation[field]["action"] == "max" else float(0)
53+
)
54+
for field in columns_aggregation.keys()
55+
}
56+
skipped_counts = {field: 0 for field in columns_aggregation.keys()}
57+
counts = {field: 0 for field in columns_aggregation.keys()}
58+
59+
try:
60+
with open(file_path, "r", encoding="utf-8") as file:
61+
reader = csv.reader(file)
62+
header = next(reader)
63+
64+
if header:
65+
for row in reader:
66+
for field in columns_aggregation.keys():
67+
action = header_actions[field]
68+
value = row[header.index(field)]
69+
if action == "cnt":
70+
if value:
71+
header_values[field] += float(1)
72+
else:
73+
skipped_counts[field] += 1
74+
elif is_number(value):
75+
if action == "sum":
76+
header_values[field] += float(value)
77+
elif action == "avg":
78+
header_values[field] += float(value)
79+
counts[field] += 1
80+
elif action == "min":
81+
header_values[field] = min(header_values[field], float(value))
82+
elif action == "max":
83+
header_values[field] = max(header_values[field], float(value))
84+
else:
85+
skipped_counts[field] += 1
86+
87+
for field in columns_aggregation.keys():
88+
action = header_actions[field]
89+
if action == "avg":
90+
if counts[field] != 0:
91+
header_values[field] /= counts[field]
92+
93+
# Format the values for the response
94+
formatted_values = {
95+
field: (
96+
"N/A"
97+
if header_values[field] == float("inf")
98+
or header_values[field] == float("-inf")
99+
or (
100+
header_values[field] == float(0)
101+
and (
102+
header_actions[field] != "min"
103+
or header_actions[field] != "max"
104+
or header_actions[field] != "cnt"
105+
)
106+
)
107+
else (
108+
str(int(header_values[field]))
109+
if isinstance(header_values[field], (int, float))
110+
and header_values[field].is_integer()
111+
else f"{header_values[field]:.3f}"
112+
)
113+
)
114+
for field in columns_aggregation.keys()
115+
}
116+
117+
# Build the response data
118+
response_data = {
119+
"fileId": relative_path,
120+
"columnsAggregation": {
121+
field: {
122+
"action": columns_aggregation[field]["action"],
123+
"value": formatted_values[field],
124+
}
125+
for field in columns_aggregation.keys()
126+
},
127+
}
128+
129+
# Emit a feedback to the user's console
130+
skipped_columns_info = []
131+
for field in columns_aggregation.keys():
132+
skipped_count = skipped_counts[field]
133+
if skipped_count != 0:
134+
skipped_columns_info.append(f"'{field}': {skipped_count} cells")
135+
136+
if skipped_columns_info:
137+
socketio_emit_to_user_session(
138+
CONSOLE_FEEDBACK_EVENT,
139+
{
140+
"type": "warn",
141+
"message": f"The following columns had cells skipped due to non-numeric values: {', '.join(skipped_columns_info)}",
142+
},
143+
uuid,
144+
sid,
145+
)
146+
147+
socketio_emit_to_user_session(
148+
CONSOLE_FEEDBACK_EVENT,
149+
{
150+
"type": "succ",
151+
"message": f"File at '{relative_path}' all calculated successfully.",
152+
},
153+
uuid,
154+
sid,
155+
)
156+
157+
return jsonify(response_data)
158+
159+
except FileNotFoundError as e:
160+
logger.error("FileNotFoundError: %s while calculating all %s", e, file_path)
161+
# Emit a feedback to the user's console
162+
socketio_emit_to_user_session(
163+
CONSOLE_FEEDBACK_EVENT,
164+
{
165+
"type": "errr",
166+
"message": f"FileNotFoundError: {e} while calculating all {file_path}",
167+
},
168+
uuid,
169+
sid,
170+
)
171+
return jsonify({"error": "Requested file not found"}), 404
172+
except PermissionError as e:
173+
logger.error("PermissionError: %s while calculating all %s", e, file_path)
174+
# Emit a feedback to the user's console
175+
socketio_emit_to_user_session(
176+
CONSOLE_FEEDBACK_EVENT,
177+
{
178+
"type": "errr",
179+
"message": f"PermissionError: {e} while calculating all {file_path}",
180+
},
181+
uuid,
182+
sid,
183+
)
184+
return jsonify({"error": "Permission denied"}), 403
185+
except UnexpectedError as e:
186+
logger.error("UnexpectedError: %s while calculating all %s", e.message, file_path)
187+
# Emit a feedback to the user's console
188+
socketio_emit_to_user_session(
189+
CONSOLE_FEEDBACK_EVENT,
190+
{
191+
"type": "errr",
192+
"message": f"UnexpectedError: {e.message} while calculating all {file_path}",
193+
},
194+
uuid,
195+
sid,
196+
)
197+
return jsonify({"error": "An internal error occurred"}), 500
198+
199+
200+
@workspace_aggregate_route_bp.route(
201+
f"{WORKSPACE_AGGREGATE_ROUTE}/<path:relative_path>", methods=["GET"]
202+
)
203+
def get_workspace_aggregate(relative_path):
204+
uuid = request.headers.get("uuid")
205+
sid = request.headers.get("sid")
206+
207+
# Ensure the uuid header is present
208+
if not uuid:
209+
return jsonify({"error": "UUID header is missing"}), 400
210+
211+
# Ensure the sid header is present
212+
if not sid:
213+
return jsonify({"error": "SID header is missing"}), 400
214+
215+
# Emit a feedback to the user's console
216+
socketio_emit_to_user_session(
217+
CONSOLE_FEEDBACK_EVENT,
218+
{"type": "info", "message": f"Calculating file at '{relative_path}'..."},
219+
uuid,
220+
sid,
221+
)
222+
223+
user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid)
224+
file_path = os.path.join(user_workspace_dir, relative_path)
225+
226+
field = request.args.get("field")
227+
action = request.args.get("action")
228+
result = float("inf") if action == "min" else float("-inf") if action == "max" else float(0)
229+
count = 0
230+
skipped_count = 0
231+
232+
try:
233+
with open(file_path, "r", encoding="utf-8") as file:
234+
reader = csv.reader(file)
235+
header = next(reader)
236+
237+
if header:
238+
if field not in header:
239+
# Emit a feedback to the user's console
240+
socketio_emit_to_user_session(
241+
CONSOLE_FEEDBACK_EVENT,
242+
{
243+
"type": "errr",
244+
"message": f"Column '{field}' not found in the file '{relative_path}'",
245+
},
246+
uuid,
247+
sid,
248+
)
249+
return (
250+
jsonify(
251+
{"error": f"Column '{field}' not found in the file '{relative_path}'"}
252+
),
253+
404,
254+
)
255+
256+
header_index = header.index(field)
257+
258+
for row in reader:
259+
value = row[header_index]
260+
if action == "cnt":
261+
if value:
262+
result += float(1)
263+
else:
264+
skipped_count += 1
265+
elif is_number(value):
266+
if action == "sum":
267+
result += float(value)
268+
elif action == "avg":
269+
result += float(value)
270+
count += 1
271+
elif action == "min":
272+
result = min(result, float(value))
273+
elif action == "max":
274+
result = max(result, float(value))
275+
else:
276+
skipped_count += 1
277+
278+
if action == "avg" and count != 0:
279+
result /= count
280+
281+
# Emit a feedback to the user's console
282+
if skipped_count != 0:
283+
socketio_emit_to_user_session(
284+
CONSOLE_FEEDBACK_EVENT,
285+
{
286+
"type": "warn",
287+
"message": f"At column '{field}' {skipped_count} cells were skipped because they contain non-numeric values.",
288+
},
289+
uuid,
290+
sid,
291+
)
292+
293+
socketio_emit_to_user_session(
294+
CONSOLE_FEEDBACK_EVENT,
295+
{
296+
"type": "succ",
297+
"message": f"File at '{relative_path}' calculated successfully.",
298+
},
299+
uuid,
300+
sid,
301+
)
302+
303+
formatted_value = (
304+
"N/A"
305+
if result == float("inf")
306+
or result == float("-inf")
307+
or (result == float(0) and action not in ["min", "max", "cnt"])
308+
else str(int(result)) if result.is_integer() else f"{result:.3f}"
309+
)
310+
311+
# Build the response data
312+
response_data = {
313+
"fileId": relative_path,
314+
"field": field,
315+
"action": action,
316+
"value": formatted_value,
317+
}
318+
319+
return jsonify(response_data)
320+
321+
except FileNotFoundError as e:
322+
logger.error("FileNotFoundError: %s while calculating %s", e, file_path)
323+
# Emit a feedback to the user's console
324+
socketio_emit_to_user_session(
325+
CONSOLE_FEEDBACK_EVENT,
326+
{
327+
"type": "errr",
328+
"message": f"FileNotFoundError: {e} while calculating {file_path}",
329+
},
330+
uuid,
331+
sid,
332+
)
333+
return jsonify({"error": "Requested file not found"}), 404
334+
except PermissionError as e:
335+
logger.error("PermissionError: %s while calculating %s", e, file_path)
336+
# Emit a feedback to the user's console
337+
socketio_emit_to_user_session(
338+
CONSOLE_FEEDBACK_EVENT,
339+
{
340+
"type": "errr",
341+
"message": f"PermissionError: {e} while calculating {file_path}",
342+
},
343+
uuid,
344+
sid,
345+
)
346+
return jsonify({"error": "Permission denied"}), 403
347+
except UnexpectedError as e:
348+
logger.error("UnexpectedError: %s while calculating %s", e.message, file_path)
349+
# Emit a feedback to the user's console
350+
socketio_emit_to_user_session(
351+
CONSOLE_FEEDBACK_EVENT,
352+
{
353+
"type": "errr",
354+
"message": f"UnexpectedError: {e.message} while calculating {file_path}",
355+
},
356+
uuid,
357+
sid,
358+
)
359+
return jsonify({"error": "An internal error occurred"}), 500

0 commit comments

Comments
 (0)