From d52efa6fe7784986fa2d2e99ca7d81a65e417a07 Mon Sep 17 00:00:00 2001 From: Mantvydas Deltuva Date: Tue, 10 Sep 2024 19:17:54 +0300 Subject: [PATCH] MDE/PKFE-46 fixed pylint errors and minor structure changes --- .../src/routes/workspace_aggregate_route.py | 100 +++++++++- .../src/routes/workspace_export_route.py | 117 +++++++++++ .../src/routes/workspace_import_route.py | 167 ++++++++++++++++ app/back-end/src/routes/workspace_route.py | 181 +----------------- app/back-end/src/setup/router.py | 6 +- 5 files changed, 387 insertions(+), 184 deletions(-) create mode 100644 app/back-end/src/routes/workspace_export_route.py create mode 100644 app/back-end/src/routes/workspace_import_route.py diff --git a/app/back-end/src/routes/workspace_aggregate_route.py b/app/back-end/src/routes/workspace_aggregate_route.py index 4db5cf6..8a4e7a7 100644 --- a/app/back-end/src/routes/workspace_aggregate_route.py +++ b/app/back-end/src/routes/workspace_aggregate_route.py @@ -1,9 +1,29 @@ +""" +This module defines the routes for aggregating data from user workspaces in a Flask application. +It provides two main routes for performing column-level calculations on CSV files stored in the +user's workspace. The supported operations include summing, averaging, counting, finding the +minimum, and finding the maximum values in specified columns. + +The module emits real-time feedback to the user’s session via Socket.IO, providing status updates +on the calculations, handling skipped cells due to invalid data, and notifying the user of errors +such as file not found, permission denied, or unexpected issues. + +Routes: + - get_workspace_aggregate_all(relative_path): + Calculates aggregate values (sum, avg, min, max, cnt) for multiple columns in a CSV file. + + - get_workspace_aggregate(relative_path): + Calculates an aggregate value (sum, avg, min, max, cnt) for a single column in a CSV file. + +Exceptions are handled to provide feedback through the user’s console using Socket.IO. +""" + # pylint: disable=import-error import os import csv -from flask import Blueprint, request, jsonify from ast import literal_eval +from flask import Blueprint, request, jsonify from src.setup.extensions import logger from src.utils.helpers import socketio_emit_to_user_session, is_number @@ -18,6 +38,39 @@ f"{WORKSPACE_AGGREGATE_ROUTE}/all/", methods=["GET"] ) def get_workspace_aggregate_all(relative_path): + """ + Route to calculate aggregate values (e.g., sum, avg, min, max, cnt) for multiple columns + in a CSV file located in the user's workspace. The columns and their aggregation actions + are specified in the request's query parameters. + + Args: + relative_path (str): The relative path to the CSV file inside the user's workspace. + + Request Headers: + - uuid: A unique identifier for the user's session. + - sid: A session identifier for emitting real-time console feedback via Socket.IO. + + Query Parameters: + - columnsAggregation (str): A stringified dictionary where the keys are column names and + the values are dictionaries with an "action" key specifying the aggregation operation + ('sum', 'avg', 'min', 'max', or 'cnt'). + + Returns: + Response (JSON): + - On success: A JSON object with aggregated results for each specified column. + - On error: A JSON object with an error message and appropriate HTTP status code. + + Emits: + - Real-time console feedback using Socket.IO via the `socketio_emit_to_user_session` + function. Feedback includes the start, completion, and any warnings or errors during + the aggregation process. + + Possible Errors: + - FileNotFoundError: The specified CSV file does not exist. + - PermissionError: Insufficient permissions to read the CSV file. + - UnexpectedError: Any other unexpected error during the aggregation process. + """ + uuid = request.headers.get("uuid") sid = request.headers.get("sid") @@ -140,7 +193,8 @@ def get_workspace_aggregate_all(relative_path): CONSOLE_FEEDBACK_EVENT, { "type": "warn", - "message": f"The following columns had cells skipped due to non-numeric values: {', '.join(skipped_columns_info)}", + "message": "The following columns had cells skipped due to non-numeric " + + f"values: {', '.join(skipped_columns_info)}", }, uuid, sid, @@ -203,6 +257,45 @@ def get_workspace_aggregate_all(relative_path): f"{WORKSPACE_AGGREGATE_ROUTE}/", methods=["GET"] ) def get_workspace_aggregate(relative_path): + """ + Route to calculate an aggregate value (e.g., sum, avg, min, max, cnt) for a single column + in a CSV file located in the user's workspace. The column and the aggregation action + are specified in the request's query parameters. + + Args: + relative_path (str): The relative path to the CSV file inside the user's workspace. + + Request Headers: + - uuid: A unique identifier for the user's session. + - sid: A session identifier for emitting real-time console feedback via Socket.IO. + + Query Parameters: + - field (str): The name of the column to perform the aggregation on. + - action (str): The type of aggregation action to perform + ('sum', 'avg', 'min', 'max', or 'cnt'). + + Returns: + Response (JSON): + - On success: A JSON object with the aggregated result for the specified column. + - On error: A JSON object with an error message and appropriate HTTP status code. + + Emits: + - Real-time console feedback using Socket.IO via the `socketio_emit_to_user_session` + function.Feedback includes the start, completion, and any warnings or errors during the + aggregation process. + + Possible Errors: + - FileNotFoundError: The specified CSV file does not exist. + - PermissionError: Insufficient permissions to read the CSV file. + - UnexpectedError: Any other unexpected error during the aggregation process. + + Notes: + - If the column contains non-numeric values, those cells are skipped and a warning is sent + via Socket.IO. + - The result is formatted as "N/A" if no valid numeric data is found or if the specified + action is invalid for the data present in the column. + """ + uuid = request.headers.get("uuid") sid = request.headers.get("sid") @@ -289,7 +382,8 @@ def get_workspace_aggregate(relative_path): CONSOLE_FEEDBACK_EVENT, { "type": "warn", - "message": f"At column '{field}' {skipped_count} cells were skipped because they contain non-numeric values.", + "message": f"At column '{field}' {skipped_count} cells " + + "were skipped because they contain non-numeric values.", }, uuid, sid, diff --git a/app/back-end/src/routes/workspace_export_route.py b/app/back-end/src/routes/workspace_export_route.py new file mode 100644 index 0000000..94b9dab --- /dev/null +++ b/app/back-end/src/routes/workspace_export_route.py @@ -0,0 +1,117 @@ +""" +Handle file export from the workspace. + +This module provides a Flask route for exporting files from a user's workspace. +It performs the following: +- Retrieves the file based on the user's workspace and the requested path. +- Emits feedback to the user's console via SocketIO during the export process. +- Handles exceptions like file not found, permission errors, or unexpected errors, + returning appropriate HTTP responses. + +Routes: + GET /workspace_export/: Exports the requested file. +""" + +# pylint: disable=import-error + +import os +from flask import Blueprint, request, jsonify, send_file + +from src.setup.extensions import compress, logger +from src.utils.helpers import socketio_emit_to_user_session +from src.utils.exceptions import UnexpectedError +from src.constants import ( + WORKSPACE_DIR, + CONSOLE_FEEDBACK_EVENT, + WORKSPACE_EXPORT_ROUTE, +) + +workspace_export_route_bp = Blueprint("workspace_export_route", __name__) + + +@workspace_export_route_bp.route(f"{WORKSPACE_EXPORT_ROUTE}/", methods=["GET"]) +@compress.compressed() +def get_workspace_export(relative_path): + """ + Export a file from the user's workspace. + + Handles file export requests by retrieving the file from the user's workspace and + sending it as a downloadable attachment. Emits real-time feedback during the process. + + Args: + relative_path (str): The path to the file to be exported within the user's workspace. + + Headers: + uuid: Unique identifier for the user's workspace. + sid: Session identifier for emitting real-time events. + + Returns: + A downloadable file or a JSON response with an error message. + """ + + uuid = request.headers.get("uuid") + sid = request.headers.get("sid") + + # Ensure the uuid header is present + if not uuid: + return jsonify({"error": "UUID header is missing"}), 400 + + # Ensure the sid header is present + if not sid: + return jsonify({"error": "SID header is missing"}), 400 + + try: + user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) + file_path = os.path.join(user_workspace_dir, relative_path) + + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + {"type": "info", "message": f"Exporting file '{relative_path}'..."}, + uuid, + sid, + ) + + response = send_file(file_path, as_attachment=True) + + return response + + except FileNotFoundError as e: + logger.error("FileNotFoundError: %s while exporting %s", e, user_workspace_dir) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"FileNotFoundError: {e} while exporting {user_workspace_dir}", + }, + uuid, + sid, + ) + return jsonify({"error": "Requested file not found"}), 404 + except PermissionError as e: + logger.error("PermissionError: %s while exporting %s", e, user_workspace_dir) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"PermissionError: {e} while exporting {user_workspace_dir}", + }, + uuid, + sid, + ) + return jsonify({"error": "Permission denied"}), 403 + except UnexpectedError as e: + logger.error("UnexpectedError: %s while exporting %s", e.message, user_workspace_dir) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"UnexpectedError: {e.message} while exporting {user_workspace_dir}", + }, + uuid, + sid, + ) + return jsonify({"error": "An internal error occurred"}), 500 diff --git a/app/back-end/src/routes/workspace_import_route.py b/app/back-end/src/routes/workspace_import_route.py new file mode 100644 index 0000000..7bda8d3 --- /dev/null +++ b/app/back-end/src/routes/workspace_import_route.py @@ -0,0 +1,167 @@ +""" +Handle file import into the workspace. + +This module provides a Flask route for importing files into a user's workspace. +It performs the following: +- Accepts files of type 'csv' or 'txt' for upload. +- Saves the file to the specified workspace directory. +- Emits feedback to the user's console via SocketIO during the import process. +- Handles exceptions like file not found, permission errors, or unexpected errors, + returning appropriate HTTP responses. + +Routes: + POST /workspace_import: Imports a file to the root folder. + POST /workspace_import/: Imports a file to the specified folder. +""" + +# pylint: disable=import-error + +import os +from flask import Blueprint, request, jsonify + +from src.setup.extensions import compress, logger +from src.utils.helpers import socketio_emit_to_user_session +from src.utils.exceptions import UnexpectedError +from src.constants import ( + WORKSPACE_DIR, + WORKSPACE_UPDATE_FEEDBACK_EVENT, + CONSOLE_FEEDBACK_EVENT, + WORKSPACE_IMPORT_ROUTE, +) + +workspace_import_route_bp = Blueprint("workspace_import_route", __name__) + + +@workspace_import_route_bp.route(f"{WORKSPACE_IMPORT_ROUTE}", methods=["POST"]) +@workspace_import_route_bp.route(f"{WORKSPACE_IMPORT_ROUTE}/", methods=["POST"]) +@compress.compressed() +def post_workspace_import(relative_path=None): + """ + Import a file into the user's workspace. + + Handles file uploads to the workspace, ensuring the file is of type 'csv' or 'txt'. + Saves the file to the appropriate folder and sends real-time feedback to the user's console. + + Args: + relative_path (str, optional): The folder within the workspace where the file should + be saved. + + Headers: + uuid: Unique identifier for the user's workspace. + sid: Session identifier for emitting real-time events. + + Request: + A file part must be included in the request. + + Returns: + JSON response with a success or error message. + """ + + uuid = request.headers.get("uuid") + sid = request.headers.get("sid") + + # Ensure the uuid header is present + if not uuid: + return jsonify({"error": "UUID header is missing"}), 400 + + # Ensure the sid header is present + if not sid: + return jsonify({"error": "SID header is missing"}), 400 + + if "file" not in request.files: + return jsonify({"error": "No file part in the request"}), 400 + + file = request.files["file"] + + if file.filename == "": + return jsonify({"error": "No file selected for importing"}), 400 + + file_extension = file.filename.rsplit(".", 1)[1].lower() if "." in file.filename else None + if file_extension not in ["csv", "txt"]: + return ( + jsonify( + { + "error": f"FileImportError: Incorrect file type for '{file.filename}'. " + + "Accepted file types: 'csv', 'txt'." + } + ), + 400, + ) + + relative_path_title = relative_path + + if relative_path is None: + relative_path = "" + relative_path_title = "root folder" + + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "info", + "message": f"Importing file '{file.filename}' to '{relative_path_title}'...", + }, + uuid, + sid, + ) + + try: + user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) + folder_path = os.path.join(user_workspace_dir, relative_path) + destination_path = os.path.join(folder_path, file.filename) + file.save(destination_path) + + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + {"type": "succ", "message": f"File {file.filename} was imported successfully."}, + uuid, + sid, + ) + + socketio_emit_to_user_session( + WORKSPACE_UPDATE_FEEDBACK_EVENT, + {"status": "updated"}, + uuid, + sid, + ) + + return jsonify({"message": "File imported successfully"}), 200 + + except FileNotFoundError as e: + logger.error("FileNotFoundError: %s while importing %s", e, user_workspace_dir) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"FileNotFoundError: {e} while importing {user_workspace_dir}", + }, + uuid, + sid, + ) + return jsonify({"error": "Requested file not found"}), 404 + except PermissionError as e: + logger.error("PermissionError: %s while importing %s", e, user_workspace_dir) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"PermissionError: {e} while importing {user_workspace_dir}", + }, + uuid, + sid, + ) + return jsonify({"error": "Permission denied"}), 403 + except UnexpectedError as e: + logger.error("UnexpectedError: %s while importing %s", e.message, user_workspace_dir) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"UnexpectedError: {e.message} while importing {user_workspace_dir}", + }, + uuid, + sid, + ) + return jsonify({"error": "An internal error occurred"}), 500 diff --git a/app/back-end/src/routes/workspace_route.py b/app/back-end/src/routes/workspace_route.py index 0e8843d..2a5347f 100644 --- a/app/back-end/src/routes/workspace_route.py +++ b/app/back-end/src/routes/workspace_route.py @@ -76,13 +76,12 @@ """ # pylint: disable=import-error -# pylint: disable=too-many-locals # pylint: disable=too-many-lines import os import shutil import csv -from flask import Blueprint, request, jsonify, send_file +from flask import Blueprint, request, jsonify from src.setup.extensions import compress, logger from src.utils.helpers import socketio_emit_to_user_session, build_workspace_structure @@ -98,8 +97,6 @@ WORKSPACE_UPDATE_FEEDBACK_EVENT, CONSOLE_FEEDBACK_EVENT, WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, - WORKSPACE_IMPORT_ROUTE, - WORKSPACE_EXPORT_ROUTE, ) workspace_route_bp = Blueprint("workspace_route", __name__) @@ -1065,179 +1062,3 @@ def put_workspace_delete(relative_path): sid, ) return jsonify({"error": "An internal error occurred"}), 500 - -@workspace_route_bp.route(f"{WORKSPACE_IMPORT_ROUTE}", methods=["POST"]) -@workspace_route_bp.route(f"{WORKSPACE_IMPORT_ROUTE}/", methods=["POST"]) -@compress.compressed() -def import_file(relative_path=None): - - uuid = request.headers.get("uuid") - sid = request.headers.get("sid") - - # Ensure the uuid header is present - if not uuid: - return jsonify({"error": "UUID header is missing"}), 400 - - # Ensure the sid header is present - if not sid: - return jsonify({"error": "SID header is missing"}), 400 - - - if 'file' not in request.files: - return jsonify({'error': 'No file part in the request'}), 400 - - file = request.files['file'] - - if file.filename == '': - return jsonify({'error': 'No file selected for importing'}), 400 - - file_extension = file.filename.rsplit('.', 1)[1].lower() if '.' in file.filename else None - if file_extension not in ['csv', 'txt']: - return jsonify({'error': f"FileImportError: Incorrect file type for '{file.filename}'. Accepted file types: 'csv', 'txt'."}), 400 - - relative_path_title = relative_path - - if relative_path is None: - relative_path = "" - relative_path_title = "root folder" - - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - {"type": "info", "message": f"Importing file '{file.filename}' to '{relative_path_title}'..."}, - uuid, - sid, - ) - - try: - user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) - folder_path = os.path.join(user_workspace_dir, relative_path) - destination_path = os.path.join(folder_path, file.filename) - file.save(destination_path) - - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - {"type": "succ", "message": f"File {file.filename} was imported successfully."}, - uuid, - sid, - ) - - socketio_emit_to_user_session( - WORKSPACE_UPDATE_FEEDBACK_EVENT, - {"status": "updated"}, - uuid, - sid, - ) - - except FileNotFoundError as e: - logger.error("FileNotFoundError: %s while accessing %s", e, user_workspace_dir) - # Emit a feedback to the user's console - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - { - "type": "errr", - "message": f"FileNotFoundError: {e} while accessing {user_workspace_dir}", - }, - uuid, - sid, - ) - return jsonify({"error": "Requested file not found"}), 404 - except PermissionError as e: - logger.error("PermissionError: %s while accessing %s", e, user_workspace_dir) - # Emit a feedback to the user's console - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - { - "type": "errr", - "message": f"PermissionError: {e} while accessing {user_workspace_dir}", - }, - uuid, - sid, - ) - return jsonify({"error": "Permission denied"}), 403 - except UnexpectedError as e: - logger.error("UnexpectedError: %s while accessing %s", e.message, user_workspace_dir) - # Emit a feedback to the user's console - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - { - "type": "errr", - "message": f"UnexpectedError: {e.message} while accessing {user_workspace_dir}", - }, - uuid, - sid, - ) - return jsonify({"error": "An internal error occurred"}), 500 - - return jsonify({'message': 'File imported successfully'}), 200 - -@workspace_route_bp.route(f"{WORKSPACE_EXPORT_ROUTE}/", methods=["GET"]) -def export_file(relative_path): - - uuid = request.headers.get("uuid") - sid = request.headers.get("sid") - - # Ensure the uuid header is present - if not uuid: - return jsonify({"error": "UUID header is missing"}), 400 - - # Ensure the sid header is present - if not sid: - return jsonify({"error": "SID header is missing"}), 400 - - try: - user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) - file_path = os.path.join(user_workspace_dir, relative_path) - - #Emit a feedback to the user's console - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - {"type": "info", "message": f"Exporting file '{relative_path}'..."}, - uuid, - sid, - ) - - response = send_file(file_path, as_attachment=True) - - return response - - except FileNotFoundError as e: - logger.error("FileNotFoundError: %s while exporting %s", e, user_workspace_dir) - # Emit a feedback to the user's console - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - { - "type": "errr", - "message": f"FileNotFoundError: {e} while exporting {user_workspace_dir}", - }, - uuid, - sid, - ) - return jsonify({"error": "Requested file not found"}), 404 - except PermissionError as e: - logger.error("PermissionError: %s while exporting %s", e, user_workspace_dir) - # Emit a feedback to the user's console - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - { - "type": "errr", - "message": f"PermissionError: {e} while exporting {user_workspace_dir}", - }, - uuid, - sid, - ) - return jsonify({"error": "Permission denied"}), 403 - except UnexpectedError as e: - logger.error("UnexpectedError: %s while exporting %s", e.message, user_workspace_dir) - # Emit a feedback to the user's console - socketio_emit_to_user_session( - CONSOLE_FEEDBACK_EVENT, - { - "type": "errr", - "message": f"UnexpectedError: {e.message} while exporting {user_workspace_dir}", - }, - uuid, - sid, - ) - return jsonify({"error": "An internal error occurred"}), 500 - - diff --git a/app/back-end/src/setup/router.py b/app/back-end/src/setup/router.py index 068a10f..102e080 100644 --- a/app/back-end/src/setup/router.py +++ b/app/back-end/src/setup/router.py @@ -17,6 +17,8 @@ from src.routes.workspace_route import workspace_route_bp from src.routes.workspace_aggregate_route import workspace_aggregate_route_bp +from src.routes.workspace_export_route import workspace_export_route_bp +from src.routes.workspace_import_route import workspace_import_route_bp def router(prefix): @@ -35,7 +37,9 @@ def router(prefix): router_bp = Blueprint("router", __name__, url_prefix=prefix) # Register API routes with the main router blueprint - router_bp.register_blueprint(workspace_route_bp) + router_bp.register_blueprint(workspace_export_route_bp) + router_bp.register_blueprint(workspace_import_route_bp) router_bp.register_blueprint(workspace_aggregate_route_bp) + router_bp.register_blueprint(workspace_route_bp) return router_bp