diff --git a/app/back-end/gunicorn_config.py b/app/back-end/gunicorn_config.py index 1e83542..944630f 100644 --- a/app/back-end/gunicorn_config.py +++ b/app/back-end/gunicorn_config.py @@ -23,5 +23,8 @@ # Use Gevent worker class for handling asynchronous requests with WebSocket support worker_class = "geventwebsocket.gunicorn.workers.GeventWebSocketWorker" +# Maximum number of simultaneous connections +worker_connections = 1 + # Number of worker processes workers = multiprocessing.cpu_count() * 2 + 1 diff --git a/app/back-end/src/constants.py b/app/back-end/src/constants.py index c045a08..c7d91f5 100644 --- a/app/back-end/src/constants.py +++ b/app/back-end/src/constants.py @@ -31,3 +31,4 @@ # Events CONSOLE_FEEDBACK_EVENT = "console_feedback" +WORKSPACE_FILE_SAVE_FEEDBACK_EVENT = "workspace_file_save_feedback" diff --git a/app/back-end/src/routes/workspace_route.py b/app/back-end/src/routes/workspace_route.py index 2765b48..c821341 100644 --- a/app/back-end/src/routes/workspace_route.py +++ b/app/back-end/src/routes/workspace_route.py @@ -1,18 +1,19 @@ """ This module defines routes for managing workspace files and directories in a Flask application. -Endpoints include: +Endpoints: - `/workspace`: Retrieves the structure of the workspace directory, including files and folders. -- `/workspace/`: Retrieves a specific file from the workspace directory. +- `/workspace/`: Retrieves or updates a specific file in the workspace + directory. Dependencies: - os: For file and directory operations, such as checking existence and copying directories. - shutil: For copying directory trees, ensuring that user-specific directories are properly initialized. +- csv: For handling CSV file reading and writing. - flask.Blueprint: To create and organize route blueprints for modular route management in Flask. - flask.request: To handle incoming HTTP requests and extract headers and parameters. - flask.jsonify: To create JSON responses for API endpoints. -- flask.send_file: To serve files for download. Extensions: - src.setup.extensions.compress: Used for compressing responses to optimize data transfer. @@ -20,13 +21,67 @@ and errors. - src.setup.constants: Contains constants for directory paths and routes used in the workspace management. +- src.utils.helpers: Provides utility functions for socket communication and workspace structure + building. +- src.utils.exceptions: Defines custom exceptions used for error handling. + +Endpoints: + +1. `/workspace` (GET): + - **Description**: Retrieves the structure of the user's workspace directory. If the directory + does not exist, initializes it by copying a template directory. + - **Headers**: Requires `uuid` and `sid` headers to identify the user session. + - **Returns**: + - `200 OK`: JSON representation of the workspace directory structure. + - `400 Bad Request`: If `uuid` or `sid` headers are missing. + - `403 Forbidden`: If there is a permission issue accessing the workspace. + - `404 Not Found`: If the workspace directory or files are not found. + - `500 Internal Server Error`: For unexpected errors. + +2. `/workspace/` (GET): + - **Description**: Retrieves a specific file from the user's workspace directory. Supports + pagination for large files. + - **Headers**: Requires `uuid` and `sid` headers. + - **Query Parameters**: + - `page` (int): Page number of data to retrieve (default is 0). + - `rowsPerPage` (int): Number of rows per page (default is 100). + - **Returns**: + - `200 OK`: JSON response containing paginated file data. + - `400 Bad Request`: If `uuid` or `sid` headers are missing. + - `403 Forbidden`: If there is a permission issue accessing the file. + - `404 Not Found`: If the requested file does not exist. + - `500 Internal Server Error`: For unexpected errors. + +3. `/workspace/` (PUT): + - **Description**: Saves or updates a file in the user's workspace directory. Supports CSV files + and updates rows in the specified range. + - **Headers**: Requires `uuid` and `sid` headers. + - **Request Body**: + - `page` (int): Current page number of data to be saved. + - `rowsPerPage` (int): Number of rows per page. + - `header` (list): Header row for the CSV file. + - `rows` (list): Rows of data to be saved. + - **Returns**: + - `200 OK`: Success message indicating the file was saved successfully. + - `400 Bad Request`: If `uuid` or `sid` headers are missing. + - `403 Forbidden`: If there is a permission issue while saving the file. + - `404 Not Found`: If the requested file does not exist. + - `500 Internal Server Error`: For unexpected errors. + +Errors and Feedback: +- Feedback is sent to the user's console via WebSocket events about the status of workspace and file + operations. +- Errors include detailed logging and console feedback for issues such as missing files, permission + errors, and unexpected exceptions. """ # pylint: disable=import-error +# pylint: disable=too-many-locals import os import shutil -from flask import Blueprint, request, jsonify, send_file +import csv +from flask import Blueprint, request, jsonify from src.setup.extensions import compress, logger from src.utils.helpers import socketio_emit_to_user_session, build_workspace_structure @@ -36,6 +91,7 @@ WORKSPACE_TEMPLATE_DIR, WORKSPACE_ROUTE, CONSOLE_FEEDBACK_EVENT, + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, ) workspace_route_bp = Blueprint("workspace_route", __name__) @@ -171,30 +227,42 @@ def get_workspace_file(relative_path): """ Retrieve a specific file from the workspace directory. - This endpoint serves files from the user's workspace directory. If the directory does not exist, - it copies a template directory to the user's workspace. The file specified by `relative_path` - is then served for download. Feedback about the file retrieval process is sent to the user's - console via Socket.IO. + This endpoint retrieves a file from the user's workspace directory based on the provided + `relative_path`. If the user's workspace directory does not exist, it initializes the workspace + by copying a template directory. The function supports pagination for large files, returning + a specified range of rows from a CSV file. Feedback about the file retrieval process is sent to + the user's console via WebSocket events. Args: relative_path (str): The path to the file within the user's workspace directory. + Headers: + uuid (str): The unique identifier for the user. + sid (str): The session identifier for the user. + + Query Parameters: + page (int): The page number of data to retrieve (default is 0). + rowsPerPage (int): The number of rows per page (default is 100). + Returns: - Response: A Flask response object containing the file or an error message. The response is: - - `200 OK` with the file if successful. + Response: A JSON response containing the paginated file data or an error message. The + response includes: + - `200 OK` with the file data if successful. - `400 Bad Request` if required headers are missing. - `403 Forbidden` if there is a permission error. - `404 Not Found` if the requested file does not exist. - `500 Internal Server Error` for unexpected errors. + Emits: + CONSOLE_FEEDBACK_EVENT (str): Emits feedback messages to the user's console. + Errors and Feedback: - - If the `uuid` or `sid` headers are missing, a `400 Bad Request` response is returned. - - On successful file retrieval, a success message is emitted to the user's console. - - On errors, appropriate feedback is emitted to the user's console and an error response is - returned: - - `FileNotFoundError`: Indicates the requested file was not found. - - `PermissionError`: Indicates permission issues while accessing the file. - - Other exceptions: Logs and reports unexpected errors. + - Missing `uuid` or `sid` headers result in a `400 Bad Request` response. + - Successful file retrieval emits a success message to the user's console. + - File not found or permission errors emit corresponding error messages to the user's + console and return appropriate HTTP error responses. + - Unexpected errors are logged, reported to the user's console, and result in a `500 + Internal Server Error` response. """ uuid = request.headers.get("uuid") @@ -219,12 +287,48 @@ def get_workspace_file(relative_path): user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) file_path = os.path.join(user_workspace_dir, relative_path) + page = int(request.args.get("page", 0)) + rows_per_page = int(request.args.get("rowsPerPage", 100)) + + total_rows = 0 + paginated_rows = [] + start_row = page * rows_per_page + end_row = start_row + rows_per_page + try: # Ensure the user specific directory exists if not os.path.exists(user_workspace_dir): # Copy the template from the template directory to the user's workspace shutil.copytree(WORKSPACE_TEMPLATE_DIR, user_workspace_dir) + # Costly operation to read the file and return the required rows. + # It gets more expensive as the page number increases, needs to go deeper into the file. + # Currently supports CSV files only. + + # Read the file and retrieve the rows + with open(file_path, "r", encoding="utf-8") as file: + reader = csv.reader(file) + # First line as header + header = next(reader) + + # Read the rows within the specified range, otherwise skip to the next row. + # Loop ends when the end row is reached or the end of the file is reached. + for i, row in enumerate(reader): + if start_row <= i < end_row: + paginated_rows.append(row) + total_rows += 1 + + if i >= end_row: + break + + # Build the response data + response_data = { + "page": page, + "totalRows": total_rows, + "header": header, + "rows": paginated_rows, + } + # Emit a feedback to the user's console socketio_emit_to_user_session( CONSOLE_FEEDBACK_EVENT, @@ -236,8 +340,8 @@ def get_workspace_file(relative_path): sid, ) - # Serve the file - return send_file(file_path, as_attachment=False) + # Serve the file in batches + return jsonify(response_data) except FileNotFoundError as e: logger.error("FileNotFoundError: %s while accessing %s", e, file_path) @@ -278,3 +382,190 @@ def get_workspace_file(relative_path): sid, ) return jsonify({"error": "An internal error occurred"}), 500 + + +@workspace_route_bp.route(f"{WORKSPACE_ROUTE}/", methods=["PUT"]) +@compress.compressed() +def put_workspace_file(relative_path): + """ + Handles a PUT request to save or update a workspace file for a specific user. + + This function processes a request to save or update a file in the user's workspace directory. + It validates the presence of required headers (UUID and SID), processes the provided data, and + writes the updated content to the specified file. The function also handles potential errors + and sends feedback to the user's console and button via WebSocket events. + + Args: + relative_path (str): The relative path of the file within the user's workspace directory. + + Headers: + - uuid (str): The unique identifier for the user. + - sid (str): The session identifier for the user. + + Request Body: + - page (int): The current page number of the data to be saved. + - rowsPerPage (int): The number of rows per page. + - header (list): The header row for the CSV file. + - rows (list): The rows of data to be saved, corresponding to the current page. + + Emits: + - CONSOLE_FEEDBACK_EVENT (str): Emits feedback messages to the user's console. + - WORKSPACE_FILE_SAVE_FEEDBACK_EVENT (str): Emits a status message indicating the success or + failure of the file save operation. + + Returns: + Response: A JSON response indicating the success or failure of the operation. + + Status Codes: + 200: Success - File saved successfully. + 400: Bad Request - UUID or SID header is missing. + 403: Forbidden - Permission error while saving the file. + 404: Not Found - Requested file not found. + 500: Internal Server Error - An unexpected error occurred. + """ + + uuid = request.headers.get("uuid") + sid = request.headers.get("sid") + + # Ensure the uuid header is present + if not uuid: + return jsonify({"error": "UUID header is missing"}), 400 + + # Ensure the sid header is present + if not sid: + return jsonify({"error": "SID header is missing"}), 400 + + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + {"type": "info", "message": f"Saving file at '{relative_path}'..."}, + uuid, + sid, + ) + + user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) + file_path = os.path.join(user_workspace_dir, relative_path) + + data = request.json + page = data.get("page") + rows_per_page = data.get("rowsPerPage") + header = data.get("header") + rows = data.get("rows") + + start_row = page * rows_per_page + end_row = start_row + rows_per_page + + try: + # Ensure the user specific directory exists + if not os.path.exists(user_workspace_dir): + # Copy the template from the template directory to the user's workspace + shutil.copytree(WORKSPACE_TEMPLATE_DIR, user_workspace_dir) + + # Ensure the directory exists + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + # Costly operation to read the entire file and update the required rows. + # One full file cycle. + # Currently supports CSV files only. + + # Create a temporary file to write updated content + temp_file_path = f"{file_path}.tmp" + + # Read the file and write the updated rows + with open(file_path, "r", encoding="utf-8") as infile, open( + temp_file_path, "w", encoding="utf-8" + ) as outfile: + reader = csv.reader(infile) + writer = csv.writer(outfile) + + # Skip the header + next(reader) + writer.writerow(header) # Write the new header + + for i, row in enumerate(reader): + if start_row <= i < end_row: + writer.writerow(rows[i - start_row]) # Write the updated row + else: + writer.writerow(row) # Write the existing row + + # Replace the old file with the new file + os.replace(temp_file_path, file_path) + + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + {"type": "succ", "message": f"File at '{relative_path}' saved successfully."}, + uuid, + sid, + ) + + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "success"}, + uuid, + sid, + ) + + return jsonify({"status": "success"}) + + except FileNotFoundError as e: + logger.error("FileNotFoundError: %s while saving %s", e, file_path) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"FileNotFoundError: {e} while saving {file_path}", + }, + uuid, + sid, + ) + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "error"}, + uuid, + sid, + ) + return jsonify({"error": "Requested file not found"}), 404 + except PermissionError as e: + logger.error("PermissionError: %s while saving %s", e, file_path) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"PermissionError: {e} while saving {file_path}", + }, + uuid, + sid, + ) + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "error"}, + uuid, + sid, + ) + return jsonify({"error": "Permission denied"}), 403 + except UnexpectedError as e: + logger.error("UnexpectedError: %s while saving %s", e.message, file_path) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"UnexpectedError: {e.message} while saving {file_path}", + }, + uuid, + sid, + ) + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "error"}, + uuid, + sid, + ) + return jsonify({"error": "An internal error occurred"}), 500 diff --git a/app/front-end/package.json b/app/front-end/package.json index 53a821e..3876996 100644 --- a/app/front-end/package.json +++ b/app/front-end/package.json @@ -18,7 +18,6 @@ "@mui/x-tree-view": "^7.12.1", "@react-spring/web": "^9.7.4", "axios": "^1.7.4", - "papaparse": "^5.4.1", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.26.0", @@ -27,7 +26,6 @@ "devDependencies": { "@mui/x-data-grid-generator": "^7.12.1", "@types/node": "^22.2.0", - "@types/papaparse": "^5.3.14", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", "@typescript-eslint/eslint-plugin": "^7.15.0", diff --git a/app/front-end/src/features/editor/components/consoleView/consoleView.tsx b/app/front-end/src/features/editor/components/consoleView/consoleView.tsx index 1893343..c008e41 100644 --- a/app/front-end/src/features/editor/components/consoleView/consoleView.tsx +++ b/app/front-end/src/features/editor/components/consoleView/consoleView.tsx @@ -1,6 +1,7 @@ import { ConsoleGroup, ConsoleGroupItem } from '@/features/editor/components/consoleView'; import { ConsoleFeedback } from '@/features/editor/types'; import { socket } from '@/lib'; +import { Events } from '@/types'; import { useEffect, useRef, useState } from 'react'; /** @@ -29,10 +30,10 @@ export const ConsoleView: React.FC = () => { setConsoleFeedback((prev) => [...prev, data]); }; - socket.on('console_feedback', handleConsoleFeedback); + socket.on(Events.CONSOLE_FEEDBACK_EVENT, handleConsoleFeedback); return () => { - socket.off('console_feedback'); + socket.off(Events.CONSOLE_FEEDBACK_EVENT); }; }, []); diff --git a/app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx b/app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx new file mode 100644 index 0000000..567c564 --- /dev/null +++ b/app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx @@ -0,0 +1,40 @@ +import { styled } from '@mui/material/styles'; +import { GridColumnMenuContainer, GridColumnMenuHideItem, GridColumnMenuProps } from '@mui/x-data-grid'; + +const StyledGridColumnMenuContainer = styled(GridColumnMenuContainer)(({ theme }) => ({ + backgroundColor: theme.palette.secondary.main, +})); + +interface GridColumnMenuContainerProps extends GridColumnMenuProps {} + +/** + * `EditorColumnMenu` component customizes the column menu in a DataGrid with a styled container. + * + * @description This component extends the default column menu functionality of the DataGrid by applying custom styles + * to the menu container. The `StyledGridColumnMenuContainer` applies a background color from the theme's secondary palette + * to the menu. The menu includes a `GridColumnMenuHideItem` for hiding the column, which invokes the `hideMenu` function + * when clicked. + * + * @component + * + * @param {GridColumnMenuContainerProps} props - The props for the component. + * @param {() => void} props.hideMenu - A callback function to hide the column menu. + * @param {object} props.colDef - Column definition object passed to the menu. + * @param {GridColumnMenuProps} [other] - Other props that are passed to the `GridColumnMenuContainer`. + * + * @example + * // Example usage of the EditorColumnMenu component + * console.log('Hide menu')} + * colDef={columnDefinition} + * /> + * + * @returns {JSX.Element} A styled `GridColumnMenuContainer` containing a `GridColumnMenuHideItem`. + */ +export const EditorColumnMenu: React.FC = ({ hideMenu, colDef, ...other }) => { + return ( + + + + ); +}; diff --git a/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx b/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx index 61c17d1..3235f30 100644 --- a/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx +++ b/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx @@ -1,12 +1,11 @@ import { socket } from '@/lib'; +import { Events } from '@/types'; import { Done as DoneIcon, Error as ErrorIcon } from '@mui/icons-material'; import { Box, Button, CircularProgress, useTheme } from '@mui/material'; import { GridToolbarColumnsButton, GridToolbarContainer, GridToolbarDensitySelector, - GridToolbarExport, - GridToolbarFilterButton, GridToolbarProps, ToolbarPropsOverrides, } from '@mui/x-data-grid'; @@ -55,22 +54,23 @@ export const EditorToolbar: React.FC = ({ disabled, handleSa const Theme = useTheme(); useEffect(() => { - socket.on('workspace_file_update_status', (data) => { + const handleWorkspaceFileSaveFeedback = (data: { status: 'success' | 'error' }) => { setIsSaving(false); setSaveStatus(data.status === 'success'); - }); + }; + socket.on(Events.WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, handleWorkspaceFileSaveFeedback); return () => { - socket.off('workspace_file_update_status'); + socket.off(Events.WORKSPACE_FILE_SAVE_FEEDBACK_EVENT); }; }); return ( - + {/* */} - + {/* */}