From 47599911b259a7e2514465fba688a7c1c240e4fc Mon Sep 17 00:00:00 2001 From: Mantvydas Deltuva Date: Mon, 26 Aug 2024 22:24:16 +0300 Subject: [PATCH 1/4] MDE/back-end-optimization implemented pagination during file fetching and saving --- app/back-end/src/constants.py | 1 + app/back-end/src/routes/workspace_route.py | 194 ++++++++++++++++++++- 2 files changed, 192 insertions(+), 3 deletions(-) diff --git a/app/back-end/src/constants.py b/app/back-end/src/constants.py index c045a08..c7d91f5 100644 --- a/app/back-end/src/constants.py +++ b/app/back-end/src/constants.py @@ -31,3 +31,4 @@ # Events CONSOLE_FEEDBACK_EVENT = "console_feedback" +WORKSPACE_FILE_SAVE_FEEDBACK_EVENT = "workspace_file_save_feedback" diff --git a/app/back-end/src/routes/workspace_route.py b/app/back-end/src/routes/workspace_route.py index 2765b48..c2b5325 100644 --- a/app/back-end/src/routes/workspace_route.py +++ b/app/back-end/src/routes/workspace_route.py @@ -26,7 +26,8 @@ import os import shutil -from flask import Blueprint, request, jsonify, send_file +import csv +from flask import Blueprint, request, jsonify from src.setup.extensions import compress, logger from src.utils.helpers import socketio_emit_to_user_session, build_workspace_structure @@ -36,6 +37,7 @@ WORKSPACE_TEMPLATE_DIR, WORKSPACE_ROUTE, CONSOLE_FEEDBACK_EVENT, + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, ) workspace_route_bp = Blueprint("workspace_route", __name__) @@ -219,12 +221,48 @@ def get_workspace_file(relative_path): user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) file_path = os.path.join(user_workspace_dir, relative_path) + page = int(request.args.get("page", 0)) + rows_per_page = int(request.args.get("rowsPerPage", 100)) + + total_rows = 0 + paginated_rows = [] + start_row = page * rows_per_page + end_row = start_row + rows_per_page + try: # Ensure the user specific directory exists if not os.path.exists(user_workspace_dir): # Copy the template from the template directory to the user's workspace shutil.copytree(WORKSPACE_TEMPLATE_DIR, user_workspace_dir) + # Costly operation to read the file and return the required rows. + # It gets more expensive as the page number increases, needs to go deeper into the file. + # Currently supports CSV files only. + + # Read the file and retrieve the rows + with open(file_path, "r", encoding="utf-8") as file: + reader = csv.reader(file) + # First line as header + header = next(reader) + + # Read the rows within the specified range, otherwise skip to the next row. + # Loop ends when the end row is reached or the end of the file is reached. + for i, row in enumerate(reader): + if i >= start_row and i < end_row: + paginated_rows.append(row) + total_rows += 1 + + if i >= end_row: + break + + # Build the response data + response_data = { + "page": page, + "totalRows": total_rows, + "header": header, + "rows": paginated_rows, + } + # Emit a feedback to the user's console socketio_emit_to_user_session( CONSOLE_FEEDBACK_EVENT, @@ -236,8 +274,8 @@ def get_workspace_file(relative_path): sid, ) - # Serve the file - return send_file(file_path, as_attachment=False) + # Serve the file in batches + return jsonify(response_data) except FileNotFoundError as e: logger.error("FileNotFoundError: %s while accessing %s", e, file_path) @@ -278,3 +316,153 @@ def get_workspace_file(relative_path): sid, ) return jsonify({"error": "An internal error occurred"}), 500 + + +@workspace_route_bp.route(f"{WORKSPACE_ROUTE}/", methods=["PUT"]) +@compress.compressed() +def put_workspace_file(relative_path): + uuid = request.headers.get("uuid") + sid = request.headers.get("sid") + + # Ensure the uuid header is present + if not uuid: + return jsonify({"error": "UUID header is missing"}), 400 + + # Ensure the sid header is present + if not sid: + return jsonify({"error": "SID header is missing"}), 400 + + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + {"type": "info", "message": f"Saving file at '{relative_path}'..."}, + uuid, + sid, + ) + + user_workspace_dir = os.path.join(WORKSPACE_DIR, uuid) + file_path = os.path.join(user_workspace_dir, relative_path) + + data = request.json + page = data.get("page") + rows_per_page = data.get("rowsPerPage") + header = data.get("header") + rows = data.get("rows") + + start_row = page * rows_per_page + end_row = start_row + rows_per_page + + try: + # Ensure the user specific directory exists + if not os.path.exists(user_workspace_dir): + # Copy the template from the template directory to the user's workspace + shutil.copytree(WORKSPACE_TEMPLATE_DIR, user_workspace_dir) + + # Ensure the directory exists + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + # Costly operation to read the entire file and update the required rows. + # One full file cycle. + # Currently supports CSV files only. + + # Create a temporary file to write updated content + temp_file_path = f"{file_path}.tmp" + + # Read the file and write the updated rows + with open(file_path, "r", encoding="utf-8") as infile, open( + temp_file_path, "w", encoding="utf-8" + ) as outfile: + reader = csv.reader(infile) + writer = csv.writer(outfile) + + # Skip the header + next(reader) + writer.writerow(header) # Write the new header + + for i, row in enumerate(reader): + if start_row <= i < end_row: + writer.writerow(rows[i - start_row]) # Write the updated row + else: + writer.writerow(row) # Write the existing row + + # Replace the old file with the new file + os.replace(temp_file_path, file_path) + + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + {"type": "succ", "message": f"File at '{relative_path}' saved successfully."}, + uuid, + sid, + ) + + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "success"}, + uuid, + sid, + ) + + return jsonify({"status": "success"}) + + except FileNotFoundError as e: + logger.error("FileNotFoundError: %s while saving %s", e, file_path) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"FileNotFoundError: {e} while saving {file_path}", + }, + uuid, + sid, + ) + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "error"}, + uuid, + sid, + ) + return jsonify({"error": "Requested file not found"}), 404 + except PermissionError as e: + logger.error("PermissionError: %s while saving %s", e, file_path) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"PermissionError: {e} while saving {file_path}", + }, + uuid, + sid, + ) + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "error"}, + uuid, + sid, + ) + return jsonify({"error": "Permission denied"}), 403 + except UnexpectedError as e: + logger.error("UnexpectedError: %s while saving %s", e.message, file_path) + # Emit a feedback to the user's console + socketio_emit_to_user_session( + CONSOLE_FEEDBACK_EVENT, + { + "type": "errr", + "message": f"UnexpectedError: {e.message} while saving {file_path}", + }, + uuid, + sid, + ) + # Emit a feedback to the user's button + socketio_emit_to_user_session( + WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, + {"status": "error"}, + uuid, + sid, + ) + return jsonify({"error": "An internal error occurred"}), 500 From a8f2e31c4eda90184ec1df7c34a220aa5c22e05d Mon Sep 17 00:00:00 2001 From: Mantvydas Deltuva Date: Mon, 26 Aug 2024 22:26:57 +0300 Subject: [PATCH 2/4] MDE/back-end-optimization fixes on the front-end --- app/front-end/package.json | 2 - .../components/consoleView/consoleView.tsx | 5 +- .../editorView/editorColumnMenu.tsx | 16 +++ .../components/editorView/editorToolbar.tsx | 14 +- .../components/editorView/editorView.tsx | 125 ++++++++++-------- .../editor/components/editorView/index.ts | 1 + .../src/features/editor/types/fileData.ts | 13 ++ .../src/features/editor/types/index.ts | 1 + app/front-end/src/types/constants/events.ts | 4 + app/front-end/src/types/index.ts | 1 + 10 files changed, 115 insertions(+), 67 deletions(-) create mode 100644 app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx create mode 100644 app/front-end/src/features/editor/types/fileData.ts create mode 100644 app/front-end/src/types/constants/events.ts diff --git a/app/front-end/package.json b/app/front-end/package.json index 53a821e..3876996 100644 --- a/app/front-end/package.json +++ b/app/front-end/package.json @@ -18,7 +18,6 @@ "@mui/x-tree-view": "^7.12.1", "@react-spring/web": "^9.7.4", "axios": "^1.7.4", - "papaparse": "^5.4.1", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.26.0", @@ -27,7 +26,6 @@ "devDependencies": { "@mui/x-data-grid-generator": "^7.12.1", "@types/node": "^22.2.0", - "@types/papaparse": "^5.3.14", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", "@typescript-eslint/eslint-plugin": "^7.15.0", diff --git a/app/front-end/src/features/editor/components/consoleView/consoleView.tsx b/app/front-end/src/features/editor/components/consoleView/consoleView.tsx index 1893343..c008e41 100644 --- a/app/front-end/src/features/editor/components/consoleView/consoleView.tsx +++ b/app/front-end/src/features/editor/components/consoleView/consoleView.tsx @@ -1,6 +1,7 @@ import { ConsoleGroup, ConsoleGroupItem } from '@/features/editor/components/consoleView'; import { ConsoleFeedback } from '@/features/editor/types'; import { socket } from '@/lib'; +import { Events } from '@/types'; import { useEffect, useRef, useState } from 'react'; /** @@ -29,10 +30,10 @@ export const ConsoleView: React.FC = () => { setConsoleFeedback((prev) => [...prev, data]); }; - socket.on('console_feedback', handleConsoleFeedback); + socket.on(Events.CONSOLE_FEEDBACK_EVENT, handleConsoleFeedback); return () => { - socket.off('console_feedback'); + socket.off(Events.CONSOLE_FEEDBACK_EVENT); }; }, []); diff --git a/app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx b/app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx new file mode 100644 index 0000000..95868c4 --- /dev/null +++ b/app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx @@ -0,0 +1,16 @@ +import { styled } from '@mui/material/styles'; +import { GridColumnMenuContainer, GridColumnMenuHideItem, GridColumnMenuProps } from '@mui/x-data-grid'; + +const StyledGridColumnMenuContainer = styled(GridColumnMenuContainer)(({ theme }) => ({ + backgroundColor: theme.palette.secondary.main, +})); + +interface GridColumnMenuContainerProps extends GridColumnMenuProps {} + +export const EditorColumnMenu: React.FC = ({ hideMenu, colDef, ...other }) => { + return ( + + + + ); +}; diff --git a/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx b/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx index 61c17d1..3235f30 100644 --- a/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx +++ b/app/front-end/src/features/editor/components/editorView/editorToolbar.tsx @@ -1,12 +1,11 @@ import { socket } from '@/lib'; +import { Events } from '@/types'; import { Done as DoneIcon, Error as ErrorIcon } from '@mui/icons-material'; import { Box, Button, CircularProgress, useTheme } from '@mui/material'; import { GridToolbarColumnsButton, GridToolbarContainer, GridToolbarDensitySelector, - GridToolbarExport, - GridToolbarFilterButton, GridToolbarProps, ToolbarPropsOverrides, } from '@mui/x-data-grid'; @@ -55,22 +54,23 @@ export const EditorToolbar: React.FC = ({ disabled, handleSa const Theme = useTheme(); useEffect(() => { - socket.on('workspace_file_update_status', (data) => { + const handleWorkspaceFileSaveFeedback = (data: { status: 'success' | 'error' }) => { setIsSaving(false); setSaveStatus(data.status === 'success'); - }); + }; + socket.on(Events.WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, handleWorkspaceFileSaveFeedback); return () => { - socket.off('workspace_file_update_status'); + socket.off(Events.WORKSPACE_FILE_SAVE_FEEDBACK_EVENT); }; }); return ( - + {/* */} - + {/* */}