Merge pull request #48 from Strexas/MDE/back-end-optimization

Mde/back-end-optimization

app/back-end/gunicorn_config.py

@@ -23,5 +23,8 @@
 # Use Gevent worker class for handling asynchronous requests with WebSocket support
 worker_class = "geventwebsocket.gunicorn.workers.GeventWebSocketWorker"
 
+# Maximum number of simultaneous connections
+worker_connections = 1
+
 # Number of worker processes
 workers = multiprocessing.cpu_count() * 2 + 1

app/back-end/src/constants.py

@@ -31,3 +31,4 @@
 
 # Events
 CONSOLE_FEEDBACK_EVENT = "console_feedback"
+WORKSPACE_FILE_SAVE_FEEDBACK_EVENT = "workspace_file_save_feedback"

app/front-end/package.json

@@ -18,7 +18,6 @@
     "@mui/x-tree-view": "^7.12.1",
     "@react-spring/web": "^9.7.4",
     "axios": "^1.7.4",
-    "papaparse": "^5.4.1",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-router-dom": "^6.26.0",
@@ -27,7 +26,6 @@
   "devDependencies": {
     "@mui/x-data-grid-generator": "^7.12.1",
     "@types/node": "^22.2.0",
-    "@types/papaparse": "^5.3.14",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "@typescript-eslint/eslint-plugin": "^7.15.0",

app/front-end/src/features/editor/components/consoleView/consoleView.tsx

@@ -1,6 +1,7 @@
 import { ConsoleGroup, ConsoleGroupItem } from '@/features/editor/components/consoleView';
 import { ConsoleFeedback } from '@/features/editor/types';
 import { socket } from '@/lib';
+import { Events } from '@/types';
 import { useEffect, useRef, useState } from 'react';
 
 /**
@@ -29,10 +30,10 @@ export const ConsoleView: React.FC = () => {
       setConsoleFeedback((prev) => [...prev, data]);
     };
 
-    socket.on('console_feedback', handleConsoleFeedback);
+    socket.on(Events.CONSOLE_FEEDBACK_EVENT, handleConsoleFeedback);
 
     return () => {
-      socket.off('console_feedback');
+      socket.off(Events.CONSOLE_FEEDBACK_EVENT);
     };
   }, []);
 

app/front-end/src/features/editor/components/editorView/editorColumnMenu.tsx

@@ -0,0 +1,40 @@
+import { styled } from '@mui/material/styles';
+import { GridColumnMenuContainer, GridColumnMenuHideItem, GridColumnMenuProps } from '@mui/x-data-grid';
+
+const StyledGridColumnMenuContainer = styled(GridColumnMenuContainer)(({ theme }) => ({
+  backgroundColor: theme.palette.secondary.main,
+}));
+
+interface GridColumnMenuContainerProps extends GridColumnMenuProps {}
+
+/**
+ * `EditorColumnMenu` component customizes the column menu in a DataGrid with a styled container.
+ *
+ * @description This component extends the default column menu functionality of the DataGrid by applying custom styles
+ * to the menu container. The `StyledGridColumnMenuContainer` applies a background color from the theme's secondary palette
+ * to the menu. The menu includes a `GridColumnMenuHideItem` for hiding the column, which invokes the `hideMenu` function
+ * when clicked.
+ *
+ * @component
+ *
+ * @param {GridColumnMenuContainerProps} props - The props for the component.
+ * @param {() => void} props.hideMenu - A callback function to hide the column menu.
+ * @param {object} props.colDef - Column definition object passed to the menu.
+ * @param {GridColumnMenuProps} [other] - Other props that are passed to the `GridColumnMenuContainer`.
+ *
+ * @example
+ * // Example usage of the EditorColumnMenu component
+ * <EditorColumnMenu
+ *   hideMenu={() => console.log('Hide menu')}
+ *   colDef={columnDefinition}
+ * />
+ *
+ * @returns {JSX.Element} A styled `GridColumnMenuContainer` containing a `GridColumnMenuHideItem`.
+ */
+export const EditorColumnMenu: React.FC<GridColumnMenuContainerProps> = ({ hideMenu, colDef, ...other }) => {
+  return (
+    <StyledGridColumnMenuContainer hideMenu={hideMenu} colDef={colDef} {...other}>
+      <GridColumnMenuHideItem onClick={hideMenu} colDef={colDef!} />
+    </StyledGridColumnMenuContainer>
+  );
+};

app/front-end/src/features/editor/components/editorView/editorToolbar.tsx

@@ -1,12 +1,11 @@
 import { socket } from '@/lib';
+import { Events } from '@/types';
 import { Done as DoneIcon, Error as ErrorIcon } from '@mui/icons-material';
 import { Box, Button, CircularProgress, useTheme } from '@mui/material';
 import {
   GridToolbarColumnsButton,
   GridToolbarContainer,
   GridToolbarDensitySelector,
-  GridToolbarExport,
-  GridToolbarFilterButton,
   GridToolbarProps,
   ToolbarPropsOverrides,
 } from '@mui/x-data-grid';
@@ -55,22 +54,23 @@ export const EditorToolbar: React.FC<EditorToolbarProps> = ({ disabled, handleSa
   const Theme = useTheme();
 
   useEffect(() => {
-    socket.on('workspace_file_update_status', (data) => {
+    const handleWorkspaceFileSaveFeedback = (data: { status: 'success' | 'error' }) => {
       setIsSaving(false);
       setSaveStatus(data.status === 'success');
-    });
+    };
+    socket.on(Events.WORKSPACE_FILE_SAVE_FEEDBACK_EVENT, handleWorkspaceFileSaveFeedback);
 
     return () => {
-      socket.off('workspace_file_update_status');
+      socket.off(Events.WORKSPACE_FILE_SAVE_FEEDBACK_EVENT);
     };
   });
 
   return (
     <GridToolbarContainer>
       <GridToolbarColumnsButton />
-      <GridToolbarFilterButton />
+      {/* <GridToolbarFilterButton /> */}
       <GridToolbarDensitySelector />
-      <GridToolbarExport />
+      {/* <GridToolbarExport /> */}
       <Box sx={{ flexGrow: 1 }} />
       <Button
         onClick={() => {

app/front-end/src/features/editor/components/editorView/editorView.tsx

@@ -1,11 +1,10 @@
-import { EditorToolbar } from '@/features/editor/components/editorView';
+import { EditorColumnMenu, EditorToolbar } from '@/features/editor/components/editorView';
 import { useWorkspaceContext } from '@/features/editor/hooks';
+import { FileDataRequestDTO, FileDataResponseDTO } from '@/features/editor/types';
 import { useSessionContext } from '@/hooks';
-import { axios, socket } from '@/lib';
+import { axios } from '@/lib';
 import { Endpoints } from '@/types';
-import { getUUID } from '@/utils';
 import { DataGrid, GridColDef, GridRowsProp, useGridApiRef } from '@mui/x-data-grid';
-import Papa from 'papaparse';
 import { useEffect, useState } from 'react';
 
 /**
@@ -37,41 +36,54 @@ export const EditorView: React.FC = () => {
   const gridApiRef = useGridApiRef();
   const Workspace = useWorkspaceContext();
 
-  const [rows, setRows] = useState<GridRowsProp>([]);
-  const [columns, setColumns] = useState<GridColDef[]>([]);
+  const [gridColumns, setgridColumns] = useState<GridColDef[]>([]);
+  const [gridRows, setgridRows] = useState<GridRowsProp>([]);
+  const [page, setPage] = useState(0);
+  const [rowsPerPage, setRowsPerPage] = useState(100);
+  const [totalRows, setTotalRows] = useState(0);
   const [isLoading, setIsLoading] = useState(false);
 
   useEffect(() => {
     const getWorkspaceFile = async () => {
       if (!Workspace.fileId) {
-        setRows([]);
-        setColumns([]);
+        setgridColumns([]);
+        setgridRows([]);
         return;
       }
 
       setIsLoading(true);
 
       try {
-        const response = await axios.get(`${Endpoints.WORKSPACE}/${Workspace.fileId}`);
-        const data = response.data;
-        const result = Papa.parse(data, { header: true, skipEmptyLines: true });
+        const response = await axios.get(`${Endpoints.WORKSPACE}/${Workspace.fileId}`, {
+          params: {
+            page: page,
+            rowsPerPage: rowsPerPage,
+          },
+        });
 
-        const columns = result.meta.fields?.map((field, index) => ({
-          field: `col${index}`,
-          headerName: field,
-          width: 150,
-          editable: true,
-        }));
+        const { totalRows, header, rows } = response.data as FileDataResponseDTO;
 
-        // TODO: might need to adress this with type declaration
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const rows = result.data.map((row: any, index) => ({
-          id: index,
-          ...result.meta.fields?.reduce((acc, field, index) => ({ ...acc, [`col${index}`]: row[field] }), {}),
-        }));
+        const parsedColumns = header.map((value) => {
+          return {
+            field: value,
+            headerName: value,
+            width: 150,
+            editable: true,
+          };
+        });
 
-        setColumns(columns || []);
-        setRows(rows || []);
+        const parsedRows = rows.map((row, index) => {
+          return {
+            id: index,
+            ...row.reduce((acc, value, index) => {
+              return { ...acc, [header[index]]: value };
+            }, {}),
+          };
+        });
+
+        setgridColumns(parsedColumns || []);
+        setgridRows(parsedRows || []);
+        setTotalRows(totalRows);
       } catch (error) {
         console.error('Failed to fetch file content:', error);
       } finally {
@@ -80,46 +92,47 @@ export const EditorView: React.FC = () => {
     };
 
     if (connected) getWorkspaceFile();
-  }, [connected, Workspace.fileId, Workspace.fileType]);
+  }, [connected, Workspace.fileId, Workspace.fileType, page, rowsPerPage]);
 
   const handleSave = () => {
-    const header = gridApiRef.current
-      .getAllColumns()
-      .map((column) => column.headerName)
-      .join(',')
-      .concat('\n');
-
-    const rows = gridApiRef.current
-      .getAllRowIds()
-      .map((rowId) => {
-        return gridApiRef.current
-          .getAllColumns()
-          .map((column) => {
-            const cellValue = gridApiRef.current.getCellValue(rowId, column.field);
-
-            // If the cell value contains a comma, wrap it in quotes
-            return cellValue.includes(',') ? `"${cellValue}"` : cellValue;
-          })
-          .join(',');
-      })
-      .join('\n');
-
-    const content = header.concat(rows);
+    const data: FileDataRequestDTO = {
+      page: page,
+      rowsPerPage: rowsPerPage,
+      header: gridApiRef.current.getAllColumns().map((column) => column.field),
+      rows: gridApiRef.current
+        .getAllRowIds()
+        .map((rowId) =>
+          gridApiRef.current.getAllColumns().map((column) => gridApiRef.current.getCellValue(rowId, column.field))
+        ),
+    };
 
-    socket.emit('workspace_file_update', {
-      uuid: getUUID(),
-      fileId: Workspace.fileId,
-      content: content,
-    });
+    axios.put(`${Endpoints.WORKSPACE}/${Workspace.fileId}`, data);
   };
 
   return (
     <DataGrid
       sx={{ height: '100%', border: 'none' }}
       loading={isLoading}
-      rows={rows}
-      columns={columns}
-      slots={{ toolbar: (props) => <EditorToolbar {...props} disabled={isLoading} handleSave={handleSave} /> }}
+      rows={gridRows}
+      columns={gridColumns}
+      pagination
+      paginationMode='server'
+      rowCount={totalRows}
+      disableColumnSorting
+      initialState={{
+        pagination: {
+          paginationModel: { pageSize: rowsPerPage, page: page },
+        },
+      }}
+      pageSizeOptions={[25, 50, 100]}
+      onPaginationModelChange={(model) => {
+        setPage(model.page);
+        setRowsPerPage(model.pageSize);
+      }}
+      slots={{
+        toolbar: (props) => <EditorToolbar {...props} disabled={isLoading} handleSave={handleSave} />,
+        columnMenu: (props) => <EditorColumnMenu {...props} />,
+      }}
       apiRef={gridApiRef}
     />
   );

app/front-end/src/features/editor/components/editorView/index.ts

@@ -1,2 +1,3 @@
+export { EditorColumnMenu } from './editorColumnMenu';
 export { EditorToolbar } from './editorToolbar';
 export { EditorView } from './editorView';

app/front-end/src/features/editor/types/fileData.ts

@@ -0,0 +1,13 @@
+type FileData = {
+  page: number;
+  header: string[];
+  rows: string[][];
+};
+
+export type FileDataRequestDTO = FileData & {
+  rowsPerPage: number;
+};
+
+export type FileDataResponseDTO = FileData & {
+  totalRows: number;
+};

app/front-end/src/features/editor/types/index.ts

@@ -1,3 +1,4 @@
 export { ConsoleFeedbackTypes } from './consoleFeedback';
 export type { ConsoleFeedback } from './consoleFeedback';
+export type { FileDataRequestDTO, FileDataResponseDTO } from './fileData';
 export type { FileTreeViewItemProps } from './fileTreeViewItemProps';

app/front-end/src/types/constants/events.ts

@@ -0,0 +1,4 @@
+export const Events = {
+  CONSOLE_FEEDBACK_EVENT: 'console_feedback',
+  WORKSPACE_FILE_SAVE_FEEDBACK_EVENT: 'workspace_file_save_feedback',
+};

app/front-end/src/types/index.ts

@@ -1,5 +1,6 @@
 export { API_URL, SOCKET_URL } from './constants/constants';
 export { Endpoints } from './constants/endpoints';
+export { Events } from './constants/events';
 export { Paths } from './constants/paths';
 
 export { Colors } from './enums/colors';