Preparing ops logs for public release (#983)

* Adding back in setting a variable for where the logs should go

* Starting to wire back up the AddingLogTypes

* Cleaning up the store a bit as we no longer need to care what fetching is happening

* Some more cleaning up of state in store

* Starting to make the refresh call to be directly passing props to reduce
  chance that states can start clashing when called in quick succession

* Passing around the start and end of parsing logs

* Getting the loading of new logs working

* Prevent duplicate fetching

* Cleaning up logging

* No longer need the add type as we can just pass the parsed values

* Removing the add type
Starting to pass the offset when fetching newer logs

* Removing the add type
Starting to pass the offset when fetching newer logs

* Clean up logging
Do not override the start if provided
Remove the adjusted bytes for now

* Do not think this applies anymore

* Passing the range with the docs

* Updating types a bit to plan for future typing

* logging and comment cleanup

* Handling intervals

* Cleaning up the debounce

* commenting

* cleaning up logging

* Renaming and typing work

* Refactoring and moving stuff around to prepare for it to be shared
Trying to get the logs to show open/closed/height properly

* Adding a bit more padding to help with showing on scroll

* Trying to make sure the waiting rows show up all the time

* Trying to get the waiting rows to show more consistently

* typing log levels

* commenting

Author: travjenkins

src/components/collection/DataPreview/ListView.tsx

@@ -8,7 +8,8 @@ import {
     jsonViewTheme,
     semiTransparentBackground,
 } from 'context/Theme';
-import { JournalRecord, useJournalData } from 'hooks/journals/useJournalData';
+import { JournalRecord } from 'hooks/journals/types';
+import { useJournalData } from 'hooks/journals/useJournalData';
 import { LiveSpecsQuery_spec } from 'hooks/useLiveSpecs';
 import { JsonPointer } from 'json-ptr';
 import { isEmpty } from 'lodash';

src/components/tables/Logs/Body.tsx

@@ -18,6 +18,8 @@ import {
     DEFAULT_ROW_HEIGHT_WITHOUT_FIELDS,
     UUID_NEWEST_LOG,
     UUID_OLDEST_LOG,
+    VIRTUAL_TABLE_BODY_PADDING,
+    WAITING_ROW_HEIGHT,
 } from './shared';
 import useLogColumns from './useLogColumns';
 import { LogsTableRow } from './Row';
@@ -52,7 +54,7 @@ function LogsTableBody({ outerRef, tableScroller, virtualRows }: Props) {
                     _meta: {
                         uuid: UUID_OLDEST_LOG,
                     },
-                    level: 'waiting',
+                    level: 'ui_waiting',
                     message: '',
                     ts: '',
                 },
@@ -61,7 +63,7 @@ function LogsTableBody({ outerRef, tableScroller, virtualRows }: Props) {
                     _meta: {
                         uuid: UUID_NEWEST_LOG,
                     },
-                    level: 'waiting',
+                    level: 'ui_waiting',
                     message: '',
                     ts: '',
                 },
@@ -71,68 +73,25 @@ function LogsTableBody({ outerRef, tableScroller, virtualRows }: Props) {
         return null;
     }, [documents]);
 
-    const getItemSize = useCallback(
-        (rowIndex: number) => {
-            const row = itemData?.[rowIndex];
-
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            if (!row) {
-                return 0;
-            }
-
-            const customHeight = expandedHeights.current.get(row._meta.uuid);
-
-            return customHeight && customHeight > 0
-                ? customHeight
-                : isEmpty(row.fields)
-                ? DEFAULT_ROW_HEIGHT_WITHOUT_FIELDS
-                : DEFAULT_ROW_HEIGHT;
-        },
-        [itemData]
-    );
-
-    const openRow = useCallback((uuid: string, isOpen: boolean) => {
-        if (isOpen) {
-            openRows.current.set(uuid, isOpen);
-        } else {
-            openRows.current.delete(uuid);
-        }
-    }, []);
-
-    const updateRowHeight = useCallback(
-        (index: number, uuid: string, height: number) => {
-            if (
-                height > 0 ||
-                height === DEFAULT_ROW_HEIGHT_WITHOUT_FIELDS ||
-                height === DEFAULT_ROW_HEIGHT
-            ) {
-                expandedHeights.current.set(uuid, height);
-            } else {
-                expandedHeights.current.delete(uuid);
-            }
-
-            tableScroller()?.resetAfterIndex(index);
-        },
-        [tableScroller]
-    );
-
     const renderRow = useCallback(
         ({ data, index, style }: ListChildComponentProps) => {
             const row = data[index];
-            const uuid = row._meta.uuid;
             return (
                 <LogsTableRow
                     row={row}
                     style={style}
-                    rowExpanded={(height) =>
-                        updateRowHeight(index, uuid, height)
+                    rowExpanded={(uuid, height) => {
+                        expandedHeights.current.set(uuid, height);
+                        tableScroller()?.resetAfterIndex(index);
+                    }}
+                    rowOpened={(uuid, isOpen) =>
+                        openRows.current.set(uuid, isOpen)
                     }
-                    rowOpened={(isOpen) => openRow(uuid, isOpen)}
-                    renderOpen={Boolean(openRows.current.get(uuid))}
+                    renderOpen={Boolean(openRows.current.get(row._meta.uuid))}
                 />
             );
         },
-        [openRow, updateRowHeight]
+        [tableScroller]
     );
 
     if (itemData && itemData.length > 0) {
@@ -152,9 +111,35 @@ function LogsTableBody({ outerRef, tableScroller, virtualRows }: Props) {
                                 itemKey={(index, data) => {
                                     return data[index]._meta.uuid;
                                 }}
-                                itemSize={getItemSize}
+                                itemSize={(index) => {
+                                    const row = itemData[index];
+
+                                    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+                                    if (!row) {
+                                        return 0;
+                                    }
+
+                                    // Due to the intersection observer we need to force a specific height
+                                    if (row.level === 'ui_waiting') {
+                                        return WAITING_ROW_HEIGHT;
+                                    }
+
+                                    const customHeight =
+                                        expandedHeights.current.get(
+                                            row._meta.uuid
+                                        );
+
+                                    return customHeight && customHeight > 0
+                                        ? customHeight
+                                        : isEmpty(row.fields)
+                                        ? DEFAULT_ROW_HEIGHT_WITHOUT_FIELDS
+                                        : DEFAULT_ROW_HEIGHT;
+                                }}
                                 overscanCount={10}
-                                style={{ paddingBottom: 10, paddingTop: 10 }}
+                                style={{
+                                    paddingBottom: VIRTUAL_TABLE_BODY_PADDING,
+                                    paddingTop: VIRTUAL_TABLE_BODY_PADDING,
+                                }}
                                 width={width}
                             >
                                 {renderRow}

src/components/tables/Logs/Row.tsx

@@ -10,8 +10,8 @@ import WaitingForNewLogsRow from './WaitingForRow/NewLogs';
 
 interface RowProps {
     row: OpsLogFlowDocument;
-    rowExpanded: (height: number) => void;
-    rowOpened: (isOpen: boolean) => void;
+    rowExpanded: (uuid: string, height: number) => void;
+    rowOpened: (uuid: string, isOpen: boolean) => void;
     style: CSSProperties;
     renderOpen?: boolean;
 }
@@ -24,9 +24,10 @@ export function LogsTableRow({
     style,
 }: RowProps) {
     const theme = useTheme();
-    const renderedHeight = style.height;
 
-    const previousHeight = useRef(renderedHeight);
+    const uuid = useRef(row._meta.uuid);
+    const previousHeight = useRef(style.height);
+
     const [open, setOpen] = useState(renderOpen);
     const [heightChanging, setHeightChanging] = useState(false);
 
@@ -39,7 +40,7 @@ export function LogsTableRow({
         }
 
         previousHeight.current = rowSizeHeight;
-        rowExpanded(rowSizeHeight);
+        rowExpanded(uuid.current, rowSizeHeight);
         setHeightChanging(false);
     }, [rowExpanded, rowSizeHeight]);
 
@@ -48,19 +49,17 @@ export function LogsTableRow({
 
         setHeightChanging(true);
         setOpen(newVal);
-        rowOpened(newVal);
+        rowOpened(uuid.current, newVal);
     };
 
-    const uuid = row._meta.uuid;
     const WaitingComponent =
-        uuid === UUID_NEWEST_LOG
+        uuid.current === UUID_NEWEST_LOG
             ? WaitingForNewLogsRow
-            : uuid === UUID_OLDEST_LOG
+            : uuid.current === UUID_OLDEST_LOG
             ? WaitingForOldLogsRow
             : null;
 
     if (WaitingComponent) {
-        // The fields thing is pretty janky and hacky but it worked and made this much easier
         return <WaitingComponent style={style} sizeRef={sizeRef} />;
     }
 

src/components/tables/Logs/WaitingForRow/Base.tsx

@@ -1,7 +1,6 @@
 import { Box, TableCell, TableRow, Typography, useTheme } from '@mui/material';
 import SpinnerIcon from 'components/logs/SpinnerIcon';
 import { BaseTypographySx } from 'components/tables/cells/logs/shared';
-import { DEFAULT_POLLING } from 'context/SWR';
 import {
     errorOutlinedButtonBackground,
     tableRowActive_Finished__Background,
@@ -17,74 +16,85 @@ import {
     useJournalDataLogsStore_fetchMoreLogs,
     useJournalDataLogsStore_lastFetchFailed,
 } from 'stores/JournalData/Logs/hooks';
+import { VIRTUAL_TABLE_BODY_PADDING } from '../shared';
 import { FetchMoreLogsOptions, WaitingForRowProps } from '../types';
 
 interface Props extends WaitingForRowProps {
     fetchOption: FetchMoreLogsOptions;
     disabled?: boolean;
+    interval?: number;
 }
 
-function WaitingForRowBase({ disabled, fetchOption, sizeRef, style }: Props) {
+function WaitingForRowBase({
+    disabled,
+    interval = 500,
+    fetchOption,
+    sizeRef,
+    style,
+}: Props) {
     const theme = useTheme();
 
-    const runFetch = useRef(true);
-
-    const [intervalLength, setIntervalLength] = useState(500);
+    const [allowFetch, setAllowFetch] = useState(false);
 
     const intersectionRef = useRef<HTMLElement>(null);
     const intersection = useIntersection(intersectionRef, {
         root: null,
-        rootMargin: '0px',
+        rootMargin: `${VIRTUAL_TABLE_BODY_PADDING}px`,
         threshold: 0.7,
     });
 
     const messageKey = `ops.logsTable.waitingForLogs.${fetchOption}`;
 
     const lastFetchFailed = useJournalDataLogsStore_lastFetchFailed();
     const fetchMoreLogs = useJournalDataLogsStore_fetchMoreLogs();
+
+    // Kinda hacky - but checking this flag here keeps the effect trigger
+    //  as it is flipped back and forth
     const fetchingMore = useJournalDataLogsStore_fetchingMore();
 
     const fetchMore = useCallback(() => {
-        if (lastFetchFailed) {
-            return;
-        }
-        if (!fetchingMore && intersection?.isIntersecting) {
-            runFetch.current = false;
-
-            // When checking for new ones fall back to give some time
-            //  for the entity to actually write logs
-            if (fetchOption === 'new') {
-                setIntervalLength(DEFAULT_POLLING);
-            }
-            fetchMoreLogs(fetchOption);
-        } else {
-            runFetch.current = true;
-        }
-    }, [
-        fetchMoreLogs,
-        fetchOption,
-        fetchingMore,
-        intersection?.isIntersecting,
-        lastFetchFailed,
-    ]);
+        setAllowFetch(false);
+        fetchMoreLogs(fetchOption);
+    }, [fetchMoreLogs, fetchOption]);
 
+    // Cannot figure out the deps
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    const debouncedFetch = useCallback(debounce(fetchMore, intervalLength), [
+    const debouncedFetch = useCallback(debounce(fetchMore, interval), [
         fetchMore,
-        intervalLength,
     ]);
 
+    // If at anytime the row is not visible cancel any ongoing loading
     useEffect(() => {
-        if (!lastFetchFailed && !disabled && intersection?.isIntersecting) {
-            debouncedFetch();
+        if (!intersection?.isIntersecting) {
+            debouncedFetch.cancel();
         }
+    }, [debouncedFetch, intersection?.isIntersecting]);
+
+    // Keeping all this logic in a stand alone effect/state because we might
+    //  need to expand this beyond just checking some simple booleans
+    useEffect(() => {
+        setAllowFetch(
+            Boolean(
+                !fetchingMore &&
+                    !lastFetchFailed &&
+                    !disabled &&
+                    intersection?.isIntersecting
+            )
+        );
     }, [
         debouncedFetch,
         disabled,
+        fetchingMore,
         intersection?.isIntersecting,
         lastFetchFailed,
     ]);
 
+    useEffect(() => {
+        if (allowFetch) {
+            debouncedFetch();
+        }
+    }, [allowFetch, debouncedFetch]);
+
     return (
         <TableRow
             component={Box}
@@ -100,7 +110,7 @@ function WaitingForRowBase({ disabled, fetchOption, sizeRef, style }: Props) {
                     lastFetchFailed || disabled || intersection?.isIntersecting
                         ? 1
                         : 0,
-                transition: 'all 50ms ease-in-out',
+                transition: 'all 100ms ease-in-out',
             }}
         >
             <Box ref={intersectionRef}>

src/components/tables/Logs/WaitingForRow/NewLogs.tsx

@@ -1,8 +1,16 @@
+import { DEFAULT_POLLING } from 'context/SWR';
 import { WaitingForRowProps } from '../types';
 import WaitingForRowBase from './Base';
 
 function WaitingForNewLogsRow(props: WaitingForRowProps) {
-    return <WaitingForRowBase {...props} fetchOption="new" disabled={false} />;
+    return (
+        <WaitingForRowBase
+            {...props}
+            fetchOption="new"
+            disabled={false}
+            interval={DEFAULT_POLLING}
+        />
+    );
 }
 
 export default WaitingForNewLogsRow;

src/components/tables/Logs/shared.ts

@@ -8,9 +8,12 @@ export const maxBytes = Math.round(MEGABYTE / 10);
 //  row should render expanded
 export const DEFAULT_ROW_HEIGHT = 55;
 export const DEFAULT_ROW_HEIGHT_WITHOUT_FIELDS = 35;
+export const WAITING_ROW_HEIGHT = 37; // This is just a bit taller due to the spinner used
 
 export const UUID_START_OF_LOGS = 'UI-start-of-logs';
 export const UUID_OLDEST_LOG = 'UI-oldest-log-line';
 export const UUID_NEWEST_LOG = 'UI-newest-log-line';
 
 export const EXPAND_ROW_TRANSITION = 200;
+
+export const VIRTUAL_TABLE_BODY_PADDING = 10;

src/components/tables/Logs/types.ts

@@ -1,4 +1,4 @@
-import { LoadDocumentsOffsets } from 'hooks/journals/shared';
+import { LoadDocumentsOffsets } from 'hooks/journals/types';
 import { CSSProperties, RefCallback } from 'react';
 
 export type FetchMoreLogsOptions = 'old' | 'new';
@@ -10,3 +10,11 @@ export interface WaitingForRowProps {
 }
 
 export type RefreshLogsFunction = (newOffset?: LoadDocumentsOffsets) => void;
+
+export type LogLevels =
+    | 'error'
+    | 'warn'
+    | 'debug'
+    | 'trace'
+    | 'done'
+    | 'ui_waiting';