Merge branch 'main' into phil/evolve-backfill

src/api/connectors.ts

@@ -1,16 +1,16 @@
 import {
-    ConnectorWithTagDetailQuery,
     CONNECTOR_WITH_TAG_QUERY,
+    ConnectorWithTagDetailQuery,
 } from 'hooks/useConnectorWithTagDetail';
 import {
     CONNECTOR_NAME,
     CONNECTOR_RECOMMENDED,
+    TABLES,
     defaultTableFilter,
     handleFailure,
     handleSuccess,
     supabaseClient,
     supabaseRetry,
-    TABLES,
 } from 'services/supabase';
 import { SortDirection } from 'types';
 
@@ -46,15 +46,17 @@ const getConnectors = (
 };
 
 // Hydration-specific queries
+export interface ConnectorTag_Base {
+    id: string;
+    connector_id: string;
+    image_tag: string;
+}
+
 export interface ConnectorsQuery_DetailsForm {
     id: string;
     image_name: string;
     logo_url: string;
-    connector_tags: {
-        id: string;
-        connector_id: string;
-        image_tag: string;
-    }[];
+    connector_tags: ConnectorTag_Base[];
 }
 
 const DETAILS_FORM_QUERY = `

src/app/routes.ts

@@ -123,11 +123,12 @@ const collections = {
             path: 'history',
             fullPath: '/collections/details/history',
         },
-        ops: {
-            title: 'routeTitle.collectionDetails.ops',
-            path: 'ops',
-            fullPath: '/collections/details/ops',
-        },
+        // TODO (ops logs) need to handle collections (really derivations)
+        // ops: {
+        //     title: 'routeTitle.collectionDetails.ops',
+        //     path: 'ops',
+        //     fullPath: '/collections/details/ops',
+        // },
     },
 };
 

src/components/collection/DataPreview/index.tsx

@@ -53,7 +53,9 @@ export function DataPreview({ collectionName }: Props) {
     // TODO (typing) we need to fix typing
     // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
     const journal = useMemo(() => journalsData?.journals?.[0], [journalsData]);
-    const journalData = useJournalData(journal?.name, 20, collectionName);
+    const journalData = useJournalData(journal?.name, collectionName, {
+        desiredCount: 20,
+    });
 
     // There is a brief delay between when the data preview card is rendered and the two journal-related
     // hooks are called, which resulted in `isLoading` being a false negative. If the journal client is
@@ -78,7 +80,7 @@ export function DataPreview({ collectionName }: Props) {
                     <Button
                         variant="text"
                         startIcon={<Refresh style={{ fontSize: 12 }} />}
-                        onClick={journalData.refresh}
+                        onClick={() => journalData.refresh()}
                         disabled={
                             isLoading || !hasLength(journalData.data?.documents)
                         }

src/components/shared/Entity/Details/Ops/index.tsx

@@ -1,34 +1,114 @@
-import { Box, Button, LinearProgress, Stack } from '@mui/material';
+import { Box, Button, Stack } from '@mui/material';
+import KeyValueList from 'components/shared/KeyValueList';
 import UnderDev from 'components/shared/UnderDev';
 import LogsTable from 'components/tables/Logs';
 import { useJournalData } from 'hooks/journals/useJournalData';
 import useJournalNameForLogs from 'hooks/journals/useJournalNameForLogs';
 import useGlobalSearchParams, {
     GlobalSearchParams,
 } from 'hooks/searchParams/useGlobalSearchParams';
+import { useEffect, useMemo, useState } from 'react';
 import { OpsLogFlowDocument } from 'types';
+import { MEGABYTE } from 'utils/dataPlane-utils';
 
-const docsRequested = 25;
+const maxBytes = Math.round(MEGABYTE / 25);
 
 function Ops() {
+    const [fetchingMore, setFetchingMore] = useState(false);
+    const [olderFinished, setOlderFinished] = useState(false);
+    const [lastParsed, setLastParsed] = useState<number>(0);
+    const [docs, setDocs] = useState<OpsLogFlowDocument[]>([]);
+
     const catalogName = useGlobalSearchParams(GlobalSearchParams.CATALOG_NAME);
     const [name, collectionName] = useJournalNameForLogs(catalogName);
 
     // TODO (typing)
     //  need to handle typing
-    const journalData = useJournalData(name, docsRequested, collectionName);
-    const documents = (journalData.data?.documents ??
-        []) as OpsLogFlowDocument[];
+    const { data, loading, refresh } = useJournalData(name, collectionName, {
+        maxBytes,
+    });
+
+    const documents = useMemo(
+        () => (data?.documents ?? []) as OpsLogFlowDocument[],
+        [data?.documents]
+    );
+
+    useEffect(() => {
+        // Get the mete data out of the response
+        const meta = data?.meta;
+
+        // Figure out what the last document offset is
+        const parsedEnd = meta?.docsMetaResponse.offset
+            ? parseInt(meta.docsMetaResponse.offset, 10)
+            : null;
+
+        // Since journalData is read kinda async we need to wait to
+        //  update documents until we know the meta data changed
+        if (parsedEnd !== lastParsed) {
+            if (documents.length > 0) {
+                const newDocs = [...documents, ...docs];
+                setDocs(newDocs);
+                setFetchingMore(false);
+            }
+        }
+    }, [data?.meta, docs, documents, lastParsed]);
+
+    useEffect(() => {
+        // Get the mete data out of the response
+        const meta = data?.meta;
+
+        // Figure out what the last document offset is
+        const parsedEnd = meta?.docsMetaResponse.offset
+            ? parseInt(meta.docsMetaResponse.offset, 10)
+            : null;
+
+        // Keep track of where we last read data from so we can keep stepping backwards through the file
+        setLastParsed(parsedEnd ?? 0);
+
+        // If we have hit 0 then we now we hit the start of the data are nothing older is available
+        if (parsedEnd === 0) {
+            setOlderFinished(true);
+        }
+    }, [data?.meta]);
 
     return (
         <Box>
             <UnderDev />
             <Box>
-                <Button onClick={journalData.refresh}>Refresh</Button>
+                <KeyValueList
+                    sectionTitle="Debugging Values"
+                    data={[
+                        { title: 'Documents', val: docs.length },
+                        { title: 'Last Byte Parsed', val: lastParsed },
+                    ]}
+                />
 
-                <Stack>
-                    <Box>Documents {journalData.data?.documents.length}</Box>
+                <Stack spacing={2} direction="row">
+                    <Button
+                        disabled={loading || fetchingMore || olderFinished}
+                        onClick={() => {
+                            setFetchingMore(true);
+                            refresh({
+                                offset: 0,
+                                endOffset: lastParsed,
+                            });
+                        }}
+                    >
+                        Load Older
+                    </Button>
 
+                    <Button
+                        disabled={loading || fetchingMore}
+                        onClick={() => {
+                            setFetchingMore(true);
+                            refresh();
+                        }}
+                    >
+                        Load Newer
+                    </Button>
+                </Stack>
+
+                <Stack spacing={2}>
                     {/*                    <JournalAlerts
                         journalData={journalData}
                         notFoundTitleMessage={
@@ -41,9 +121,26 @@ function Ops() {
                         }
                     />*/}
 
-                    {journalData.loading ? <LinearProgress /> : null}
+                    <LogsTable
+                        documents={docs}
+                        loading={fetchingMore || loading}
+                        fetchNewer={() => {
+                            console.log('fetcher latest logs');
+
+                            // setLoading(true);
+                            // setTimeout(() => setLoading(false), 2500);
+                        }}
+                        fetchOlder={
+                            olderFinished
+                                ? undefined
+                                : () => {
+                                      console.log('fetch older logs');
 
-                    <LogsTable documents={documents} />
+                                      // setLoading(true);
+                                      // setTimeout(() => setLoading(false), 2500);
+                                  }
+                        }
+                    />
                 </Stack>
             </Box>
         </Box>

src/components/shared/Entity/Details/Overview/index.tsx

@@ -9,6 +9,7 @@ import useGlobalSearchParams, {
 } from 'hooks/searchParams/useGlobalSearchParams';
 import { useLiveSpecs_details } from 'hooks/useLiveSpecs';
 import { useMemo } from 'react';
+import { ShardEntityTypes } from 'stores/ShardDetail/types';
 import { hasLength, specContainsDerivation } from 'utils/misc-utils';
 import ShardInformation from '../../Shard/Information';
 import Usage from '../Usage';
@@ -40,6 +41,14 @@ function Overview({ name }: Props) {
         [liveSpecs, validatingLiveSpecs]
     );
 
+    const taskTypes: ShardEntityTypes[] = useMemo(() => {
+        if (!isCollection || isDerivation) {
+            return isDerivation ? ['derivation'] : [entityType];
+        }
+
+        return [];
+    }, [entityType, isCollection, isDerivation]);
+
     return (
         <Grid container spacing={2}>
             <Grid item xs={12} md={8} lg={9}>
@@ -67,10 +76,10 @@ function Overview({ name }: Props) {
                 </Grid>
             ) : null}
 
-            {!isCollection || isDerivation ? (
+            {hasLength(taskTypes) ? (
                 <Grid item xs={12}>
                     <ShardInformation
-                        taskTypes={[entityType]}
+                        taskTypes={taskTypes}
                         taskName={entityName}
                     />
                 </Grid>

src/components/shared/Entity/DetailsForm/Form.tsx

@@ -17,6 +17,8 @@ import { FormattedMessage, useIntl } from 'react-intl';
 import defaultRenderers from 'services/jsonforms/defaultRenderers';
 import { defaultOptions, showValidation } from 'services/jsonforms/shared';
 import {
+    useDetailsForm_changed_connectorId,
+    useDetailsForm_connectorImage_imagePath,
     useDetailsForm_details,
     useDetailsForm_setDetails,
     useDetailsForm_setDetails_connector,
@@ -30,17 +32,24 @@ import {
     useFormStateStore_messagePrefix,
 } from 'stores/FormState/hooks';
 import { hasLength } from 'utils/misc-utils';
+import {
+    ConnectorVersionEvaluationOptions,
+    evaluateConnectorVersions,
+} from 'utils/workflow-utils';
 
 export const CONFIG_EDITOR_ID = 'endpointConfigEditor';
 
 export const getConnectorImageDetails = (
-    connector: ConnectorWithTagDetailQuery
+    connector: ConnectorWithTagDetailQuery,
+    options?: { connectorId: string; existingImageTag: string }
 ): Details['data']['connectorImage'] => {
+    const connectorTag = evaluateConnectorVersions(connector, options);
+
     return {
         connectorId: connector.id,
-        id: connector.connector_tags[0].id,
+        id: connectorTag.id,
         imageName: connector.image_name,
-        imagePath: `${connector.image_name}${connector.connector_tags[0].image_tag}`,
+        imagePath: `${connector.image_name}${connectorTag.image_tag}`,
         iconPath: connector.image,
     };
 };
@@ -54,6 +63,9 @@ function DetailsFormForm({ connectorTags, entityType, readOnly }: Props) {
     const formData = useDetailsForm_details();
     const { connectorImage: originalConnectorImage } = formData;
 
+    const connectorImagePath = useDetailsForm_connectorImage_imagePath();
+    const connectorIdChanged = useDetailsForm_changed_connectorId();
+
     const setDetails = useDetailsForm_setDetails();
     const setDetails_connector = useDetailsForm_setDetails_connector();
     const setEntityNameChanged = useDetailsForm_setEntityNameChanged();
@@ -70,33 +82,55 @@ function DetailsFormForm({ connectorTags, entityType, readOnly }: Props) {
     const isEdit = useEntityWorkflow_Editing();
 
     useEffect(() => {
-        if (connectorId && hasLength(connectorTags)) {
+        if (connectorId && hasLength(connectorTags) && connectorIdChanged) {
             connectorTags.find((connector) => {
-                const response =
-                    connector.connector_tags[0].connector_id === connectorId;
+                const connectorTag = evaluateConnectorVersions(connector);
+
+                const connectorLocated =
+                    connectorTag.connector_id === connectorId;
 
-                if (response) {
+                if (connectorLocated) {
                     setDetails_connector(getConnectorImageDetails(connector));
                 }
-                return response;
+
+                return connectorLocated;
             });
         }
-    }, [setDetails_connector, connectorId, connectorTags]);
+    }, [setDetails_connector, connectorId, connectorIdChanged, connectorTags]);
+
+    const versionEvaluationOptions:
+        | ConnectorVersionEvaluationOptions
+        | undefined = useMemo(() => {
+        const imageTagStartIndex = connectorImagePath.indexOf(':');
+
+        return isEdit && hasLength(connectorId) && imageTagStartIndex > 0
+            ? {
+                  connectorId,
+                  existingImageTag: connectorImagePath.substring(
+                      imageTagStartIndex,
+                      connectorImagePath.length
+                  ),
+              }
+            : undefined;
+    }, [connectorId, connectorImagePath, isEdit]);
 
     const connectorsOneOf = useMemo(() => {
         const response = [] as { title: string; const: Object }[];
 
         if (connectorTags.length > 0) {
             connectorTags.forEach((connector) => {
                 response.push({
-                    const: getConnectorImageDetails(connector),
+                    const: getConnectorImageDetails(
+                        connector,
+                        versionEvaluationOptions
+                    ),
                     title: connector.title,
                 });
             });
         }
 
         return response;
-    }, [connectorTags]);
+    }, [connectorTags, versionEvaluationOptions]);
 
     const schema = useMemo(() => {
         return {