diff --git a/webapps/console/components/ConfigObjectEditor/ConfigEditor.tsx b/webapps/console/components/ConfigObjectEditor/ConfigEditor.tsx index 367c2b820..8d5bc4492 100644 --- a/webapps/console/components/ConfigObjectEditor/ConfigEditor.tsx +++ b/webapps/console/components/ConfigObjectEditor/ConfigEditor.tsx @@ -95,6 +95,7 @@ export type ConfigEditorProps //for providing custom editor component editorComponent?: EditorComponentFactory; testConnectionEnabled?: (o: any) => boolean; + testButtonLabel?: string; onTest?: (o: T) => Promise; backTo?: string; }; @@ -326,6 +327,7 @@ const EditorComponent: React.FC = props => { isNew={isNew} isTouched={isTouched} hasErrors={hasErrors} + testButtonLabel={props.testButtonLabel} onTest={ onTest && testConnectionEnabled && testConnectionEnabled(formState?.formData || object) ? async () => { diff --git a/webapps/console/components/ConfigObjectEditor/EditorButtons.tsx b/webapps/console/components/ConfigObjectEditor/EditorButtons.tsx index 62820378a..af8faa46b 100644 --- a/webapps/console/components/ConfigObjectEditor/EditorButtons.tsx +++ b/webapps/console/components/ConfigObjectEditor/EditorButtons.tsx @@ -17,6 +17,7 @@ export type EditorButtonProps = { isTouched?: boolean; hasErrors?: boolean; testStatus?: string; + testButtonLabel?: string; }; export const EditorButtons: React.FC = ({ @@ -28,6 +29,7 @@ export const EditorButtons: React.FC = ({ onSave, isTouched, hasErrors, + testButtonLabel = "Test Connection", }) => { const buttonDivRef = useRef(null); const appConfig = useAppConfig(); @@ -93,17 +95,19 @@ export const EditorButtons: React.FC = ({ (testStatus === "success" ? ( ) : ( ))} diff --git a/webapps/console/components/PageLayout/WorkspacePageLayout.tsx b/webapps/console/components/PageLayout/WorkspacePageLayout.tsx index eb1cf69b1..9aba212f3 100644 --- a/webapps/console/components/PageLayout/WorkspacePageLayout.tsx +++ b/webapps/console/components/PageLayout/WorkspacePageLayout.tsx @@ -1,7 +1,7 @@ import React, { PropsWithChildren, ReactNode, useEffect, useState } from "react"; import { branding } from "../../lib/branding"; import { HiSelector } from "react-icons/hi"; -import { FaSignOutAlt, FaUserCircle } from "react-icons/fa"; +import { FaDocker, FaSignOutAlt, FaUserCircle } from "react-icons/fa"; import { FiSettings } from "react-icons/fi"; import { Button, Drawer, Dropdown, Menu, MenuProps } from "antd"; import MenuItem from "antd/lib/menu/MenuItem"; @@ -582,6 +582,7 @@ function PageHeader() { { title: "Service Connections", path: "/services", icon: }, { title: "Syncs", path: "/syncs", icon: }, { title: "All Logs", path: "/syncs/tasks", icon: }, + { title: "Custom Images", path: "/custom-images", icon: }, ], }, appConfig.ee?.available && { diff --git a/webapps/console/components/ServicesCatalog/ServicesCatalog.tsx b/webapps/console/components/ServicesCatalog/ServicesCatalog.tsx index f677cb328..2967825ab 100644 --- a/webapps/console/components/ServicesCatalog/ServicesCatalog.tsx +++ b/webapps/console/components/ServicesCatalog/ServicesCatalog.tsx @@ -7,13 +7,14 @@ import capitalize from "lodash/capitalize"; import { LoadingAnimation } from "../GlobalLoader/GlobalLoader"; import React from "react"; import { ErrorCard } from "../GlobalError/GlobalError"; -import { Button, Input, Popover } from "antd"; +import { Input } from "antd"; import { useAppConfig, useWorkspace } from "../../lib/context"; +import { useConfigObjectList } from "../../lib/store"; function groupByType(sources: SourceType[]): Record { const groups: Record = {}; const otherGroup = "other"; - const sortOrder = ["Datawarehouse", "Product Analytics", "CRM", "Block Storage"]; + const sortOrder = ["api", "database", "file", "custom image"]; sources.forEach(s => { if (s.packageId.endsWith("strict-encrypt") || s.packageId === "airbyte/source-file-secure") { @@ -58,11 +59,10 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin onClick, }) => { const { data, isLoading, error } = useApi<{ sources: SourceType[] }>(`/api/sources?mode=meta`); + const customImages = useConfigObjectList("custom-image"); const sourcesIconsLoader = useApi<{ sources: SourceType[] }>(`/api/sources?mode=icons-only`); const workspace = useWorkspace(); const [filter, setFilter] = React.useState(""); - const [customImage, setCustomImage] = React.useState(""); - const [customPopupOpen, setCustomPopupOpen] = React.useState(false); const appconfig = useAppConfig(); const sourcesIcons: Record = sourcesIconsLoader.data ? sourcesIconsLoader.data.sources.reduce( @@ -79,7 +79,17 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin } else if (error) { return ; } - const groups = groupByType(data.sources); + const sources = [ + ...data.sources, + ...customImages.map(c => ({ + id: c.package, + packageId: c.package, + packageType: "airbyte", + meta: { name: c.name, connectorSubtype: "custom image", dockerImageTag: c.version }, + })), + ] as SourceType[]; + + const groups = groupByType(sources); return (
@@ -123,14 +133,23 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
onClick(source.packageType, source.packageId)} + onClick={() => + onClick( + source.packageType, + source.packageId, + source.meta?.connectorSubtype === "custom image" ? source.meta?.dockerImageTag : undefined + ) + } >
{getServiceIcon(source, sourcesIcons)}
{source.meta.name}
-
{source.packageId}
+
+ {source.packageId} + {source.meta?.connectorSubtype === "custom image" ? ":" + source.meta?.dockerImageTag : ""} +
); @@ -139,52 +158,6 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
); })} -
-
Advanced
-
- - setCustomImage(e.target.value)} /> - -
- } - onOpenChange={setCustomPopupOpen} - open={customPopupOpen} - title="Enter docker image name" - placement={"right"} - trigger="click" - > -
-
- -
-
-
-
Custom connector
-
-
Custom docker image
-
-
- -
-
); diff --git a/webapps/console/lib/schema/config-objects.ts b/webapps/console/lib/schema/config-objects.ts index c6dd624cc..2c4d7f8a0 100644 --- a/webapps/console/lib/schema/config-objects.ts +++ b/webapps/console/lib/schema/config-objects.ts @@ -1,7 +1,15 @@ import { coreDestinationsMap } from "./destinations"; import { safeParseWithDate } from "../zod"; import { ApiError } from "../shared/errors"; -import { ApiKey, ConfigObjectType, DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "./index"; +import { + ApiKey, + ConfigObjectType, + ConnectorImageConfig, + DestinationConfig, + FunctionConfig, + ServiceConfig, + StreamConfig, +} from "./index"; import { assertDefined, createHash, requireDefined } from "juava"; import { checkOrAddToIngress, isDomainAvailable } from "../server/custom-domains"; import { ZodType, ZodTypeDef } from "zod"; @@ -162,4 +170,7 @@ const configObjectTypes: Record = { service: { schema: ServiceConfig, }, + "custom-image": { + schema: ConnectorImageConfig, + }, } as const; diff --git a/webapps/console/lib/schema/index.ts b/webapps/console/lib/schema/index.ts index 1480f2dd4..0ab8b085c 100644 --- a/webapps/console/lib/schema/index.ts +++ b/webapps/console/lib/schema/index.ts @@ -170,6 +170,15 @@ export const ServiceConfig = ConfigEntityBase.merge( ); export type ServiceConfig = z.infer; +export const ConnectorImageConfig = ConfigEntityBase.merge( + z.object({ + name: z.string(), + package: z.string(), + version: z.string(), + }) +); +export type ConnectorImageConfig = z.infer; + /** * What happens to an object before it is saved to DB. * diff --git a/webapps/console/lib/store/index.tsx b/webapps/console/lib/store/index.tsx index 905df8402..3a2e848de 100644 --- a/webapps/console/lib/store/index.tsx +++ b/webapps/console/lib/store/index.tsx @@ -1,4 +1,4 @@ -import type { DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "../schema"; +import type { ConnectorImageConfig, DestinationConfig, FunctionConfig, ServiceConfig, StreamConfig } from "../schema"; import { useCallback, useEffect, useMemo, useState } from "react"; import { getLog, requireDefined, rpc } from "juava"; import { useWorkspace } from "../context"; @@ -7,7 +7,7 @@ import { z } from "zod"; import { ConfigurationObjectLinkDbModel, ProfileBuilderDbModel, WorkspaceDbModel } from "../../prisma/schema"; import { UseMutationResult } from "@tanstack/react-query/src/types"; -export const allConfigTypes = ["stream", "service", "function", "destination"] as const; +export const allConfigTypes = ["stream", "service", "function", "destination", "custom-image"] as const; export type ConfigType = (typeof allConfigTypes)[number]; @@ -16,6 +16,7 @@ export type ConfigTypes = { service: ServiceConfig; function: FunctionConfig; destination: DestinationConfig; + "custom-image": ConnectorImageConfig; }; export function asConfigType(type: string): ConfigType { diff --git a/webapps/console/pages/[workspaceId]/custom-images.tsx b/webapps/console/pages/[workspaceId]/custom-images.tsx new file mode 100644 index 000000000..0ff23b8f4 --- /dev/null +++ b/webapps/console/pages/[workspaceId]/custom-images.tsx @@ -0,0 +1,133 @@ +import { WorkspacePageLayout } from "../../components/PageLayout/WorkspacePageLayout"; +import { ConfigEditor, ConfigEditorProps } from "../../components/ConfigObjectEditor/ConfigEditor"; +import { ConnectorImageConfig } from "../../lib/schema"; +import { useAppConfig, useWorkspace } from "../../lib/context"; +import React from "react"; +import { SourceType } from "../api/sources"; +import { ErrorCard } from "../../components/GlobalError/GlobalError"; +import { ServerCog } from "lucide-react"; +import { FaDocker } from "react-icons/fa"; +import { Htmlizer } from "../../components/Htmlizer/Htmlizer"; +import { rpc } from "juava"; +import { UpgradeDialog } from "../../components/Billing/UpgradeDialog"; +import { useBilling } from "../../components/Billing/BillingProvider"; +import { LoadingAnimation } from "../../components/GlobalLoader/GlobalLoader"; + +const CustomImages: React.FC = () => { + return ( + + + + ); +}; + +const CustomImagesList: React.FC<{}> = () => { + const workspace = useWorkspace(); + const appconfig = useAppConfig(); + const billing = useBilling(); + + if (billing.loading) { + return ; + } + if (billing.enabled && billing.settings?.planId === "free") { + return ; + } + + if (!(appconfig.syncs.enabled || workspace.featuresEnabled.includes("syncs"))) { + return ( + + ); + } + + const config: ConfigEditorProps = { + listColumns: [ + { + title: "Package", + render: (s: ConnectorImageConfig) => {`${s.package}:${s.version}`}, + }, + ], + objectType: ConnectorImageConfig, + fields: { + type: { constant: "custom-image" }, + workspaceId: { constant: workspace.id }, + package: { + documentation: ( + + { + "Docker image name. Images can also include a registry hostname, e.g.: fictional.registry.example/imagename, and possibly a port number as well." + } + + ), + }, + version: { + documentation: "Docker image tag", + }, + }, + noun: "custom image", + type: "custom-image", + explanation: "Custom connector images that can be used to setup Service connector", + icon: () => , + testConnectionEnabled: () => true, + testButtonLabel: "Check Image", + onTest: async obj => { + try { + const firstRes = await rpc( + `/api/${workspace.id}/sources/spec?package=${obj.package}&version=${obj.version}&force=true` + ); + if (firstRes.ok) { + return { ok: true }; + } else if (firstRes.error) { + return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${firstRes.error}` }; + } else { + for (let i = 0; i < 60; i++) { + await new Promise(resolve => setTimeout(resolve, 2000)); + const resp = await rpc(`/api/${workspace.id}/sources/spec?package=${obj.package}&version=${obj.version}`); + if (!resp.pending) { + if (resp.error) { + return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${resp.error}` }; + } else { + return { ok: true }; + } + } + } + return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: Timeout` }; + } + } catch (error: any) { + return { ok: false, error: `Cannot load specs for ${obj.package}:${obj.version}: ${error.message}` }; + } + }, + editorTitle: (_: ConnectorImageConfig, isNew: boolean) => { + const verb = isNew ? "New" : "Edit"; + return ( +
+
+ +
+ {verb} custom image +
+ ); + }, + actions: [ + { + icon: , + title: "Setup Connector", + collapsed: false, + link: s => + `/services?id=new&packageType=airbyte&packageId=${encodeURIComponent(s.package)}&version=${encodeURIComponent( + s.version + )}`, + }, + ], + }; + return ( + <> + + + ); +}; + +export default CustomImages; diff --git a/webapps/console/pages/[workspaceId]/services.tsx b/webapps/console/pages/[workspaceId]/services.tsx index 1ac316c19..628195214 100644 --- a/webapps/console/pages/[workspaceId]/services.tsx +++ b/webapps/console/pages/[workspaceId]/services.tsx @@ -151,10 +151,16 @@ const ServicesList: React.FC<{}> = () => { packageType = router.query["packageType"] as string; packageId = router.query["packageId"] as string; } - const rawVersions = await rpc( - `/api/sources/versions?type=${packageType}&package=${encodeURIComponent(packageId)}` - ); - const versions = rawVersions.versions + let rawVersions: any[] = []; + try { + const rv = await rpc(`/api/sources/versions?type=${packageType}&package=${encodeURIComponent(packageId)}`); + rawVersions = rv.versions; + } catch (error: any) { + console.warn( + `Couldn't load package versions from docker repository. Probably ${packageId} is not a docker repository package.` + ); + } + const versions = rawVersions .filter( (v: any) => v.isRelease &&