Draft: Add auto-generated plugins to import data to Splitgraph from all supported data sources (Airbyte, Singer Taps, etc), and add export plugin to export from Splitgraph to Seafowl via Splitgraph GraphQL API

CONTRIBUTING.md

@@ -42,6 +42,11 @@ VITE_TEST_INTEGRATION=1
 VITE_TEST_DDN_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 VITE_TEST_DDN_API_SECRET=yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
 VITE_TEST_SEAFOWL_SECRET=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+VITE_TEST_GITHUB_PAT=uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
+VITE_TEST_SEAFOWL_EXPORT_DEST_URL=https://demo.seafowl.cloud
+# should match the username associated with the API_KEY for Splitgraph
+VITE_TEST_SEAFOWL_EXPORT_DEST_DBNAME=miles
+VITE_TEST_SEAFOWL_EXPORT_DEST_SECRET=tttttttttttttttttttttttttttttttt
 ```
 
 Then simply append `--mode integration` flag to any variant of `yarn test` that
@@ -68,6 +73,24 @@ where a separate process like `mitmproxy` can intercept outbound requests:
 yarn test-mitm --mode integration
 ```
 
+### Run tests in VSCode "JavaScript Debug Terminal"
+
+If using VSCode, the easiest debugging method is to open a "JavaScript Debug
+Terminal" (which you can do via the command palette
+"`Debug: JavaScript Debug Terminal`"). Put a `debugger;` statement where you
+want to break, and then run vitest in single-threaded mode:
+
+```bash
+yarn test --single-thread
+```
+
+Or, if you also want to use mitmproxy (assumed to be listening on port `7979`),
+then:
+
+```bash
+yarn test-mitm --single-thread
+```
+
 ### Typecheck
 
 We use `tsc` for typechecking, with the default solution file `tsconfig.json`
@@ -160,6 +183,9 @@ defined, in topological order. Therefore, to indicate a workspace is
 publishable, make sure the `package.json` includes `scripts.version` and
 `scripts.publish`.
 
+If you want to publish a prerelease ("tag"), then add `--tag` to the
+`publish-all` command, e.g. `yarn publish-all --tag canary --otp <your otp>`
+
 Create deferred patch (0.0.x) changes (if necessary) in topological order
 
 (NOTE: To update all versions immediately, change `-d` to `-i`, and then there

packages/base-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@madatdata/base-client",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "packageManager": "yarn@3.2.0",
   "main": "index.ts",
   "types": "./build/es2020/index.d.ts",

packages/base-db/base-db.ts

@@ -21,22 +21,69 @@ import {
 
 import type { HTTPStrategies } from "@madatdata/client-http";
 
-export interface ImportPlugin extends Plugin {
+export interface ImportPlugin<
+  PluginName extends string,
+  ConcreteSourceOptions extends object = any,
+  ConcreteDestOptions extends object = any
+> extends Plugin {
+  __name: PluginName;
   importData: (
-    sourceOptions: any,
-    destOptions: any
-  ) => Promise<{ response: any | null; error: any | null; info?: any | null }>;
+    sourceOptions: ConcreteSourceOptions,
+    destOptions: ConcreteDestOptions,
+    importOptions?: { defer: boolean }
+  ) => Promise<{
+    taskId?: string | null;
+    response: any | null;
+    error: any | null;
+    info?: any | null;
+  }>;
 }
 
 // interface ImportPluginWithOptions extends ImportPlugin {
 //   withOptions: WithOptions<ImportPlugin>;
 // }
 
-export interface ExportPlugin extends Plugin {
+export interface ExportPlugin<
+  PluginName extends string,
+  ConcreteSourceOptions extends object = any,
+  ConcreteDestOptions extends object = any
+> extends Plugin {
+  __name: PluginName;
   exportData: (
-    sourceOptions: any,
-    destOptions: any
-  ) => Promise<{ response: any | null; error: any | null; info?: any | null }>;
+    sourceOptions: ConcreteSourceOptions,
+    destOptions: ConcreteDestOptions,
+    exportOptions?: { defer: boolean }
+  ) => Promise<{
+    response: any | null;
+    error: any | null;
+    info?: any | null;
+    taskId?: string | null;
+    taskIds?: any;
+  }>;
+}
+
+export interface DeferredTaskPlugin<
+  PluginName extends string,
+  // TODO: maybe should extend { __name: PluginName }  to ensure finding
+  // correct task plugin by serialized deferred task
+
+  DeferredTaskResponse extends {
+    completed: boolean;
+    response: any | null;
+    error: any | null;
+    info: any | null;
+  } = {
+    completed: boolean;
+    response: Record<string, unknown>;
+    error: any;
+    info: any;
+  },
+  MemoizedDeferredTask extends object = any
+> extends Plugin {
+  __name: PluginName;
+  pollDeferredTask: (
+    memoizedDeferredTask: MemoizedDeferredTask
+  ) => Promise<DeferredTaskResponse>;
 }
 
 // interface ExportPluginWithOptions extends ExportPlugin {
@@ -51,6 +98,13 @@ export interface DbPluggableInterface<ConcretePluginList extends PluginList>
   exportData: <MatchingPlugin extends ExportPluginFromList<ConcretePluginList>>(
     ...exportDataArgsForPlugin: Parameters<MatchingPlugin["exportData"]>
   ) => Promise<unknown>;
+  pollDeferredTask: <
+    MatchingPlugin extends DeferredTaskPluginFromList<ConcretePluginList>
+  >(
+    ...pollDeferredTaskArgsForPlugin: Parameters<
+      MatchingPlugin["pollDeferredTask"]
+    >
+  ) => Promise<unknown>;
 }
 
 export interface Db<ConcretePluginList extends PluginList> {
@@ -62,6 +116,12 @@ export interface Db<ConcretePluginList extends PluginList> {
     pluginName: MatchingPlugin["__name"],
     ...rest: Parameters<MatchingPlugin["exportData"]>
   ) => Promise<unknown>;
+  pollDeferredTask: <
+    MatchingPlugin extends DeferredTaskPluginFromList<ConcretePluginList>
+  >(
+    pluginName: MatchingPlugin["__name"],
+    ...rest: Parameters<MatchingPlugin["pollDeferredTask"]>
+  ) => Promise<unknown>;
   makeClient: <ImplementationSpecificClientOptions extends ClientOptions>(
     makeClientForProtocol: (
       wrappedOptions: ImplementationSpecificClientOptions
@@ -74,17 +134,25 @@ export type ImportPluginFromList<
   ConcretePluginList extends PluginList,
   PluginName extends ExtractPlugin<
     ConcretePluginList,
-    ImportPlugin
+    ImportPlugin<string>
   >["__name"] = string
-> = ExtractPlugin<ConcretePluginList, ImportPlugin & { __name: PluginName }>;
+> = ExtractPlugin<ConcretePluginList, ImportPlugin<PluginName>>;
 
 export type ExportPluginFromList<
   ConcretePluginList extends PluginList,
   PluginName extends ExtractPlugin<
     ConcretePluginList,
-    ExportPlugin
+    ExportPlugin<string>
   >["__name"] = string
-> = ExtractPlugin<ConcretePluginList, ExportPlugin & { __name: PluginName }>;
+> = ExtractPlugin<ConcretePluginList, ExportPlugin<PluginName>>;
+
+export type DeferredTaskPluginFromList<
+  ConcretePluginList extends PluginList,
+  PluginName extends ExtractPlugin<
+    ConcretePluginList,
+    DeferredTaskPlugin<string>
+  >["__name"] = string
+> = ExtractPlugin<ConcretePluginList, DeferredTaskPlugin<PluginName>>;
 
 export interface DbOptions<ConcretePluginList extends PluginList> {
   plugins: ConcretePluginList;
@@ -173,6 +241,20 @@ export abstract class BaseDb<
     ...rest: Parameters<MatchingPlugin["exportData"]>
   ): Promise<unknown>;
 
+  abstract pollDeferredTask<
+    PluginName extends DeferredTaskPluginFromList<
+      ConcretePluginList,
+      string
+    >["__name"],
+    MatchingPlugin extends DeferredTaskPluginFromList<
+      ConcretePluginList,
+      PluginName
+    >
+  >(
+    pluginName: PluginName,
+    ...rest: Parameters<MatchingPlugin["pollDeferredTask"]>
+  ): Promise<unknown>;
+
   /**
    * Return a fingerprint and normalized query (used as input to the fingerprint)
    * for a given SQL string. Default to SHA-256 and normalizing for HTTP headers.
@@ -187,7 +269,7 @@ export abstract class BaseDb<
     // In vitest, really JSDOM, it's a bit of a mix between the two (window is available?)
     // NOTE: Need to test how this will work in a browser bundle which we don't even have yet
     const subtle = await (async () => {
-      if (!window?.crypto?.subtle) {
+      if (typeof window === "undefined" || !window?.crypto?.subtle) {
         const { webcrypto } = await import("crypto");
 
         if (webcrypto.subtle) {

packages/base-db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@madatdata/base-db",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "packageManager": "yarn@3.2.0",
   "main": "index.ts",
   "types": "./build/es2020/index.d.ts",

packages/client-http/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@madatdata/client-http",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "packageManager": "yarn@3.2.0",
   "main": "index.ts",
   "types": "./build/es2020/index.d.ts",

packages/client-postgres/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@madatdata/client-postgres",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "packageManager": "yarn@3.2.0",
   "main": "index.ts",
   "types": "./build/es2020/index.d.ts",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@madatdata/core",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "packageManager": "yarn@3.2.0",
   "main": "index.ts",
   "types": "./build/es2020/index.d.ts",

packages/core/splitgraph-seafowl-sync.test.ts

@@ -16,7 +16,7 @@ describe.skipIf(shouldSkipSeafowlTests() || shouldSkipIntegrationTests())(
       const seafowl = createRealSeafowlDataContext();
 
       const { response } = await splitgraph.db.exportData(
-        "exportQuery",
+        "export-query-to-file",
         {
           query: `SELECT a as int_val, string_agg(random()::text, '') as text_val
 FROM generate_series(1, 5) a, generate_series(1, 50) b

packages/core/splitgraph.test.ts

@@ -291,7 +291,7 @@ describe("makeSplitgraphHTTPContext", () => {
               },
             },
             "plugins": [
-              _SplitgraphImportCSVPlugin {
+              SplitgraphImportCSVPlugin {
                 "__name": "csv",
                 "graphqlClient": SplitgraphGraphQLClient {
                   "graphqlClient": GraphQLClient {
@@ -310,8 +310,46 @@ describe("makeSplitgraphHTTPContext", () => {
                 },
                 "transformRequestHeaders": [Function],
               },
-              _ExportQueryPlugin {
-                "__name": "exportQuery",
+              SplitgraphGeneratedImportPlugin {
+                "__name": "airbyte-github",
+                "graphqlClient": SplitgraphGraphQLClient {
+                  "graphqlClient": GraphQLClient {
+                    "options": {
+                      "headers": [Function],
+                    },
+                    "url": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  },
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                "opts": {
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "transformRequestHeaders": [Function],
+              },
+              SplitgraphExportQueryToFilePlugin {
+                "__name": "export-query-to-file",
+                "graphqlClient": SplitgraphGraphQLClient {
+                  "graphqlClient": GraphQLClient {
+                    "options": {
+                      "headers": [Function],
+                    },
+                    "url": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  },
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                "opts": {
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "transformRequestHeaders": [Function],
+              },
+              SplitgraphExportToSeafowlPlugin {
+                "__name": "export-to-seafowl",
                 "graphqlClient": SplitgraphGraphQLClient {
                   "graphqlClient": GraphQLClient {
                     "options": {
@@ -334,7 +372,7 @@ describe("makeSplitgraphHTTPContext", () => {
           "plugins": PluginRegistry {
             "hostContext": {},
             "plugins": [
-              _SplitgraphImportCSVPlugin {
+              SplitgraphImportCSVPlugin {
                 "__name": "csv",
                 "graphqlClient": SplitgraphGraphQLClient {
                   "graphqlClient": GraphQLClient {
@@ -353,8 +391,46 @@ describe("makeSplitgraphHTTPContext", () => {
                 },
                 "transformRequestHeaders": [Function],
               },
-              _ExportQueryPlugin {
-                "__name": "exportQuery",
+              SplitgraphGeneratedImportPlugin {
+                "__name": "airbyte-github",
+                "graphqlClient": SplitgraphGraphQLClient {
+                  "graphqlClient": GraphQLClient {
+                    "options": {
+                      "headers": [Function],
+                    },
+                    "url": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  },
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                "opts": {
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "transformRequestHeaders": [Function],
+              },
+              SplitgraphExportQueryToFilePlugin {
+                "__name": "export-query-to-file",
+                "graphqlClient": SplitgraphGraphQLClient {
+                  "graphqlClient": GraphQLClient {
+                    "options": {
+                      "headers": [Function],
+                    },
+                    "url": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  },
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                "opts": {
+                  "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql",
+                  "transformRequestHeaders": [Function],
+                },
+                "transformRequestHeaders": [Function],
+              },
+              SplitgraphExportToSeafowlPlugin {
+                "__name": "export-to-seafowl",
                 "graphqlClient": SplitgraphGraphQLClient {
                   "graphqlClient": GraphQLClient {
                     "options": {