From 0f64a8177c7d4a7ed93044c06c773ae4eb748d79 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Tue, 23 May 2023 01:26:00 +0100
Subject: [PATCH 01/36] Add stub
`examples/nextjs-import-airbyte-github-export-seafowl/`
---
.../.gitignore | 1 +
.../README.md | 26 +++++++
.../next-env.d.ts | 5 ++
.../package.json | 20 ++++++
.../pages/index.tsx | 67 +++++++++++++++++++
.../tsconfig.json | 20 ++++++
examples/yarn.lock | 17 ++++-
7 files changed, 155 insertions(+), 1 deletion(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/.gitignore
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/README.md
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/package.json
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore
new file mode 100644
index 0000000..a680367
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore
@@ -0,0 +1 @@
+.next
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/README.md b/examples/nextjs-import-airbyte-github-export-seafowl/README.md
new file mode 100644
index 0000000..97e13da
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/README.md
@@ -0,0 +1,26 @@
+# End-to-End Example: Use `airbyte-github` to import GitHub repository into Splitgraph, then export it to Seafowl, via Next.js API routes
+
+This is a full end-to-end example demonstrating importing data to Splitgraph
+(using the `airbyte-github` plugin), exporting it to Seafowl (using the
+`export-to-seafowl` plugin), and then querying it (with `DbSeafowl` and React
+hooks from `@madatdata/react`). The importers and exporting of data is triggered
+by backend API routes (e.g. the Vecel runtime), which execute in an environment
+with secrets (an `API_SECRET` for Splitgraph, and a GitHub access token for
+`airbyte-github`). The client side queries Seafowl directly by sending raw SQL
+queries in HTP requests, which is what Seafowl is ultimately designed for.
+
+## Try Now
+
+### Preview Immediately
+
+_No signup required, just click the button!_
+
+[](https://stackblitz.com/github/splitgraph/madatdata/tree/main/examples/nextjs-import-airbyte-github-export-seafowl?file=pages/index.tsx)
+
+[](https://codesandbox.io/p/sandbox/github/splitgraph/madatdata/main/examples/nextjs-import-airbyte-github-export-seafowl?file=pages/index.tsx&hardReloadOnChange=true&startScript=dev&node=16&port=3000)
+
+### Or, deploy to Vercel (signup required)
+
+_Signup, fork the repo, and import it_
+
+[](https://vercel.com/new/git/external?repository-url=https://github.com/splitgraph/madatdata/tree/main/examples/nextjs-import-airbyte-github-export-seafowl&project-name=madatdata-basic-hooks&repository-name=madatdata-nextjs-basic-hooks)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts
new file mode 100644
index 0000000..4f11a03
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts
@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/basic-features/typescript for more information.
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/package.json b/examples/nextjs-import-airbyte-github-export-seafowl/package.json
new file mode 100644
index 0000000..6d17d38
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/package.json
@@ -0,0 +1,20 @@
+{
+ "private": true,
+ "scripts": {
+ "dev": "yarn next",
+ "build": "yarn next build",
+ "start": "yarn next start"
+ },
+ "dependencies": {
+ "@madatdata/core": "latest",
+ "@madatdata/react": "latest",
+ "next": "latest",
+ "react": "18.2.0",
+ "react-dom": "18.2.0"
+ },
+ "devDependencies": {
+ "@types/node": "^18.0.0",
+ "@types/react": "^18.0.14",
+ "typescript": "^4.7.4"
+ }
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
new file mode 100644
index 0000000..c3a646d
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
@@ -0,0 +1,67 @@
+import {
+ SqlProvider,
+ useSql,
+ makeSplitgraphHTTPContext,
+} from "@madatdata/react";
+import { useMemo } from "react";
+
+const ExampleComponentUsingSQL = () => {
+ const { loading, error, response } = useSql<{
+ origin_airport: string;
+ destination_airport: string;
+ origin_city: string;
+ destination_city: string;
+ passengers: number;
+ seats: number;
+ flights: number;
+ distance: number;
+ fly_month: string;
+ origin_pop: number;
+ destination_pop: number;
+ id: number;
+ }>(
+ `SELECT
+ "origin_airport",
+ "destination_airport",
+ "origin_city",
+ "destination_city",
+ "passengers",
+ "seats",
+ "flights",
+ "distance",
+ "fly_month",
+ "origin_pop",
+ "destination_pop",
+ "id"
+FROM
+ "splitgraph/domestic_us_flights:latest"."flights"
+LIMIT 100;`
+ );
+
+ return (
+ <pre
+ style={{ minWidth: "100%", minHeight: 500 }}
+ data-testid={`result-pre-${
+ loading ? "loading" : response ? "pass" : error ? "fail" : "unknown"
+ }`}
+ >
+ {JSON.stringify({ loading, error, response }, null, 2)}
+ </pre>
+ );
+};
+
+const SplitgraphSampleQuery = () => {
+ const splitgraphDataContext = useMemo(
+ () => makeSplitgraphHTTPContext({ credential: null }),
+ []
+ );
+
+ // Uses splitgraph.com by default (anon access supported for public data)
+ return (
+ <SqlProvider dataContext={splitgraphDataContext}>
+ <ExampleComponentUsingSQL />
+ </SqlProvider>
+ );
+};
+
+export default SplitgraphSampleQuery;
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json
new file mode 100644
index 0000000..16bb209
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json
@@ -0,0 +1,20 @@
+{
+ "compilerOptions": {
+ "target": "es5",
+ "lib": ["dom", "dom.iterable", "esnext"],
+ "allowJs": true,
+ "skipLibCheck": true,
+ "strict": false,
+ "forceConsistentCasingInFileNames": true,
+ "noEmit": true,
+ "incremental": true,
+ "esModuleInterop": true,
+ "module": "esnext",
+ "resolveJsonModule": true,
+ "moduleResolution": "Node",
+ "isolatedModules": true,
+ "jsx": "preserve"
+ },
+ "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+ "exclude": ["node_modules"]
+}
diff --git a/examples/yarn.lock b/examples/yarn.lock
index dfc7d9a..3cbd334 100644
--- a/examples/yarn.lock
+++ b/examples/yarn.lock
@@ -533,7 +533,7 @@ __metadata:
languageName: node
linkType: hard
-"@madatdata/core@npm:0.0.11":
+"@madatdata/core@npm:0.0.11, @madatdata/core@npm:latest":
version: 0.0.11
resolution: "@madatdata/core@npm:0.0.11"
dependencies:
@@ -1894,6 +1894,21 @@ __metadata:
languageName: node
linkType: hard
+"nextjs-import-airbyte-github-export-seafowl-acabed@workspace:nextjs-import-airbyte-github-export-seafowl":
+ version: 0.0.0-use.local
+ resolution: "nextjs-import-airbyte-github-export-seafowl-acabed@workspace:nextjs-import-airbyte-github-export-seafowl"
+ dependencies:
+ "@madatdata/core": latest
+ "@madatdata/react": latest
+ "@types/node": ^18.0.0
+ "@types/react": ^18.0.14
+ next: latest
+ react: 18.2.0
+ react-dom: 18.2.0
+ typescript: ^4.7.4
+ languageName: unknown
+ linkType: soft
+
"node-fetch@npm:2.6.7":
version: 2.6.7
resolution: "node-fetch@npm:2.6.7"
From 776e11f77a486dee5d07fa0217aa8c6da701340c Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Tue, 23 May 2023 20:13:23 +0100
Subject: [PATCH 02/36] Stub out layout and sidebar of GitHub analytics example
Assisted by the one and only GPT-4
---
.../components/BaseLayout.module.css | 37 ++++++++++
.../components/BaseLayout.tsx | 21 ++++++
.../components/Header.module.css | 44 ++++++++++++
.../components/Header.tsx | 25 +++++++
.../components/Logo.tsx | 46 ++++++++++++
.../components/Sidebar.module.css | 54 ++++++++++++++
.../components/Sidebar.tsx | 33 +++++++++
.../components/global-styles/reset.css | 71 +++++++++++++++++++
.../components/global-styles/theme.css | 14 ++++
.../pages/_app.tsx | 7 ++
.../pages/index.tsx | 36 +++++++++-
11 files changed, 387 insertions(+), 1 deletion(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css
new file mode 100644
index 0000000..5cee596
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css
@@ -0,0 +1,37 @@
+.container {
+ display: flex;
+ flex-direction: column;
+ height: 100vh;
+ background-color: var(--background);
+}
+
+.header {
+ width: 100%;
+ position: sticky;
+ top: 0;
+ z-index: 100;
+ background-color: var(--header);
+ color: var(--text);
+}
+
+.main {
+ display: flex;
+ flex-grow: 1;
+ overflow: hidden;
+}
+
+.sidebar {
+ width: 20%; /* adjust as per your needs */
+ overflow-y: auto;
+ /* add additional styles for your sidebar */
+ color: var(--text);
+}
+
+.content {
+ width: 80%; /* adjust as per your needs */
+ overflow-y: auto;
+ position: relative;
+ /* add additional styles for your content area */
+ color: var(--text);
+ background-color: var(--background);
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx
new file mode 100644
index 0000000..c996b35
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx
@@ -0,0 +1,21 @@
+import styles from "./BaseLayout.module.css";
+import { Header } from "./Header";
+
+export const BaseLayout = ({
+ children,
+ sidebar,
+}: React.PropsWithChildren<{
+ sidebar: React.ReactNode;
+}>) => {
+ return (
+ <div className={styles.container}>
+ <div className={styles.header}>
+ <Header />
+ </div>
+ <div className={styles.main}>
+ <div className={styles.sidebar}>{sidebar}</div>
+ <div className={styles.content}>{children}</div>
+ </div>
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css
new file mode 100644
index 0000000..1b7d443
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css
@@ -0,0 +1,44 @@
+.header {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ padding: 8px;
+}
+
+.logo a {
+ text-decoration: none;
+}
+
+.logo a:hover .wordmark {
+ text-shadow: 0 0 5px rgba(0, 0, 0, 0.1);
+}
+
+.logo .wordmark {
+ color: var(--primary);
+ font-size: large;
+}
+
+.logo .logo_link {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+}
+
+.logo svg {
+ height: 36px;
+ margin: 8px;
+}
+
+.nav {
+ margin: 16px;
+}
+
+.nav a {
+ margin-left: 20px;
+ color: var(--secondary);
+ text-decoration: none;
+}
+
+.nav a:hover {
+ text-decoration: underline;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx
new file mode 100644
index 0000000..71de3fa
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx
@@ -0,0 +1,25 @@
+import React from "react";
+import Link from "next/link";
+import styles from "./Header.module.css";
+import { LogoSVG } from "./Logo";
+
+interface HeaderProps {}
+
+export const Header: React.FC<HeaderProps> = () => {
+ return (
+ <header className={styles.header}>
+ <div className={styles.logo}>
+ <Link href="/" className={styles.logo_link}>
+ <LogoSVG size={36} />
+ <div className={styles.wordmark}>GitHub Analytics</div>
+ </Link>
+ </div>
+ <nav className={styles.nav}>
+ <Link href="/link1">Link 1</Link>
+ <Link href="/link2">Link 2</Link>
+ <Link href="/link3">Link 3</Link>
+ {/* Add more links as needed */}
+ </nav>
+ </header>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx
new file mode 100644
index 0000000..a0e82b8
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx
@@ -0,0 +1,46 @@
+export const LogoSVG = ({ size }: { size: number }) => (
+ <svg
+ width={size}
+ height={size}
+ viewBox="0 0 1024 1024"
+ xmlns="http://www.w3.org/2000/svg"
+ >
+ <path
+ d="M38.31021 15.96259h946.58145c12.77007 0 22.34762 10.37568 22.34762 22.34762v946.58145c0 12.77007-10.37568 22.34762-22.34762 22.34762H38.31021c-12.77007 0-22.34762-10.37568-22.34762-22.34762V38.31021c0-11.97194 10.37568-22.34762 22.34762-22.34762z"
+ fill="#7CB0F1"
+ />
+ <path
+ d="M984.89166 1024H38.31021C16.76071 1024 0 1006.44115 0 985.68979V38.31021C0 17.55885 17.55885 0 38.31021 0h946.58145c21.5495 0 38.31021 17.55885 38.31021 38.31021v946.58145C1024 1006.44116 1006.44115 1024 984.89166 1024zM38.31021 31.92517c-3.19252 0-6.38504 3.19252-6.38504 6.38504v946.58145c0 3.99065 3.19252 6.38504 6.38504 6.38504h946.58145c3.99065 0 6.38504-3.19252 6.38504-6.38504V38.31021c0-3.99065-3.19252-6.38504-6.38504-6.38504H38.31021z"
+ fill="#131313"
+ />
+ <path d="M79.81294 77.41855h865.97038v865.97038H79.81294z" fill="#F3F3F3" />
+ <path
+ d="M945.78332 960.14965H79.81294c-8.77942 0-15.96259-7.18316-15.96259-15.96259V77.41855c0-8.77942 7.18317-15.96259 15.96259-15.96259h865.97038c8.77942 0 15.96259 7.18317 15.96259 15.96259v865.97038c0 9.57755-7.18317 16.76072-15.96259 16.76072zm-850.0078-31.92518h834.04521V93.38114H95.77553v834.84333z"
+ fill="#131313"
+ />
+ <path
+ d="M547.51676 247.4201h-10.37568v240.23695h241.03507v-8.77942c.79813-127.7007-102.9587-231.45752-230.6594-231.45752z"
+ fill="#FB7E7E"
+ />
+ <path
+ d="M778.97428 503.61964H537.9392c-8.77942 0-15.96258-7.18316-15.96258-15.96259V247.42011c0-8.77942 7.18316-15.96259 15.96259-15.96259h10.37568c136.48012 0 247.4201 111.73811 246.62198 248.21824v8.77942c0 7.9813-7.18317 15.16446-15.9626 15.16446zm-225.07249-31.92517h209.1099c-3.99065-113.33438-95.77553-205.11926-209.1099-208.31177v208.31177z"
+ fill="#131313"
+ />
+ <path
+ d="M484.46454 273.75838c-142.86516 6.38503-253.00702 126.90257-246.62198 268.9696 2.39439 63.05222 28.73266 123.71006 72.62977 168.4053l173.9922-173.9922v-263.3827z"
+ fill="#7CB0F1"
+ />
+ <path
+ d="M310.47233 727.894c-3.99065 0-8.77942-1.59626-11.17381-4.78878-47.08963-47.88776-74.22603-111.73811-77.41855-179.5791-6.38504-150.84646 110.93998-279.3453 261.78644-285.73033 4.78877 0 8.77942 1.59626 11.97194 4.78878s4.78877 7.18316 4.78877 11.1738v264.18084c0 3.99064-1.59626 7.98129-4.78877 11.1738l-173.9922 173.99221c-2.3944 3.19252-6.38504 4.78878-11.17382 4.78878zm158.02962-436.57677c-125.30632 14.36632-220.28371 122.91192-214.6968 250.61262 2.39438 54.2728 22.34762 105.35308 57.46531 146.05768l157.23149-157.23149V291.31722z"
+ fill="#131313"
+ />
+ <path
+ d="M495.63835 790.14809c138.87451 0 252.20888-110.14186 257.79579-248.21824h-268.9696v-3.19252L310.47233 711.93141c48.6859 49.48402 114.93063 78.21668 185.16602 78.21668z"
+ fill="#FFD28B"
+ />
+ <path
+ d="M495.63835 806.11068c-73.4279 0-145.25955-30.32892-196.33983-83.00546-6.38504-6.38503-5.5869-15.96259 0-22.34762l173.9922-173.9922c4.78878-4.78878 11.17382-5.58691 17.55885-3.19252 1.59626.79813 2.39439 1.59626 3.99065 2.39438h259.39205c3.99065 0 8.77942 1.59626 11.17381 4.78878s4.78878 7.18317 4.78878 11.97194c-6.38504 147.65394-126.90258 263.3827-274.55651 263.3827zm-162.02027-94.9774c43.89712 39.90647 102.16056 62.2541 162.02027 63.05222 125.30631 0 227.46687-94.17926 241.03507-216.29306H487.65705L333.61808 711.13328zm-2.39438-513.99532H178.78097c-8.77942 0-15.96259-7.18317-15.96259-15.96259s7.18317-15.96259 15.9626-15.96259h152.4427c8.77943 0 15.9626 7.18317 15.9626 15.96259s-7.18317 15.96259-15.9626 15.96259zm0 66.24474H214.6968c-8.77942 0-15.96258-7.18317-15.96258-15.9626s7.18316-15.96258 15.96258-15.96258h116.5269c8.77942 0 15.96258 7.18316 15.96258 15.96259s-7.18316 15.96259-15.96259 15.96259zm513.19719 542.72798H682.40062c-8.77942 0-15.96258-7.18317-15.96258-15.96259s7.18316-15.96259 15.96258-15.96259H844.4209c8.77942 0 15.96259 7.18317 15.96259 15.96259s-7.18317 15.96259-15.9626 15.96259zm-30.32892 62.25409H682.40062c-8.77942 0-15.96258-7.18316-15.96258-15.96259s7.18316-15.96259 15.96258-15.96259h131.69135c8.77943 0 15.96259 7.18317 15.96259 15.9626s-7.18316 15.96258-15.96259 15.96258zm42.30086-671.22681h-173.9922c-8.77943 0-15.9626-7.18317-15.9626-15.96259s7.18317-15.96259 15.9626-15.96259h173.9922c8.77942 0 15.96259 7.18317 15.96259 15.96259s-7.18317 15.96259-15.9626 15.96259z"
+ fill="#131313"
+ />
+ </svg>
+);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css
new file mode 100644
index 0000000..a39db81
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css
@@ -0,0 +1,54 @@
+.sidebar {
+ background-color: var(--background);
+ border-right: 1px solid var(--header);
+ position: relative;
+}
+
+.importButtonContainer {
+ position: sticky;
+ top: 0;
+ width: 100%;
+ background-color: var(--background);
+ display: flex;
+ align-items: center;
+ padding: 8px;
+ border-bottom: 1px dotted var(--sidebar);
+}
+
+.importButton {
+ color: var(--background);
+ background-color: var(--secondary);
+ padding: 16px;
+ border-radius: 16px;
+}
+
+.importButton {
+ text-decoration: none;
+ font-weight: bold;
+}
+
+.importButton:hover {
+ text-shadow: 0 0 5px rgba(43, 0, 255, 0.5);
+}
+
+.repoList {
+ list-style: none;
+ padding: 0;
+}
+
+.repoList li {
+ margin-left: 0;
+ border-bottom: 1px dotted var(--sidebar);
+ padding-top: 8px;
+ padding-bottom: 8px;
+ padding-left: 16px;
+ padding-right: 16px;
+}
+
+.repoList li a {
+ text-decoration: none;
+}
+
+.repoList li a:hover {
+ text-decoration: underline;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
new file mode 100644
index 0000000..f3ccb2f
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
@@ -0,0 +1,33 @@
+import React from "react";
+import Link from "next/link";
+import styles from "./Sidebar.module.css";
+
+export interface GitHubRepository {
+ namespace: string;
+ repository: string;
+}
+
+interface SidebarProps {
+ repositories: GitHubRepository[];
+}
+
+export const Sidebar = ({ repositories }: React.PropsWithRef<SidebarProps>) => {
+ return (
+ <aside className={styles.sidebar}>
+ <div className={styles.importButtonContainer}>
+ <Link href="/start-import" className={styles.importButton}>
+ Import Your Repository
+ </Link>
+ </div>
+ <ul className={styles.repoList}>
+ {repositories.map((repo, index) => (
+ <li key={index}>
+ <Link href={`/${repo.namespace}/${repo.repository}`}>
+ {repo.namespace}/{repo.repository}
+ </Link>
+ </li>
+ ))}
+ </ul>
+ </aside>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css
new file mode 100644
index 0000000..dd0e1d8
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css
@@ -0,0 +1,71 @@
+/* https://www.joshwcomeau.com/css/custom-css-reset/ */
+
+/*
+ 1. Use a more-intuitive box-sizing model.
+*/
+*,
+*::before,
+*::after {
+ box-sizing: border-box;
+}
+/*
+ 2. Remove default margin
+*/
+* {
+ margin: 0;
+}
+/*
+ 3. Allow percentage-based heights in the application
+*/
+html,
+body {
+ height: 100%;
+}
+/*
+ Typographic tweaks!
+ 4. Add accessible line-height
+ 5. Improve text rendering
+*/
+body {
+ line-height: 1.5;
+ -webkit-font-smoothing: antialiased;
+}
+/*
+ 6. Improve media defaults
+*/
+img,
+picture,
+video,
+canvas,
+svg {
+ display: block;
+ max-width: 100%;
+}
+/*
+ 7. Remove built-in form typography styles
+*/
+input,
+button,
+textarea,
+select {
+ font: inherit;
+}
+/*
+ 8. Avoid text overflows
+*/
+p,
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+ overflow-wrap: break-word;
+}
+/*
+ 9. Create a root stacking context
+*/
+#root,
+#__next {
+ isolation: isolate;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
new file mode 100644
index 0000000..ecf2ea9
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
@@ -0,0 +1,14 @@
+:root {
+ --primary: #007bff;
+ --secondary: #ef00a7;
+ --background: #ffffff;
+ --header: #f2f6ff;
+ /* --header: #718096; */
+ --sidebar: #718096;
+ --text: #1a202c;
+ --subtext: #718096;
+}
+
+body {
+ font-family: Consolas, Monaco, "Andale Mono", "Ubuntu Mono", monospace;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
new file mode 100644
index 0000000..3a6568a
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
@@ -0,0 +1,7 @@
+import type { AppProps } from "next/app";
+import "../components/global-styles/reset.css";
+import "../components/global-styles/theme.css";
+
+export default function GitHubAnalyticsApp({ Component, pageProps }: AppProps) {
+ return <Component {...pageProps} />;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
index c3a646d..bb79aff 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
@@ -5,6 +5,10 @@ import {
} from "@madatdata/react";
import { useMemo } from "react";
+import { BaseLayout } from "../components/BaseLayout";
+
+import { Sidebar, type GitHubRepository } from "../components/Sidebar";
+
const ExampleComponentUsingSQL = () => {
const { loading, error, response } = useSql<{
origin_airport: string;
@@ -59,9 +63,39 @@ const SplitgraphSampleQuery = () => {
// Uses splitgraph.com by default (anon access supported for public data)
return (
<SqlProvider dataContext={splitgraphDataContext}>
- <ExampleComponentUsingSQL />
+ <BaseLayout sidebar={<Sidebar repositories={sampleRepositories} />}>
+ <ExampleComponentUsingSQL />
+ </BaseLayout>
</SqlProvider>
);
};
export default SplitgraphSampleQuery;
+
+const sampleRepositories: GitHubRepository[] = [
+ { namespace: "OpenTech", repository: "data-structures" },
+ { namespace: "AiSolutions", repository: "machine-learning-api" },
+ { namespace: "DevToolsInc", repository: "react-components" },
+ { namespace: "QuantumComputing", repository: "quantum-algorithms" },
+ { namespace: "GlobalNetworks", repository: "network-optimization" },
+ { namespace: "CyberSec", repository: "firewall-config" },
+ { namespace: "DataSci", repository: "data-analysis" },
+ { namespace: "WebDevCo", repository: "responsive-templates" },
+ { namespace: "CloudNet", repository: "cloud-infrastructure" },
+ { namespace: "AiData", repository: "neural-networks" },
+ { namespace: "DistributedSys", repository: "microservices-arch" },
+ { namespace: "KernelDev", repository: "os-development" },
+ { namespace: "FrontEndMagic", repository: "vue-utilities" },
+ { namespace: "BackEndLogix", repository: "nodejs-server" },
+ { namespace: "Securitech", repository: "encryption-utils" },
+ { namespace: "FullStack", repository: "end-to-end-app" },
+ { namespace: "DBMasters", repository: "database-design" },
+ { namespace: "MobileApps", repository: "android-development" },
+ { namespace: "GameFactory", repository: "game-engine" },
+ { namespace: "WebAssembly", repository: "wasm-runtime" },
+ { namespace: "RoboLogic", repository: "robot-navigation" },
+ { namespace: "IoTDesign", repository: "iot-devices" },
+ { namespace: "BlockchainTech", repository: "blockchain-network" },
+ { namespace: "CryptoCoins", repository: "cryptocurrency" },
+ { namespace: "VRWorld", repository: "vr-applications" },
+];
From 1f3b6d1c6d9159d84db9894d91100e8b7c93b786 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 24 May 2023 20:29:15 +0100
Subject: [PATCH 03/36] Add backend config and API routes for starting,
awaiting import task
---
.../.env.test.local | 18 +++
.../.gitignore | 3 +
.../env-vars.d.ts | 41 ++++++
.../lib-backend/splitgraph-db.ts | 52 ++++++++
.../next.config.js | 33 +++++
.../package.json | 1 +
.../pages/api/await-import-from-github.ts | 89 +++++++++++++
.../pages/api/start-import-from-github.ts | 117 ++++++++++++++++++
.../tsconfig.json | 2 +-
9 files changed, 355 insertions(+), 1 deletion(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/next.config.js
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
new file mode 100644
index 0000000..473bf7d
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
@@ -0,0 +1,18 @@
+# IMPORTANT: Put your own values in `.env.local` (a git-ignored file) when running this locally
+# Configure them in Vercel settings when running in production
+# This file is mostly to show which variables exist, since it's the only one checked into the repo.
+# SEE: https://nextjs.org/docs/app/building-your-application/configuring/environment-variables
+
+# Create your own API key and secret: https://www.splitgraph.com/connect
+SPLITGRAPH_API_KEY="********************************"
+SPLITGRAPH_API_SECRET="********************************"
+
+# Create a GitHub token that can query the repositories you want to connect
+# For example, a token with read-only access to public repos is sufficient
+# CREATE ONE HERE: https://github.com/settings/personal-access-tokens/new
+GITHUB_PAT_SECRET="github_pat_**********************_***********************************************************"
+
+# OPTIONAL: Set this environment variable to a proxy address to capture requests from API routes
+# e.g. To intercept requests to Splitgraph API sent from madatdata libraries in API routes
+# You can also set this by running: yarn dev-mitm (see package.json)
+# MITMPROXY_ADDRESS="http://localhost:7979"
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore
index a680367..d8dcd4f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore
@@ -1 +1,4 @@
.next
+.env.local
+.env.*.local
+!.env.test.local
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
new file mode 100644
index 0000000..5328009
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
@@ -0,0 +1,41 @@
+namespace NodeJS {
+ interface ProcessEnv {
+ /**
+ * The API key of an existing Splitgraph account.
+ *
+ * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings.
+ *
+ * Get credentials: https://www.splitgraph.com/connect
+ */
+ SPLITGRAPH_API_KEY: string;
+
+ /**
+ * The API secret of an existing Splitgraph account.
+ *
+ * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings.
+ *
+ * Get credentials: https://www.splitgraph.com/connect
+ */
+ SPLITGRAPH_API_SECRET: string;
+
+ /**
+ * A GitHub personal access token that can be used for importing repositories.
+ * It will be passed to the Airbyte connector that runs on Splitgraph servers
+ * and ingests data from GitHub into Splitgraph.
+ *
+ * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings.
+ *
+ * Create one here: https://github.com/settings/personal-access-tokens/new
+ */
+ GITHUB_PAT_SECRET: string;
+
+ /**
+ * Optional environment variable containing the address of a proxy instance
+ * through which to forward requests from API routes. See next.config.js
+ * for where it's setup.
+ *
+ * This is useful for debugging and development.
+ */
+ MITMPROXY_ADDRESS?: string;
+ }
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts
new file mode 100644
index 0000000..66ba751
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts
@@ -0,0 +1,52 @@
+import { makeSplitgraphDb } from "@madatdata/core";
+
+// TODO: fix plugin exports
+import { makeDefaultPluginList } from "@madatdata/db-splitgraph";
+import { defaultSplitgraphHost } from "@madatdata/core";
+
+const SPLITGRAPH_API_KEY = process.env.SPLITGRAPH_API_KEY;
+const SPLITGRAPH_API_SECRET = process.env.SPLITGRAPH_API_SECRET;
+
+if (!SPLITGRAPH_API_KEY || !SPLITGRAPH_API_SECRET) {
+ throw new Error(
+ "Environment variable SPLITGRAPH_API_KEY or SPLITGRAPH_API_SECRET is not set." +
+ " See env-vars.d.ts for instructions."
+ );
+}
+
+const authenticatedCredential: Parameters<
+ typeof makeSplitgraphDb
+>[0]["authenticatedCredential"] = {
+ apiKey: SPLITGRAPH_API_KEY,
+ apiSecret: SPLITGRAPH_API_SECRET,
+ anonymous: false,
+};
+
+// TODO: The access token can expire and silently fail?
+
+export const makeAuthenticatedSplitgraphDb = () =>
+ makeSplitgraphDb({
+ authenticatedCredential,
+ plugins: makeDefaultPluginList({
+ graphqlEndpoint: defaultSplitgraphHost.baseUrls.gql,
+ authenticatedCredential,
+ }),
+ });
+
+// TODO: export this utility function from the library
+export const claimsFromJWT = (jwt?: string) => {
+ if (!jwt) {
+ return {};
+ }
+
+ const [_header, claims, _signature] = jwt
+ .split(".")
+ .map(fromBase64)
+ .slice(0, -1) // Signature is not parseable JSON
+ .map((o) => JSON.parse(o));
+
+ return claims;
+};
+
+const fromBase64 = (input: string) =>
+ !!globalThis.Buffer ? Buffer.from(input, "base64").toString() : atob(input);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/next.config.js b/examples/nextjs-import-airbyte-github-export-seafowl/next.config.js
new file mode 100644
index 0000000..ac7ae64
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/next.config.js
@@ -0,0 +1,33 @@
+const { ProxyAgent, setGlobalDispatcher } = require("undici");
+
+// If running `yarn dev-mitm`, then setup the proxy with MITMPROXY_ADDRESS
+// NOTE(FIXME): not all madatdata requests get sent through here for some reason
+const setupProxy = () => {
+ if (!process.env.MITMPROXY_ADDRESS) {
+ return;
+ }
+
+ const MITM = process.env.MITMPROXY_ADDRESS;
+
+ console.log("MITM SETUP:", MITM);
+
+ if (!process.env.GLOBAL_AGENT_HTTP_PROXY) {
+ process.env["GLOBAL_AGENT_HTTP_PROXY"] = MITM;
+ }
+
+ process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0";
+
+ const mitmProxyOpts = {
+ uri: MITM,
+ connect: {
+ rejectUnauthorized: false,
+ requestCert: false,
+ },
+ };
+
+ setGlobalDispatcher(new ProxyAgent(mitmProxyOpts));
+};
+
+setupProxy();
+
+module.exports = {};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/package.json b/examples/nextjs-import-airbyte-github-export-seafowl/package.json
index 6d17d38..ef04766 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/package.json
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/package.json
@@ -2,6 +2,7 @@
"private": true,
"scripts": {
"dev": "yarn next",
+ "dev-mitm": "MITMPROXY_ADDRESS=http://localhost:7979 yarn next",
"build": "yarn next build",
"start": "yarn next start"
},
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts
new file mode 100644
index 0000000..d06aac9
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts
@@ -0,0 +1,89 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+import type { DeferredSplitgraphImportTask } from "@madatdata/db-splitgraph/plugins/importers/splitgraph-base-import-plugin";
+
+type ResponseData =
+ | {
+ completed: boolean;
+ jobStatus: DeferredSplitgraphImportTask["response"]["jobStatus"];
+ }
+ | { error: string; completed: false };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+ -H "Content-Type: application/json" http://localhost:3000/api/await-import-from-github \
+ -d '{ "taskId": "xxxx", "splitgraphNamespace": "xxx", "splitgraphRepo": "yyy" }'
+```
+ */
+export default async function handler(
+ req: NextApiRequest,
+ res: NextApiResponse<ResponseData>
+) {
+ const missing = [
+ "taskId",
+ "splitgraphNamespace",
+ "splitgraphRepository",
+ ].filter((expKey) => !req.body[expKey]);
+ if (missing.length > 0) {
+ res.status(400).json({
+ error: `Missing required keys: ${missing.join(", ")}`,
+ completed: false,
+ });
+ return;
+ }
+
+ const { taskId, splitgraphNamespace, splitgraphRepository } = req.body;
+
+ try {
+ const maybeCompletedTask = await pollImport({
+ splitgraphTaskId: taskId,
+ splitgraphDestinationNamespace: splitgraphNamespace,
+ splitgraphDestinationRepository: splitgraphRepository,
+ });
+
+ if (maybeCompletedTask.error) {
+ throw new Error(JSON.stringify(maybeCompletedTask.error));
+ }
+
+ res.status(200).json(maybeCompletedTask);
+ return;
+ } catch (err) {
+ res.status(400).json({
+ error: err.message,
+ completed: false,
+ });
+ return;
+ }
+}
+
+const pollImport = async ({
+ splitgraphTaskId,
+ splitgraphDestinationNamespace,
+ splitgraphDestinationRepository,
+}: {
+ splitgraphDestinationNamespace: string;
+ splitgraphDestinationRepository: string;
+ splitgraphTaskId: string;
+}) => {
+ const db = makeAuthenticatedSplitgraphDb();
+
+ // NOTE: We must call this, or else requests will fail silently
+ await db.fetchAccessToken();
+
+ const maybeCompletedTask = (await db.pollDeferredTask("csv", {
+ taskId: splitgraphTaskId,
+ namespace: splitgraphDestinationNamespace,
+ repository: splitgraphDestinationRepository,
+ })) as DeferredSplitgraphImportTask;
+
+ // NOTE: We do not include the jobLog, in case it could leak the GitHub PAT
+ // (remember we're using our PAT on behalf of the users of this app)
+ return {
+ completed: maybeCompletedTask?.completed ?? false,
+ jobStatus: maybeCompletedTask?.response.jobStatus,
+ error: maybeCompletedTask?.error ?? undefined,
+ };
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
new file mode 100644
index 0000000..01c2b58
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
@@ -0,0 +1,117 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import {
+ makeAuthenticatedSplitgraphDb,
+ claimsFromJWT,
+} from "../../lib-backend/splitgraph-db";
+
+const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET;
+
+type ResponseData =
+ | {
+ destination: {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+ };
+ taskId: string;
+ }
+ | { error: string };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+ -H "Content-Type: application/json" http://localhost:3000/api/start-import-from-github \
+ -d '{ "githubSourceRepository": "splitgraph/seafowl", "splitgraphDestinationRepository": "import-via-nextjs" }'
+```
+ */
+export default async function handler(
+ req: NextApiRequest,
+ res: NextApiResponse<ResponseData>
+) {
+ const db = makeAuthenticatedSplitgraphDb();
+ const { username } = claimsFromJWT((await db.fetchAccessToken()).token);
+
+ const { githubSourceRepository } = req.body;
+
+ if (!githubSourceRepository) {
+ res.status(400).json({ error: "githubSourceRepository is required" });
+ return;
+ }
+
+ const splitgraphDestinationRepository =
+ req.body.splitgraphDestinationRepository ??
+ `github-import-${githubSourceRepository.replaceAll("/", "-")}`;
+
+ try {
+ const taskId = await startImport({
+ db,
+ githubSourceRepository,
+ splitgraphDestinationRepository,
+ githubStartDate: req.body.githubStartDate,
+ });
+ res.status(200).json({
+ destination: {
+ splitgraphNamespace: username,
+ splitgraphRepository: splitgraphDestinationRepository,
+ },
+ taskId,
+ });
+ } catch (err) {
+ res.status(400).json({
+ error: err.message,
+ });
+ }
+}
+
+const startImport = async ({
+ db,
+ githubSourceRepository,
+ splitgraphDestinationRepository,
+ githubStartDate,
+}: {
+ db: ReturnType<typeof makeAuthenticatedSplitgraphDb>;
+ githubSourceRepository: string;
+ splitgraphDestinationRepository: string;
+ /**
+ * Optional start date for ingestion, must be in format like: 2021-06-01T00:00:00Z
+ * Defaults to 2020-01-01T00:00:00Z
+ * */
+ githubStartDate?: string;
+}) => {
+ const { username: splitgraphNamespace } = claimsFromJWT(
+ (await db.fetchAccessToken()).token
+ );
+
+ const { taskId } = await db.importData(
+ "airbyte-github",
+ {
+ credentials: {
+ credentials: {
+ personal_access_token: GITHUB_PAT_SECRET,
+ },
+ },
+ params: {
+ repository: githubSourceRepository,
+ start_date: githubStartDate ?? "2020-01-01T00:00:00Z",
+ },
+ },
+ {
+ namespace: splitgraphNamespace,
+ repository: splitgraphDestinationRepository,
+ tables: [
+ {
+ name: "stargazers",
+ options: {
+ airbyte_cursor_field: ["starred_at"],
+ airbyte_primary_key_field: [],
+ },
+ schema: [],
+ },
+ ],
+ },
+ { defer: true }
+ );
+
+ return taskId;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json
index 16bb209..6446b1b 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json
@@ -15,6 +15,6 @@
"isolatedModules": true,
"jsx": "preserve"
},
- "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
+ "include": ["next-env.d.ts", "env-vars.d.ts", "**/*.ts", "**/*.tsx"],
"exclude": ["node_modules"]
}
From 82db03bd0590564f53d9475a59840eee2b88cbb5 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 24 May 2023 20:30:11 +0100
Subject: [PATCH 04/36] Add Stepper component with ImportPanel and ExportPanel
Implement the import panel and stub out the export panel,
using a single Stepper component and a react context with
a reducer for managing the state. Implement the fetch requests
to start the import, and also to await the import.
Co-Authored by GPT-4 ;)
---
.../components/BaseLayout.module.css | 1 +
.../ExportPanel.module.css | 6 +
.../ImportExportStepper/ExportPanel.tsx | 19 ++++
.../ImportExportStepper/ImportLoadingBar.tsx | 74 ++++++++++++
.../ImportPanel.module.css | 10 ++
.../ImportExportStepper/ImportPanel.tsx | 105 ++++++++++++++++++
.../ImportExportStepper/Stepper.module.css | 4 +
.../ImportExportStepper/Stepper.tsx | 16 +++
.../ImportExportStepper/StepperContext.tsx | 34 ++++++
.../ImportExportStepper/stepper-states.ts | 93 ++++++++++++++++
.../components/global-styles/theme.css | 1 +
.../pages/index.tsx | 54 +--------
12 files changed, 366 insertions(+), 51 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css
index 5cee596..76f89c8 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css
@@ -34,4 +34,5 @@
/* add additional styles for your content area */
color: var(--text);
background-color: var(--background);
+ padding: 24px;
}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
new file mode 100644
index 0000000..4a5ea5b
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
@@ -0,0 +1,6 @@
+/* ExportPanel.module.css */
+
+.exportPanel {
+ /* Style for export panel will go here */
+ background: inherit;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
new file mode 100644
index 0000000..903810b
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -0,0 +1,19 @@
+import styles from "./ExportPanel.module.css";
+import { useStepper } from "./StepperContext";
+
+export const ExportPanel = () => {
+ const [{ stepperState }] = useStepper();
+
+ const disabled =
+ stepperState !== "import_complete" &&
+ stepperState !== "awaiting_export" &&
+ stepperState !== "export_complete";
+
+ // We will fill this in later
+
+ return (
+ <div className={styles.exportPanel}>
+ {disabled ? "Export disabled" : "Export..."}
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
new file mode 100644
index 0000000..8130fed
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
@@ -0,0 +1,74 @@
+import { useEffect } from "react";
+import { useStepper } from "./StepperContext";
+
+type ImportLoadingBarProps = {
+ taskId: string;
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+};
+
+export const ImportLoadingBar: React.FC<ImportLoadingBarProps> = ({
+ taskId,
+ splitgraphNamespace,
+ splitgraphRepository,
+}) => {
+ const [{ stepperState }, dispatch] = useStepper();
+
+ useEffect(() => {
+ if (!taskId || !splitgraphNamespace || !splitgraphRepository) {
+ console.log("Don't check import until we have all the right variables");
+ console.table({
+ taskId: taskId ?? "no task id",
+ splitgraphNamespace: splitgraphNamespace ?? "no namespace",
+ splitgraphRepository: splitgraphRepository ?? "no repo",
+ });
+ return;
+ }
+
+ if (stepperState !== "awaiting_import") {
+ console.log("Done waiting");
+ return;
+ }
+
+ const checkImportStatus = async () => {
+ try {
+ const response = await fetch("/api/await-import-from-github", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ taskId,
+ splitgraphNamespace,
+ splitgraphRepository,
+ }),
+ });
+ const data = await response.json();
+
+ if (data.completed) {
+ dispatch({ type: "import_complete" });
+ } else if (data.error) {
+ dispatch({ type: "import_error", error: data.error });
+ }
+ } catch (error) {
+ console.error("Error occurred during import task status check:", error);
+ dispatch({
+ type: "import_error",
+ error: "An error occurred during the import process",
+ });
+ }
+ };
+
+ const interval = setInterval(checkImportStatus, 3000);
+
+ return () => clearInterval(interval);
+ }, [
+ stepperState,
+ taskId,
+ splitgraphNamespace,
+ splitgraphRepository,
+ dispatch,
+ ]);
+
+ return <div>Loading...</div>;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css
new file mode 100644
index 0000000..ea3961d
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css
@@ -0,0 +1,10 @@
+.importPanel {
+ background: inherit;
+}
+
+.error {
+ background-color: var(--danger);
+ padding: 8px;
+ border: 1px solid var(--sidebar);
+ margin-bottom: 8px;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
new file mode 100644
index 0000000..7da3725
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
@@ -0,0 +1,105 @@
+import { useState } from "react";
+import { useStepper } from "./StepperContext";
+import { ImportLoadingBar } from "./ImportLoadingBar";
+
+import styles from "./ImportPanel.module.css";
+
+export const ImportPanel = () => {
+ const [
+ { stepperState, taskId, error, splitgraphNamespace, splitgraphRepository },
+ dispatch,
+ ] = useStepper();
+ const [inputValue, setInputValue] = useState("");
+
+ const handleInputSubmit = async (e: React.FormEvent) => {
+ e.preventDefault();
+
+ if (!isValidRepoName(inputValue)) {
+ dispatch({
+ type: "import_error",
+ error:
+ "Invalid GitHub repository name. Format must be 'namespace/repository'",
+ });
+ return;
+ }
+
+ const [githubNamespace, githubRepository] = inputValue.split("/");
+
+ try {
+ const response = await fetch(`/api/start-import-from-github`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({ githubSourceRepository: inputValue }),
+ });
+
+ if (!response.ok) {
+ throw new Error("Network response was not ok");
+ }
+
+ const data = await response.json();
+
+ if (!data.taskId) {
+ throw new Error("Response missing taskId");
+ }
+
+ if (!data.destination || !data.destination.splitgraphNamespace) {
+ throw new Error("Response missing destination.splitgraphNamespace");
+ }
+
+ if (!data.destination || !data.destination.splitgraphRepository) {
+ throw new Error("Response missing destination.splitgraphRepository");
+ }
+
+ dispatch({
+ type: "start_import",
+ repository: {
+ namespace: githubNamespace,
+ repository: githubRepository,
+ },
+ taskId: data.taskId,
+ splitgraphRepository: data.destination.splitgraphRepository as string,
+ splitgraphNamespace: data.destination.splitgraphNamespace as string,
+ });
+ } catch (error) {
+ dispatch({ type: "import_error", error: error.message });
+ }
+ };
+
+ const isValidRepoName = (repoName: string) => {
+ // A valid GitHub repo name should contain exactly one '/'
+ return /^[\w-.]+\/[\w-.]+$/.test(repoName);
+ };
+
+ return (
+ <div className={styles.importPanel}>
+ {stepperState === "unstarted" && (
+ <>
+ {error && <p className={styles.error}>{error}</p>}
+ <form onSubmit={handleInputSubmit}>
+ <input
+ type="text"
+ placeholder="Enter repository name"
+ value={inputValue}
+ onChange={(e) => setInputValue(e.target.value)}
+ />
+ <button type="submit">Start Import</button>
+ </form>
+ </>
+ )}
+ {stepperState === "awaiting_import" && (
+ <ImportLoadingBar
+ taskId={taskId}
+ splitgraphNamespace={splitgraphNamespace}
+ splitgraphRepository={splitgraphRepository}
+ />
+ )}
+ {stepperState === "import_complete" && (
+ <div>
+ <p>Import Complete</p>
+ </div>
+ )}
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css
new file mode 100644
index 0000000..a1eb9c2
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css
@@ -0,0 +1,4 @@
+.stepper {
+ /* Add styling as necessary */
+ background: inherit;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
new file mode 100644
index 0000000..6af0f06
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
@@ -0,0 +1,16 @@
+import { StepperContextProvider } from "./StepperContext";
+import { ImportPanel } from "./ImportPanel"; // will create this component later
+import { ExportPanel } from "./ExportPanel"; // will create this component later
+
+import styles from "./Stepper.module.css";
+
+export const Stepper = () => {
+ return (
+ <StepperContextProvider>
+ <div className={styles.stepper}>
+ <ImportPanel />
+ <ExportPanel />
+ </div>
+ </StepperContextProvider>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
new file mode 100644
index 0000000..cbff3a1
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
@@ -0,0 +1,34 @@
+// StepperContext.tsx
+import React, { useReducer, useContext, ReactNode } from "react";
+import {
+ StepperState,
+ StepperAction,
+ initialState,
+ stepperReducer,
+} from "./stepper-states";
+
+// Define the context
+const StepperContext = React.createContext<
+ [StepperState, React.Dispatch<StepperAction>] | undefined
+>(undefined);
+
+export const StepperContextProvider: React.FC<{ children: ReactNode }> = ({
+ children,
+}) => {
+ const [state, dispatch] = useReducer(stepperReducer, initialState);
+
+ return (
+ <StepperContext.Provider value={[state, dispatch]}>
+ {children}
+ </StepperContext.Provider>
+ );
+};
+
+// Custom hook for using the stepper context
+export const useStepper = () => {
+ const context = useContext(StepperContext);
+ if (!context) {
+ throw new Error("useStepper must be used within a StepperContextProvider");
+ }
+ return context;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
new file mode 100644
index 0000000..180c0fb
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -0,0 +1,93 @@
+// stepper-states.ts
+export type GitHubRepository = { namespace: string; repository: string };
+
+// Define the state
+export type StepperState = {
+ stepperState:
+ | "unstarted"
+ | "awaiting_import"
+ | "import_complete"
+ | "awaiting_export"
+ | "export_complete";
+ repository?: GitHubRepository | null;
+ taskId?: string | null;
+ error?: string;
+ tables?: { taskId: string }[] | null;
+ splitgraphRepository?: string;
+ splitgraphNamespace?: string;
+};
+
+// Define the actions
+export type StepperAction =
+ | {
+ type: "start_import";
+ repository: GitHubRepository;
+ taskId: string;
+ splitgraphRepository: string;
+ splitgraphNamespace: string;
+ }
+ | { type: "import_complete" }
+ | { type: "start_export"; tables: { taskId: string }[] }
+ | { type: "export_complete" }
+ | { type: "import_error"; error: string }
+ | { type: "reset" };
+
+// Initial state
+export const initialState: StepperState = {
+ stepperState: "unstarted",
+ repository: null,
+ splitgraphRepository: null,
+ splitgraphNamespace: null,
+ taskId: null,
+ tables: null,
+};
+
+// Reducer function
+export const stepperReducer = (
+ state: StepperState,
+ action: StepperAction
+): StepperState => {
+ console.log("Got action", action, "prev state:", state);
+ switch (action.type) {
+ case "start_import":
+ return {
+ ...state,
+ stepperState: "awaiting_import",
+ repository: action.repository,
+ taskId: action.taskId,
+ splitgraphNamespace: action.splitgraphNamespace,
+ splitgraphRepository: action.splitgraphRepository,
+ };
+ case "import_complete":
+ return {
+ ...state,
+ stepperState: "import_complete",
+ };
+ case "start_export":
+ return {
+ ...state,
+ stepperState: "awaiting_export",
+ tables: action.tables,
+ };
+ case "export_complete":
+ return {
+ ...state,
+ stepperState: "export_complete",
+ };
+ case "import_error":
+ return {
+ ...state,
+ splitgraphRepository: null,
+ splitgraphNamespace: null,
+ taskId: null,
+ stepperState: "unstarted",
+ error: action.error,
+ };
+
+ case "reset":
+ return initialState;
+
+ default:
+ return state;
+ }
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
index ecf2ea9..66c4514 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
@@ -7,6 +7,7 @@
--sidebar: #718096;
--text: #1a202c;
--subtext: #718096;
+ --danger: #eb8585;
}
body {
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
index bb79aff..4218207 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
@@ -1,58 +1,10 @@
-import {
- SqlProvider,
- useSql,
- makeSplitgraphHTTPContext,
-} from "@madatdata/react";
+import { SqlProvider, makeSplitgraphHTTPContext } from "@madatdata/react";
import { useMemo } from "react";
import { BaseLayout } from "../components/BaseLayout";
import { Sidebar, type GitHubRepository } from "../components/Sidebar";
-
-const ExampleComponentUsingSQL = () => {
- const { loading, error, response } = useSql<{
- origin_airport: string;
- destination_airport: string;
- origin_city: string;
- destination_city: string;
- passengers: number;
- seats: number;
- flights: number;
- distance: number;
- fly_month: string;
- origin_pop: number;
- destination_pop: number;
- id: number;
- }>(
- `SELECT
- "origin_airport",
- "destination_airport",
- "origin_city",
- "destination_city",
- "passengers",
- "seats",
- "flights",
- "distance",
- "fly_month",
- "origin_pop",
- "destination_pop",
- "id"
-FROM
- "splitgraph/domestic_us_flights:latest"."flights"
-LIMIT 100;`
- );
-
- return (
- <pre
- style={{ minWidth: "100%", minHeight: 500 }}
- data-testid={`result-pre-${
- loading ? "loading" : response ? "pass" : error ? "fail" : "unknown"
- }`}
- >
- {JSON.stringify({ loading, error, response }, null, 2)}
- </pre>
- );
-};
+import { Stepper } from "../components/ImportExportStepper/Stepper";
const SplitgraphSampleQuery = () => {
const splitgraphDataContext = useMemo(
@@ -64,7 +16,7 @@ const SplitgraphSampleQuery = () => {
return (
<SqlProvider dataContext={splitgraphDataContext}>
<BaseLayout sidebar={<Sidebar repositories={sampleRepositories} />}>
- <ExampleComponentUsingSQL />
+ <Stepper />
</BaseLayout>
</SqlProvider>
);
From 09750751bb3deac74c19bc68ffe1b5869b2d2975 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Thu, 25 May 2023 04:00:22 +0100
Subject: [PATCH 05/36] Implement backend API routes `start-export-to-seafowl`
and `await-export-to-seafowl-task`
The `start-export-to-seafowl` route takes a list of source
tables from Splitgraph (list of `{namespace,repository,table}`),
and starts a task to export them to Seafowl. It returns a list
of objects `{taskId: string; tableName: string;}`, where each
item represents the currently exporting table (and `tableName` is
the source table name).
The `await-export-to-seafowl-task` route takes a single `taskId`
parameter and returns its status, i.e. `{completed: boolean; ...otherInfo}`
---
.../pages/api/await-export-to-seafowl-task.ts | 76 +++++++++++++
.../pages/api/start-export-to-seafowl.ts | 103 ++++++++++++++++++
2 files changed, 179 insertions(+)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts
new file mode 100644
index 0000000..93f6b89
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts
@@ -0,0 +1,76 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+import type { DeferredSplitgraphExportTask } from "@madatdata/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin";
+
+type ResponseData =
+ | {
+ completed: boolean;
+ jobStatus: DeferredSplitgraphExportTask["response"];
+ }
+ | { error: string; completed: false };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+ -H "Content-Type: application/json" http://localhost:3000/api/await-export-to-seafowl-task \
+ -d '{ "taskId": "2923fd6f-2197-495a-9df1-2428a9ca8dee" }'
+```
+ */
+export default async function handler(
+ req: NextApiRequest,
+ res: NextApiResponse<ResponseData>
+) {
+ if (!req.body["taskId"]) {
+ res.status(400).json({
+ error: "Missing required key: taskId",
+ completed: false,
+ });
+ return;
+ }
+
+ const { taskId } = req.body;
+
+ try {
+ const maybeCompletedTask = await pollImport({
+ splitgraphTaskId: taskId,
+ });
+
+ if (maybeCompletedTask.error) {
+ throw new Error(JSON.stringify(maybeCompletedTask.error));
+ }
+
+ res.status(200).json(maybeCompletedTask);
+ return;
+ } catch (err) {
+ res.status(400).json({
+ error: err.message,
+ completed: false,
+ });
+ return;
+ }
+}
+
+const pollImport = async ({
+ splitgraphTaskId,
+}: {
+ splitgraphTaskId: string;
+}) => {
+ const db = makeAuthenticatedSplitgraphDb();
+
+ // NOTE: We must call this, or else requests will fail silently
+ await db.fetchAccessToken();
+
+ const maybeCompletedTask = (await db.pollDeferredTask("export-to-seafowl", {
+ taskId: splitgraphTaskId,
+ })) as DeferredSplitgraphExportTask;
+
+ // NOTE: We do not include the jobLog, in case it could leak the GitHub PAT
+ // (remember we're using our PAT on behalf of the users of this app)
+ return {
+ completed: maybeCompletedTask?.completed ?? false,
+ jobStatus: maybeCompletedTask?.response,
+ error: maybeCompletedTask?.error ?? undefined,
+ };
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
new file mode 100644
index 0000000..75d6343
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
@@ -0,0 +1,103 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+
+type ResponseData =
+ | {
+ tables: {
+ tableName: string;
+ taskId: string;
+ }[];
+ }
+ | { error: string };
+
+type TableInput = { namespace: string; repository: string; table: string };
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+ -H "Content-Type: application/json" http://localhost:3000/api/start-export-to-seafowl \
+ -d '{ "tables": [{"namespace": "miles", "repository": "import-via-nextjs", "table": "stargazers"}] }'
+```
+ */
+export default async function handler(
+ req: NextApiRequest,
+ res: NextApiResponse<ResponseData>
+) {
+ const db = makeAuthenticatedSplitgraphDb();
+ const { tables } = req.body;
+
+ if (
+ !tables ||
+ !tables.length ||
+ !tables.every(
+ (t: TableInput) =>
+ t.namespace &&
+ t.repository &&
+ t.table &&
+ typeof t.namespace === "string" &&
+ typeof t.repository === "string" &&
+ typeof t.table === "string"
+ )
+ ) {
+ res.status(400).json({ error: "invalid tables input in request body" });
+ return;
+ }
+
+ try {
+ const exportingTables = await startExport({
+ db,
+ tables,
+ });
+ res.status(200).json({
+ tables: exportingTables,
+ });
+ } catch (err) {
+ res.status(400).json({
+ error: err.message,
+ });
+ }
+}
+
+const startExport = async ({
+ db,
+ tables,
+}: {
+ db: ReturnType<typeof makeAuthenticatedSplitgraphDb>;
+ tables: TableInput[];
+}) => {
+ await db.fetchAccessToken();
+
+ const response = await db.exportData(
+ "export-to-seafowl",
+ {
+ tables: tables.map((splitgraphSource) => ({
+ source: {
+ repository: splitgraphSource.repository,
+ namespace: splitgraphSource.namespace,
+ table: splitgraphSource.table,
+ },
+ })),
+ },
+ {
+ // Empty instance will trigger Splitgraph to export to demo.seafowl.cloud
+ seafowlInstance: {},
+ },
+ { defer: true }
+ );
+
+ if (response.error) {
+ throw new Error(JSON.stringify(response.error));
+ }
+
+ const loadingTables: { taskId: string; tableName: string }[] =
+ response.taskIds.tables.map(
+ (t: { jobId: string; sourceTable: string }) => ({
+ taskId: t.jobId,
+ tableName: t.sourceTable,
+ })
+ );
+
+ return loadingTables;
+};
From 5cd10be02a86d6da9d27dbf79a5fb8689e84e502 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Thu, 25 May 2023 04:02:19 +0100
Subject: [PATCH 06/36] Implement the components for the "Export Panel"
The ExportPanel first renders a "Start Export" button. Then, while
the export is running, it renders an `ExportTableLoadingBar` for
each table that is being exported. Each of thee individual
components sends its own polling request with its `taskId`
to the `await-export-to-seafowl-task` endpoint, and upon
completion of each task, sends an action to the reducer, which
handles it by updating the set of loading tasks. When the set of
loading tasks is complete, it changes the `stepperState` to
`export_complete`. If any of the tasks has an error, then the
`stepperState` changes to `export_error` which should cause all
loading bars to unmount - i.e., any error will short-circuit all
of the table loading, even if some were to complete. At that point
the user can click "start export" again.
This completes the logic necessary for import and export, and now
it's just a matter of styling the components, linking to the
Splitgraph Console, adding explanatory text, and finally rendering
a chart with the data. We'll also want to create a meta-repository
in Splitgraph for tracking which GitHub repos we've imported so far,
analogously to how we track Socrata metadata for each Socrata repo.
---
.../ExportLoadingBars.module.css | 3 +
.../ImportExportStepper/ExportLoadingBars.tsx | 19 +++++
.../ExportPanel.module.css | 19 ++++-
.../ImportExportStepper/ExportPanel.tsx | 74 +++++++++++++++--
.../ExportTableLoadingBar.module.css | 17 ++++
.../ExportTableLoadingBar.tsx | 78 +++++++++++++++++
.../ImportExportStepper/ImportLoadingBar.tsx | 1 -
.../ImportExportStepper/ImportPanel.tsx | 12 ++-
.../ImportExportStepper/stepper-states.ts | 83 +++++++++++++++----
9 files changed, 277 insertions(+), 29 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css
new file mode 100644
index 0000000..04c119f
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css
@@ -0,0 +1,3 @@
+.exportLoadingBars {
+ background-color: inherit;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
new file mode 100644
index 0000000..f830456
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
@@ -0,0 +1,19 @@
+import { useStepper } from "./StepperContext";
+import { ExportTableLoadingBar } from "./ExportTableLoadingBar";
+import styles from "./ExportLoadingBars.module.css";
+
+export const ExportLoadingBars = () => {
+ const [{ exportedTablesLoading }] = useStepper();
+
+ return (
+ <div className={styles.exportLoadingBars}>
+ {Array.from(exportedTablesLoading).map(({ tableName, taskId }) => (
+ <ExportTableLoadingBar
+ key={taskId}
+ tableName={tableName}
+ taskId={taskId}
+ />
+ ))}
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
index 4a5ea5b..c82f97c 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
@@ -1,6 +1,19 @@
-/* ExportPanel.module.css */
-
.exportPanel {
- /* Style for export panel will go here */
+ /* Styles for the export panel container */
+ background: inherit;
+}
+
+.startExportButton {
+ /* Styles for the start export button */
+ background: inherit;
+}
+
+.querySeafowlButton {
+ /* Styles for the query Seafowl button */
+ background: inherit;
+}
+
+.viewReportButton {
+ /* Styles for the view report button */
background: inherit;
}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index 903810b..8152602 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -1,19 +1,77 @@
-import styles from "./ExportPanel.module.css";
+// components/ImportExportStepper/ExportPanel.tsx
+
import { useStepper } from "./StepperContext";
+import styles from "./ExportPanel.module.css";
+import { ExportLoadingBars } from "./ExportLoadingBars";
+
+// TODO: don't hardcode this? or at least hardcode all of them and make it official
+const importedTableNames = [
+ "stargazers",
+ // NOTE: If we only specify stargazers, then stargazers_user is still included since it's a dependent table
+ "stargazers_user",
+];
export const ExportPanel = () => {
- const [{ stepperState }] = useStepper();
+ const [
+ { stepperState, exportError, splitgraphRepository, splitgraphNamespace },
+ dispatch,
+ ] = useStepper();
+
+ const handleStartExport = async () => {
+ try {
+ const response = await fetch("/api/start-export-to-seafowl", {
+ method: "POST",
+ body: JSON.stringify({
+ tables: importedTableNames.map((tableName) => ({
+ namespace: splitgraphNamespace,
+ repository: splitgraphRepository,
+ table: tableName,
+ })),
+ }),
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+ const data = await response.json();
- const disabled =
- stepperState !== "import_complete" &&
- stepperState !== "awaiting_export" &&
- stepperState !== "export_complete";
+ if (!data.tables || !data.tables.length) {
+ throw new Error("Response missing tables");
+ }
- // We will fill this in later
+ dispatch({
+ type: "start_export",
+ tables: data.tables.map(
+ ({ tableName, taskId }: { tableName: string; taskId: string }) => ({
+ taskId,
+ tableName,
+ })
+ ),
+ });
+ } catch (error) {
+ dispatch({ type: "export_error", error: error.message });
+ }
+ };
return (
<div className={styles.exportPanel}>
- {disabled ? "Export disabled" : "Export..."}
+ {exportError && <p className={styles.error}>{exportError}</p>}
+ {stepperState === "import_complete" && (
+ <button
+ className={styles.startExportButton}
+ onClick={handleStartExport}
+ >
+ Start Export
+ </button>
+ )}
+ {stepperState === "awaiting_export" && <ExportLoadingBars />}
+ {stepperState === "export_complete" && (
+ <>
+ <button className={styles.querySeafowlButton}>
+ Query Seafowl in Splitgraph Console
+ </button>
+ <button className={styles.viewReportButton}>View Report</button>
+ </>
+ )}
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css
new file mode 100644
index 0000000..9f75de7
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css
@@ -0,0 +1,17 @@
+/* components/ImportExportStepper/ExportTableLoadingBar.module.css */
+
+.exportTableLoadingBar {
+ background-color: inherit;
+}
+
+.loadingBar {
+ background-color: inherit;
+}
+
+.completedBar {
+ background-color: inherit;
+}
+
+.tableName {
+ background-color: inherit;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
new file mode 100644
index 0000000..76e8799
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
@@ -0,0 +1,78 @@
+import { useEffect } from "react";
+import { useStepper } from "./StepperContext";
+import styles from "./ExportTableLoadingBar.module.css";
+
+interface ExportTableLoadingBarProps {
+ tableName: string;
+ taskId: string;
+}
+
+export const ExportTableLoadingBar = ({
+ tableName,
+ taskId,
+}: React.PropsWithoutRef<ExportTableLoadingBarProps>) => {
+ const [{ stepperState, exportedTablesLoading }, dispatch] = useStepper();
+
+ useEffect(() => {
+ if (!taskId || !tableName) {
+ console.log("Don't check export until we have taskId and tableName");
+ console.table({
+ taskId,
+ tableName,
+ });
+ return;
+ }
+
+ if (stepperState !== "awaiting_export") {
+ console.log("Done waiting for export");
+ return;
+ }
+
+ const pollExportTask = async () => {
+ try {
+ const response = await fetch("/api/await-export-to-seafowl-task", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ taskId,
+ }),
+ });
+ const data = await response.json();
+
+ if (data.completed) {
+ dispatch({
+ type: "export_table_task_complete",
+ completedTable: { tableName, taskId },
+ });
+ } else if (data.error) {
+ throw new Error(data.error);
+ }
+ } catch (error) {
+ dispatch({
+ type: "export_error",
+ error: `Error exporting ${tableName}: ${error.message}`,
+ });
+ }
+ };
+
+ const interval = setInterval(pollExportTask, 3000);
+ return () => clearInterval(interval);
+ }, [stepperState, tableName, taskId, dispatch]);
+
+ const isLoading = !!Array.from(exportedTablesLoading).find(
+ (t) => t.taskId === taskId
+ );
+
+ return (
+ <div className={styles.exportTableLoadingBar}>
+ <div className={styles.loadingBar}>
+ {isLoading
+ ? `Loading ${tableName}...`
+ : `Successfully exported ${tableName}`}
+ </div>
+ <div className={styles.tableName}>{tableName}</div>
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
index 8130fed..3795a61 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
@@ -51,7 +51,6 @@ export const ImportLoadingBar: React.FC<ImportLoadingBarProps> = ({
dispatch({ type: "import_error", error: data.error });
}
} catch (error) {
- console.error("Error occurred during import task status check:", error);
dispatch({
type: "import_error",
error: "An error occurred during the import process",
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
index 7da3725..b57f2ef 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
@@ -6,7 +6,13 @@ import styles from "./ImportPanel.module.css";
export const ImportPanel = () => {
const [
- { stepperState, taskId, error, splitgraphNamespace, splitgraphRepository },
+ {
+ stepperState,
+ importTaskId,
+ importError,
+ splitgraphNamespace,
+ splitgraphRepository,
+ },
dispatch,
] = useStepper();
const [inputValue, setInputValue] = useState("");
@@ -76,7 +82,7 @@ export const ImportPanel = () => {
<div className={styles.importPanel}>
{stepperState === "unstarted" && (
<>
- {error && <p className={styles.error}>{error}</p>}
+ {importError && <p className={styles.error}>{importError}</p>}
<form onSubmit={handleInputSubmit}>
<input
type="text"
@@ -90,7 +96,7 @@ export const ImportPanel = () => {
)}
{stepperState === "awaiting_import" && (
<ImportLoadingBar
- taskId={taskId}
+ taskId={importTaskId}
splitgraphNamespace={splitgraphNamespace}
splitgraphRepository={splitgraphRepository}
/>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index 180c0fb..c7621a3 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -1,7 +1,8 @@
// stepper-states.ts
export type GitHubRepository = { namespace: string; repository: string };
-// Define the state
+type ExportTable = { tableName: string; taskId: string };
+
export type StepperState = {
stepperState:
| "unstarted"
@@ -10,14 +11,15 @@ export type StepperState = {
| "awaiting_export"
| "export_complete";
repository?: GitHubRepository | null;
- taskId?: string | null;
- error?: string;
- tables?: { taskId: string }[] | null;
+ importTaskId?: string | null;
+ importError?: string;
splitgraphRepository?: string;
splitgraphNamespace?: string;
+ exportedTablesLoading?: Set<ExportTable>;
+ exportedTablesCompleted?: Set<ExportTable>;
+ exportError?: string;
};
-// Define the actions
export type StepperAction =
| {
type: "start_import";
@@ -27,34 +29,45 @@ export type StepperAction =
splitgraphNamespace: string;
}
| { type: "import_complete" }
- | { type: "start_export"; tables: { taskId: string }[] }
+ | { type: "start_export"; tables: ExportTable[] }
+ | { type: "export_table_task_complete"; completedTable: ExportTable }
| { type: "export_complete" }
+ | { type: "export_error"; error: string }
| { type: "import_error"; error: string }
| { type: "reset" };
-// Initial state
export const initialState: StepperState = {
stepperState: "unstarted",
repository: null,
splitgraphRepository: null,
splitgraphNamespace: null,
- taskId: null,
- tables: null,
+ importTaskId: null,
+ exportedTablesLoading: new Set<ExportTable>(),
+ exportedTablesCompleted: new Set<ExportTable>(),
+ importError: null,
+ exportError: null,
};
+// FOR DEBUGGING: uncomment for hardcoded state initialization
+// export const initialState: StepperState = {
+// ...normalInitialState,
+// stepperState: "import_complete",
+// splitgraphNamespace: "miles",
+// splitgraphRepository: "import-via-nextjs",
+// };
+
// Reducer function
export const stepperReducer = (
state: StepperState,
action: StepperAction
): StepperState => {
- console.log("Got action", action, "prev state:", state);
switch (action.type) {
case "start_import":
return {
...state,
stepperState: "awaiting_import",
repository: action.repository,
- taskId: action.taskId,
+ importTaskId: action.taskId,
splitgraphNamespace: action.splitgraphNamespace,
splitgraphRepository: action.splitgraphRepository,
};
@@ -64,11 +77,45 @@ export const stepperReducer = (
stepperState: "import_complete",
};
case "start_export":
+ const { tables } = action;
+ const exportedTablesLoading = new Set<ExportTable>();
+ const exportedTablesCompleted = new Set<ExportTable>();
+
+ for (const { tableName, taskId } of tables) {
+ exportedTablesLoading.add({ tableName, taskId });
+ }
+
return {
...state,
+ exportedTablesLoading,
+ exportedTablesCompleted,
stepperState: "awaiting_export",
- tables: action.tables,
};
+
+ case "export_table_task_complete":
+ const { completedTable } = action;
+
+ // We're storing a set of completedTable objects, so we need to find the matching one to remove it
+ const loadingTablesAfterRemoval = new Set(state.exportedTablesLoading);
+ const loadingTabletoRemove = Array.from(loadingTablesAfterRemoval).find(
+ ({ taskId }) => taskId === completedTable.taskId
+ );
+ loadingTablesAfterRemoval.delete(loadingTabletoRemove);
+
+ // Then we can add the matching one to the completed table
+ const completedTablesAfterAdded = new Set(state.exportedTablesCompleted);
+ completedTablesAfterAdded.add(completedTable);
+
+ return {
+ ...state,
+ exportedTablesLoading: loadingTablesAfterRemoval,
+ exportedTablesCompleted: completedTablesAfterAdded,
+ stepperState:
+ loadingTablesAfterRemoval.size === 0
+ ? "export_complete"
+ : "awaiting_export",
+ };
+
case "export_complete":
return {
...state,
@@ -79,9 +126,17 @@ export const stepperReducer = (
...state,
splitgraphRepository: null,
splitgraphNamespace: null,
- taskId: null,
+ importTaskId: null,
stepperState: "unstarted",
- error: action.error,
+ importError: action.error,
+ };
+ case "export_error":
+ return {
+ ...state,
+ exportedTablesLoading: new Set<ExportTable>(),
+ exportedTablesCompleted: new Set<ExportTable>(),
+ stepperState: "import_complete",
+ exportError: action.error,
};
case "reset":
From 8ba415160b343ea45703b4b8106c6194054a8aa8 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Fri, 26 May 2023 23:36:19 +0100
Subject: [PATCH 07/36] Move lib-backend to lib/backend
---
.../{lib-backend => lib/backend}/splitgraph-db.ts | 0
.../pages/api/await-export-to-seafowl-task.ts | 2 +-
.../pages/api/await-import-from-github.ts | 2 +-
.../pages/api/start-export-to-seafowl.ts | 2 +-
.../pages/api/start-import-from-github.ts | 2 +-
5 files changed, 4 insertions(+), 4 deletions(-)
rename examples/nextjs-import-airbyte-github-export-seafowl/{lib-backend => lib/backend}/splitgraph-db.ts (100%)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
similarity index 100%
rename from examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts
rename to examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts
index 93f6b89..61a7ae1 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts
@@ -1,5 +1,5 @@
import type { NextApiRequest, NextApiResponse } from "next";
-import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db";
import type { DeferredSplitgraphExportTask } from "@madatdata/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin";
type ResponseData =
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts
index d06aac9..4563044 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts
@@ -1,5 +1,5 @@
import type { NextApiRequest, NextApiResponse } from "next";
-import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db";
import type { DeferredSplitgraphImportTask } from "@madatdata/db-splitgraph/plugins/importers/splitgraph-base-import-plugin";
type ResponseData =
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
index 75d6343..4e95752 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
@@ -1,5 +1,5 @@
import type { NextApiRequest, NextApiResponse } from "next";
-import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db";
+import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db";
type ResponseData =
| {
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
index 01c2b58..147aeba 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from "next";
import {
makeAuthenticatedSplitgraphDb,
claimsFromJWT,
-} from "../../lib-backend/splitgraph-db";
+} from "../../lib/backend/splitgraph-db";
const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET;
From 2dcec66809ccc1b096f44dee601bda5118d119f8 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Fri, 26 May 2023 23:58:26 +0100
Subject: [PATCH 08/36] Hardcode list of "relevant" table names for ingestion
from GitHub
The `airbyte-github` plugin by default imports 163 tables into
Splitgraph, but we only need a few of them for the analytics queries
we want to make in the demo app. So, hardcode the list of those,
but also hardcode the list of all 163 tables for reference, and also
the 43 tables that are imported given the relevant tables (because either
they depend on them via a foreign key relationship, or they're an
airbyte meta table).
For the 43 tables, see this recent import of `splitgraph/seafowl`:
* https://www.splitgraph.com/miles/github-import-splitgraph-seafowl/20230526-224723/-/tables
This took 3 minutes and 40 seconds to import into Splitgraph.
---
.../ImportExportStepper/ExportPanel.tsx | 9 +-
.../lib/config.ts | 249 ++++++++++++++++++
.../pages/api/start-import-from-github.ts | 12 +-
3 files changed, 256 insertions(+), 14 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index 8152602..ceb29f6 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -4,12 +4,7 @@ import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
import { ExportLoadingBars } from "./ExportLoadingBars";
-// TODO: don't hardcode this? or at least hardcode all of them and make it official
-const importedTableNames = [
- "stargazers",
- // NOTE: If we only specify stargazers, then stargazers_user is still included since it's a dependent table
- "stargazers_user",
-];
+import { relevantGitHubTableNames } from "../../lib/config";
export const ExportPanel = () => {
const [
@@ -22,7 +17,7 @@ export const ExportPanel = () => {
const response = await fetch("/api/start-export-to-seafowl", {
method: "POST",
body: JSON.stringify({
- tables: importedTableNames.map((tableName) => ({
+ tables: relevantGitHubTableNames.map((tableName) => ({
namespace: splitgraphNamespace,
repository: splitgraphRepository,
table: tableName,
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts
new file mode 100644
index 0000000..8f6719a
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts
@@ -0,0 +1,249 @@
+/**
+ * List of GitHub table names that we want to import with the Airbyte connector
+ * into Splitgraph. By default, there are 163 tables available. But we only want
+ * some of them, and by selecting them explicitly, the import will be much faster,
+ * especially for large repositories.
+ *
+ * Note that Airbyte will still import tables that depend on these tables due
+ * to foreign keys, and will also import airbyte metaata tables.
+ */
+export const relevantGitHubTableNames = `commits
+comments
+pull_requests
+pull_request_stats
+issue_reactions`
+ .split("\n")
+ .filter((t) => !!t);
+
+/**
+ * List of "downstream" GitHub table names that will be imported by default by
+ * the `airbyte-github` connector, given the list of `relevantGitHubTableNames`,
+ * because they're either an Airbyte meta table or a table that depends on
+ * one of the "relevant" tables.
+ *
+ * This is manually curated and might not be totally accurate. It's up to date
+ * given the following list of `relevantGitHubTableNames`:
+ *
+ * ```
+ * commits
+ * comments
+ * pull_requests
+ * pull_request_stats
+ * issue_reactions
+ * ```
+ */
+export const expectedImportedTableNames = `_airbyte_raw_comments
+_airbyte_raw_commits
+_airbyte_raw_issue_reactions
+_airbyte_raw_pull_request_stats
+_airbyte_raw_pull_requests
+_sg_ingestion_state
+comments
+comments_user
+commits
+commits_author
+commits_commit
+commits_commit_author
+commits_commit_committer
+commits_commit_tree
+commits_commit_verification
+commits_committer
+commits_parents
+issue_reactions
+issue_reactions_user
+pull_request_stats
+pull_request_stats_merged_by
+pull_requests
+pull_requests__links
+pull_requests__links_comments
+pull_requests__links_commits
+pull_requests__links_html
+pull_requests__links_issue
+pull_requests__links_review_comment
+pull_requests__links_review_comments
+pull_requests__links_self
+pull_requests__links_statuses
+pull_requests_assignee
+pull_requests_assignees
+pull_requests_auto_merge
+pull_requests_auto_merge_enabled_by
+pull_requests_base
+pull_requests_head
+pull_requests_labels
+pull_requests_milestone
+pull_requests_milestone_creator
+pull_requests_requested_reviewers
+pull_requests_requested_teams
+pull_requests_user
+`;
+
+/**
+ * This is the list of all tables imported by Airbyte by default when no tables
+ * are explicitly provided to the plugin.
+ *
+ * This is not consumed anywhere, but is useful for referencing, and if you'd
+ * like to extend or modify the code, you can choose tables from here to include.
+ */
+export const allGitHubTableNames = `_airbyte_raw_assignees
+_airbyte_raw_branches
+_airbyte_raw_collaborators
+_airbyte_raw_comments
+_airbyte_raw_commit_comment_reactions
+_airbyte_raw_commit_comments
+_airbyte_raw_commits
+_airbyte_raw_deployments
+_airbyte_raw_events
+_airbyte_raw_issue_comment_reactions
+_airbyte_raw_issue_events
+_airbyte_raw_issue_labels
+_airbyte_raw_issue_milestones
+_airbyte_raw_issue_reactions
+_airbyte_raw_issues
+_airbyte_raw_organizations
+_airbyte_raw_project_cards
+_airbyte_raw_project_columns
+_airbyte_raw_projects
+_airbyte_raw_pull_request_comment_reactions
+_airbyte_raw_pull_request_commits
+_airbyte_raw_pull_request_stats
+_airbyte_raw_pull_requests
+_airbyte_raw_releases
+_airbyte_raw_repositories
+_airbyte_raw_review_comments
+_airbyte_raw_reviews
+_airbyte_raw_stargazers
+_airbyte_raw_tags
+_airbyte_raw_team_members
+_airbyte_raw_team_memberships
+_airbyte_raw_teams
+_airbyte_raw_users
+_airbyte_raw_workflow_jobs
+_airbyte_raw_workflow_runs
+_airbyte_raw_workflows
+_sg_ingestion_state
+assignees
+branches
+branches_commit
+branches_protection
+branches_protection_required_status_checks
+collaborators
+collaborators_permissions
+comments
+comments_user
+commit_comment_reactions
+commit_comment_reactions_user
+commit_comments
+commit_comments_user
+commits
+commits_author
+commits_commit
+commits_commit_author
+commits_commit_committer
+commits_commit_tree
+commits_commit_verification
+commits_committer
+commits_parents
+deployments
+deployments_creator
+events
+events_actor
+events_org
+events_repo
+issue_comment_reactions
+issue_comment_reactions_user
+issue_events
+issue_events_actor
+issue_events_issue
+issue_events_issue_user
+issue_labels
+issue_milestones
+issue_milestones_creator
+issue_reactions
+issue_reactions_user
+issues
+issues_assignee
+issues_assignees
+issues_labels
+issues_milestone
+issues_milestone_creator
+issues_pull_request
+issues_user
+organizations
+organizations_plan
+project_cards
+project_cards_creator
+project_columns
+projects
+projects_creator
+pull_request_comment_reactions
+pull_request_comment_reactions_user
+pull_request_commits
+pull_request_commits_author
+pull_request_commits_commit
+pull_request_commits_commit_author
+pull_request_commits_commit_committer
+pull_request_commits_commit_tree
+pull_request_commits_commit_verification
+pull_request_commits_committer
+pull_request_commits_parents
+pull_request_stats
+pull_request_stats_merged_by
+pull_requests
+pull_requests__links
+pull_requests__links_comments
+pull_requests__links_commits
+pull_requests__links_html
+pull_requests__links_issue
+pull_requests__links_review_comment
+pull_requests__links_review_comments
+pull_requests__links_self
+pull_requests__links_statuses
+pull_requests_assignee
+pull_requests_assignees
+pull_requests_auto_merge
+pull_requests_auto_merge_enabled_by
+pull_requests_base
+pull_requests_head
+pull_requests_labels
+pull_requests_milestone
+pull_requests_milestone_creator
+pull_requests_requested_reviewers
+pull_requests_requested_teams
+pull_requests_user
+releases
+releases_assets
+releases_author
+repositories
+repositories_license
+repositories_owner
+repositories_permissions
+review_comments
+review_comments__links
+review_comments__links_html
+review_comments__links_pull_request
+review_comments__links_self
+review_comments_user
+reviews
+reviews__links
+reviews__links_html
+reviews__links_pull_request
+reviews_user
+stargazers
+stargazers_user
+tags
+tags_commit
+team_members
+team_memberships
+teams
+users
+workflow_jobs
+workflow_jobs_steps
+workflow_runs
+workflow_runs_head_commit
+workflow_runs_head_commit_author
+workflow_runs_head_commit_committer
+workflow_runs_head_repository
+workflow_runs_head_repository_owner
+workflow_runs_repository
+workflow_runs_repository_owner
+workflows`;
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
index 147aeba..dc78648 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
@@ -3,6 +3,7 @@ import {
makeAuthenticatedSplitgraphDb,
claimsFromJWT,
} from "../../lib/backend/splitgraph-db";
+import { relevantGitHubTableNames } from "../../lib/config";
const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET;
@@ -100,14 +101,11 @@ const startImport = async ({
namespace: splitgraphNamespace,
repository: splitgraphDestinationRepository,
tables: [
- {
- name: "stargazers",
- options: {
- airbyte_cursor_field: ["starred_at"],
- airbyte_primary_key_field: [],
- },
+ ...relevantGitHubTableNames.map((t) => ({
+ name: t,
+ options: {},
schema: [],
- },
+ })),
],
},
{ defer: true }
From 65ca23e59190d90e80781382d4d50d3b41b328fd Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Sat, 27 May 2023 00:00:39 +0100
Subject: [PATCH 09/36] Serialize the stepper state into the URL, so that
awaiting can be resumed across page loads
Keep track of the current stepper state (e.g. taskId, import completion, etc.)
in the URL. Update the URL when the state changes, and initialize the state
from the URL on page load. Note that we need to default to an "uninitialized"
state, and then update the state from the URL via an `initialize_from_url` action, because
the `useRouter` hook is ansynchronous, and we don't look at query parameters on
the server side with `getInitialProps` or similar. Thus we can show a loading bar
before showing the import form (or whatever we're showing based on the current state).
This makes development easier, since after a long import we can refresh the page
with the URL containing the task ID and start from there, rather than re-importing
every time. And it also makes it easier for users who can refresh the page without
losing progress if an import has already started (it will just poll the taskId from
the URL).
---
.../ImportExportStepper/DebugPanel.tsx | 13 ++
.../ExportTableLoadingBar.tsx | 6 +-
.../ImportExportStepper/Stepper.tsx | 22 +-
.../ImportExportStepper/StepperContext.tsx | 5 +-
.../ImportExportStepper/stepper-states.ts | 198 +++++++++++++++++-
5 files changed, 230 insertions(+), 14 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
new file mode 100644
index 0000000..d7306b4
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
@@ -0,0 +1,13 @@
+import { useStepper } from "./StepperContext";
+
+export const DebugPanel = () => {
+ const [state, _] = useStepper();
+
+ return (
+ <div>
+ <pre style={{ minWidth: "80%", minHeight: "300px" }}>
+ {JSON.stringify(state, null, 2)}
+ </pre>
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
index 76e8799..7551b58 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
@@ -47,7 +47,11 @@ export const ExportTableLoadingBar = ({
completedTable: { tableName, taskId },
});
} else if (data.error) {
- throw new Error(data.error);
+ if (!data.completed) {
+ console.log("WARN: Failed status, not completed:", data.error);
+ } else {
+ throw new Error(data.error);
+ }
}
} catch (error) {
dispatch({
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
index 6af0f06..93f987f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
@@ -1,15 +1,27 @@
-import { StepperContextProvider } from "./StepperContext";
-import { ImportPanel } from "./ImportPanel"; // will create this component later
-import { ExportPanel } from "./ExportPanel"; // will create this component later
+import { StepperContextProvider, useStepper } from "./StepperContext";
+import { DebugPanel } from "./DebugPanel";
+import { ImportPanel } from "./ImportPanel";
+import { ExportPanel } from "./ExportPanel";
import styles from "./Stepper.module.css";
+const StepperOrLoading = ({ children }: { children: React.ReactNode }) => {
+ const [{ stepperState }] = useStepper();
+
+ return (
+ <>{stepperState === "uninitialized" ? <div>........</div> : children}</>
+ );
+};
+
export const Stepper = () => {
return (
<StepperContextProvider>
<div className={styles.stepper}>
- <ImportPanel />
- <ExportPanel />
+ <StepperOrLoading>
+ <DebugPanel />
+ <ImportPanel />
+ <ExportPanel />
+ </StepperOrLoading>
</div>
</StepperContextProvider>
);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
index cbff3a1..0fdb7c0 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
@@ -3,8 +3,7 @@ import React, { useReducer, useContext, ReactNode } from "react";
import {
StepperState,
StepperAction,
- initialState,
- stepperReducer,
+ useStepperReducer,
} from "./stepper-states";
// Define the context
@@ -15,7 +14,7 @@ const StepperContext = React.createContext<
export const StepperContextProvider: React.FC<{ children: ReactNode }> = ({
children,
}) => {
- const [state, dispatch] = useReducer(stepperReducer, initialState);
+ const [state, dispatch] = useStepperReducer();
return (
<StepperContext.Provider value={[state, dispatch]}>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index c7621a3..c8fdaf1 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -1,10 +1,13 @@
-// stepper-states.ts
+import { useRouter, type NextRouter } from "next/router";
+import { ParsedUrlQuery } from "querystring";
+import { useEffect, useReducer } from "react";
export type GitHubRepository = { namespace: string; repository: string };
type ExportTable = { tableName: string; taskId: string };
export type StepperState = {
stepperState:
+ | "uninitialized"
| "unstarted"
| "awaiting_import"
| "import_complete"
@@ -34,9 +37,15 @@ export type StepperAction =
| { type: "export_complete" }
| { type: "export_error"; error: string }
| { type: "import_error"; error: string }
- | { type: "reset" };
+ | { type: "reset" }
+ | { type: "initialize_from_url"; parsedFromUrl: StepperState };
-export const initialState: StepperState = {
+type ExtractStepperAction<T extends StepperAction["type"]> = Extract<
+ StepperAction,
+ { type: T }
+>;
+
+const initialState: StepperState = {
stepperState: "unstarted",
repository: null,
splitgraphRepository: null,
@@ -56,8 +65,128 @@ export const initialState: StepperState = {
// splitgraphRepository: "import-via-nextjs",
// };
-// Reducer function
-export const stepperReducer = (
+type ActionParams<T extends StepperAction["type"]> = Omit<
+ ExtractStepperAction<T>,
+ "type"
+>;
+
+const getQueryParamAsString = <T extends string = string>(
+ query: ParsedUrlQuery,
+ key: string
+): T | null => {
+ if (Array.isArray(query[key]) && query[key].length > 0) {
+ throw new Error(`expected only one query param but got multiple: ${key}`);
+ }
+
+ if (!(key in query)) {
+ return null;
+ }
+
+ return query[key] as T;
+};
+
+const queryParamParsers: {
+ [K in keyof StepperState]: (query: ParsedUrlQuery) => StepperState[K];
+} = {
+ stepperState: (query) =>
+ getQueryParamAsString<StepperState["stepperState"]>(
+ query,
+ "stepperState"
+ ) ?? "unstarted",
+ repository: (query) => ({
+ namespace: getQueryParamAsString(query, "githubNamespace"),
+ repository: getQueryParamAsString(query, "githubRepository"),
+ }),
+ importTaskId: (query) => getQueryParamAsString(query, "importTaskId"),
+ importError: (query) => getQueryParamAsString(query, "importError"),
+ exportError: (query) => getQueryParamAsString(query, "exportError"),
+ splitgraphNamespace: (query) =>
+ getQueryParamAsString(query, "splitgraphNamespace"),
+ splitgraphRepository: (query) =>
+ getQueryParamAsString(query, "splitgraphRepository"),
+};
+
+const requireKeys = <T extends Record<string, unknown>>(
+ obj: T,
+ requiredKeys: (keyof T)[]
+) => {
+ const missingKeys = requiredKeys.filter(
+ (requiredKey) => !(requiredKey in obj)
+ );
+
+ if (missingKeys.length > 0) {
+ throw new Error("missing required keys: " + missingKeys.join(", "));
+ }
+};
+
+const stepperStateValidators: {
+ [K in StepperState["stepperState"]]: (stateFromQuery: StepperState) => void;
+} = {
+ uninitialized: () => {},
+ unstarted: () => {},
+ awaiting_import: (stateFromQuery) =>
+ requireKeys(stateFromQuery, [
+ "repository",
+ "importTaskId",
+ "splitgraphNamespace",
+ "splitgraphRepository",
+ ]),
+ import_complete: (stateFromQuery) =>
+ requireKeys(stateFromQuery, [
+ "repository",
+ "splitgraphNamespace",
+ "splitgraphRepository",
+ ]),
+ awaiting_export: (stateFromQuery) =>
+ requireKeys(stateFromQuery, [
+ "repository",
+ "splitgraphNamespace",
+ "splitgraphRepository",
+ ]),
+ export_complete: (stateFromQuery) =>
+ requireKeys(stateFromQuery, [
+ "repository",
+ "splitgraphNamespace",
+ "splitgraphRepository",
+ ]),
+};
+
+const parseStateFromRouter = (router: NextRouter): StepperState => {
+ const { query } = router;
+
+ const stepperState = queryParamParsers.stepperState(query);
+
+ const stepper = {
+ stepperState: stepperState,
+ repository: queryParamParsers.repository(query),
+ importTaskId: queryParamParsers.importTaskId(query),
+ importError: queryParamParsers.importError(query),
+ exportError: queryParamParsers.exportError(query),
+ splitgraphNamespace: queryParamParsers.splitgraphNamespace(query),
+ splitgraphRepository: queryParamParsers.splitgraphRepository(query),
+ };
+
+ void stepperStateValidators[stepperState](stepper);
+
+ return stepper;
+};
+
+const serializeStateToQueryParams = (stepper: StepperState) => {
+ return JSON.parse(
+ JSON.stringify({
+ stepperState: stepper.stepperState,
+ githubNamespace: stepper.repository?.namespace ?? undefined,
+ githubRepository: stepper.repository?.repository ?? undefined,
+ importTaskId: stepper.importTaskId ?? undefined,
+ importError: stepper.importError ?? undefined,
+ exportError: stepper.exportError ?? undefined,
+ splitgraphNamespace: stepper.splitgraphNamespace ?? undefined,
+ splitgraphRepository: stepper.splitgraphRepository ?? undefined,
+ })
+ );
+};
+
+const stepperReducer = (
state: StepperState,
action: StepperAction
): StepperState => {
@@ -142,7 +271,66 @@ export const stepperReducer = (
case "reset":
return initialState;
+ case "initialize_from_url":
+ return {
+ ...state,
+ ...action.parsedFromUrl,
+ };
+
default:
return state;
}
};
+
+const urlNeedsChange = (state: StepperState, router: NextRouter) => {
+ const parsedFromUrl = parseStateFromRouter(router);
+
+ return (
+ state.stepperState !== parsedFromUrl.stepperState ||
+ state.repository?.namespace !== parsedFromUrl.repository?.namespace ||
+ state.repository?.repository !== parsedFromUrl.repository?.repository ||
+ state.importTaskId !== parsedFromUrl.importTaskId ||
+ state.splitgraphNamespace !== parsedFromUrl.splitgraphNamespace ||
+ state.splitgraphRepository !== parsedFromUrl.splitgraphRepository
+ );
+};
+
+export const useStepperReducer = () => {
+ const router = useRouter();
+ const [state, dispatch] = useReducer(stepperReducer, {
+ ...initialState,
+ stepperState: "uninitialized",
+ });
+
+ useEffect(() => {
+ dispatch({
+ type: "initialize_from_url",
+ parsedFromUrl: parseStateFromRouter(router),
+ });
+ }, [router.query]);
+
+ useEffect(() => {
+ if (!urlNeedsChange(state, router)) {
+ return;
+ }
+
+ if (state.stepperState === "uninitialized") {
+ return;
+ }
+
+ console.log("push", {
+ pathname: router.pathname,
+ query: serializeStateToQueryParams(state),
+ });
+ router.push(
+ {
+ pathname: router.pathname,
+ query: serializeStateToQueryParams(state),
+ },
+ undefined,
+ { shallow: true }
+ );
+ }, [state.stepperState]);
+
+ return [state, dispatch] as const;
+};
From 9e47b5ad014e62e7346e0493d166454ba150d6c9 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Sat, 27 May 2023 00:19:57 +0100
Subject: [PATCH 10/36] Move `lib/config.ts` -> `lib/config/github-tables.ts`
---
.../components/ImportExportStepper/ExportPanel.tsx | 2 +-
.../lib/{config.ts => config/github-tables.ts} | 0
.../pages/api/start-import-from-github.ts | 2 +-
3 files changed, 2 insertions(+), 2 deletions(-)
rename examples/nextjs-import-airbyte-github-export-seafowl/lib/{config.ts => config/github-tables.ts} (100%)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index ceb29f6..6ae7ebd 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -4,7 +4,7 @@ import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
import { ExportLoadingBars } from "./ExportLoadingBars";
-import { relevantGitHubTableNames } from "../../lib/config";
+import { relevantGitHubTableNames } from "../../lib/config/github-tables";
export const ExportPanel = () => {
const [
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
similarity index 100%
rename from examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts
rename to examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
index dc78648..506c529 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
@@ -3,7 +3,7 @@ import {
makeAuthenticatedSplitgraphDb,
claimsFromJWT,
} from "../../lib/backend/splitgraph-db";
-import { relevantGitHubTableNames } from "../../lib/config";
+import { relevantGitHubTableNames } from "../../lib/config/github-tables";
const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET;
From d60cd4ddba0e883a1ce0955155b70e9d01540961 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Sat, 27 May 2023 00:59:17 +0100
Subject: [PATCH 11/36] Export analytics queries to Seafowl in addition to
tables
Export queries to tables `monthly_user_stats` and `monthly_issue_stats`
in the same schema/namespace as the tables. We also export the tables,
or at least the few that we explicitly asked to import.
---
.../ImportExportStepper/ExportLoadingBars.tsx | 18 ++-
.../ImportExportStepper/ExportPanel.tsx | 78 ++++++++---
.../ExportTableLoadingBar.tsx | 33 +++--
.../ImportExportStepper/stepper-states.ts | 21 ++-
.../lib/config/github-tables.ts | 6 +-
.../lib/config/queries-to-export.ts | 116 ++++++++++++++++
.../pages/api/start-export-to-seafowl.ts | 127 +++++++++++++++---
.../pages/api/start-import-from-github.ts | 4 +-
8 files changed, 338 insertions(+), 65 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
index f830456..076cded 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
@@ -7,13 +7,17 @@ export const ExportLoadingBars = () => {
return (
<div className={styles.exportLoadingBars}>
- {Array.from(exportedTablesLoading).map(({ tableName, taskId }) => (
- <ExportTableLoadingBar
- key={taskId}
- tableName={tableName}
- taskId={taskId}
- />
- ))}
+ {Array.from(exportedTablesLoading).map(
+ ({ destinationSchema, destinationTable, sourceQuery, taskId }) => (
+ <ExportTableLoadingBar
+ key={taskId}
+ destinationSchema={destinationSchema}
+ destinationTable={destinationTable}
+ sourceQuery={sourceQuery}
+ taskId={taskId}
+ />
+ )
+ )}
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index 6ae7ebd..ad360be 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -4,7 +4,15 @@ import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
import { ExportLoadingBars } from "./ExportLoadingBars";
-import { relevantGitHubTableNames } from "../../lib/config/github-tables";
+import { relevantGitHubTableNamesForImport } from "../../lib/config/github-tables";
+import { makeQueriesToExport } from "../../lib/config/queries-to-export";
+import type {
+ ExportQueryInput,
+ ExportTableInput,
+ StartExportToSeafowlRequestShape,
+ StartExportToSeafowlResponseData,
+} from "../../pages/api/start-export-to-seafowl";
+import { useMemo, useCallback } from "react";
export const ExportPanel = () => {
const [
@@ -12,40 +20,76 @@ export const ExportPanel = () => {
dispatch,
] = useStepper();
- const handleStartExport = async () => {
+ const queriesToExport = useMemo<ExportQueryInput[]>(
+ () =>
+ makeQueriesToExport({
+ splitgraphSourceRepository: splitgraphRepository,
+ splitgraphSourceNamespace: splitgraphNamespace,
+ seafowlDestinationSchema: `${splitgraphNamespace}/${splitgraphRepository}`,
+ }),
+ [splitgraphRepository, splitgraphNamespace]
+ );
+
+ const tablesToExport = useMemo<ExportTableInput[]>(
+ () =>
+ relevantGitHubTableNamesForImport.map((tableName) => ({
+ namespace: splitgraphNamespace,
+ repository: splitgraphRepository,
+ table: tableName,
+ })),
+ [
+ splitgraphNamespace,
+ splitgraphRepository,
+ relevantGitHubTableNamesForImport,
+ ]
+ );
+
+ const handleStartExport = useCallback(async () => {
try {
const response = await fetch("/api/start-export-to-seafowl", {
method: "POST",
body: JSON.stringify({
- tables: relevantGitHubTableNames.map((tableName) => ({
- namespace: splitgraphNamespace,
- repository: splitgraphRepository,
- table: tableName,
- })),
- }),
+ tables: tablesToExport,
+ queries: queriesToExport,
+ } as StartExportToSeafowlRequestShape),
headers: {
"Content-Type": "application/json",
},
});
- const data = await response.json();
+ const data = (await response.json()) as StartExportToSeafowlResponseData;
+
+ if ("error" in data && data["error"]) {
+ throw new Error(data["error"]);
+ }
- if (!data.tables || !data.tables.length) {
+ if (!("tables" in data) || !("queries" in data)) {
throw new Error("Response missing tables");
}
dispatch({
type: "start_export",
- tables: data.tables.map(
- ({ tableName, taskId }: { tableName: string; taskId: string }) => ({
- taskId,
- tableName,
- })
- ),
+ tables: [
+ ...data["queries"].map(
+ ({ sourceQuery, taskId, destinationSchema, destinationTable }) => ({
+ taskId,
+ destinationTable,
+ destinationSchema,
+ sourceQuery,
+ })
+ ),
+ ...data["tables"].map(
+ ({ destinationTable, destinationSchema, taskId }) => ({
+ taskId,
+ destinationTable,
+ destinationSchema,
+ })
+ ),
+ ],
});
} catch (error) {
dispatch({ type: "export_error", error: error.message });
}
- };
+ }, [queriesToExport, tablesToExport, dispatch]);
return (
<div className={styles.exportPanel}>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
index 7551b58..945fbd3 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
@@ -3,22 +3,28 @@ import { useStepper } from "./StepperContext";
import styles from "./ExportTableLoadingBar.module.css";
interface ExportTableLoadingBarProps {
- tableName: string;
+ destinationTable: string;
+ destinationSchema: string;
+ sourceQuery?: string;
taskId: string;
}
export const ExportTableLoadingBar = ({
- tableName,
+ destinationTable,
+ destinationSchema,
+ sourceQuery,
taskId,
}: React.PropsWithoutRef<ExportTableLoadingBarProps>) => {
const [{ stepperState, exportedTablesLoading }, dispatch] = useStepper();
useEffect(() => {
- if (!taskId || !tableName) {
- console.log("Don't check export until we have taskId and tableName");
+ if (!taskId || !destinationTable) {
+ console.log(
+ "Don't check export until we have taskId and destinationTable"
+ );
console.table({
taskId,
- tableName,
+ destinationTable,
});
return;
}
@@ -44,7 +50,12 @@ export const ExportTableLoadingBar = ({
if (data.completed) {
dispatch({
type: "export_table_task_complete",
- completedTable: { tableName, taskId },
+ completedTable: {
+ destinationTable,
+ taskId,
+ destinationSchema,
+ sourceQuery,
+ },
});
} else if (data.error) {
if (!data.completed) {
@@ -56,14 +67,14 @@ export const ExportTableLoadingBar = ({
} catch (error) {
dispatch({
type: "export_error",
- error: `Error exporting ${tableName}: ${error.message}`,
+ error: `Error exporting ${destinationTable}: ${error.message}`,
});
}
};
const interval = setInterval(pollExportTask, 3000);
return () => clearInterval(interval);
- }, [stepperState, tableName, taskId, dispatch]);
+ }, [stepperState, destinationTable, taskId, dispatch]);
const isLoading = !!Array.from(exportedTablesLoading).find(
(t) => t.taskId === taskId
@@ -73,10 +84,10 @@ export const ExportTableLoadingBar = ({
<div className={styles.exportTableLoadingBar}>
<div className={styles.loadingBar}>
{isLoading
- ? `Loading ${tableName}...`
- : `Successfully exported ${tableName}`}
+ ? `Loading ${destinationTable}...`
+ : `Successfully exported ${destinationTable}`}
</div>
- <div className={styles.tableName}>{tableName}</div>
+ <div className={styles.destinationTable}>{destinationTable}</div>
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index c8fdaf1..16ca39a 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -3,7 +3,12 @@ import { ParsedUrlQuery } from "querystring";
import { useEffect, useReducer } from "react";
export type GitHubRepository = { namespace: string; repository: string };
-type ExportTable = { tableName: string; taskId: string };
+type ExportTable = {
+ destinationSchema: string;
+ destinationTable: string;
+ taskId: string;
+ sourceQuery?: string;
+};
export type StepperState = {
stepperState:
@@ -210,8 +215,18 @@ const stepperReducer = (
const exportedTablesLoading = new Set<ExportTable>();
const exportedTablesCompleted = new Set<ExportTable>();
- for (const { tableName, taskId } of tables) {
- exportedTablesLoading.add({ tableName, taskId });
+ for (const {
+ destinationTable,
+ destinationSchema,
+ sourceQuery,
+ taskId,
+ } of tables) {
+ exportedTablesLoading.add({
+ destinationTable,
+ destinationSchema,
+ sourceQuery,
+ taskId,
+ });
}
return {
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
index 8f6719a..92e3903 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
@@ -7,7 +7,7 @@
* Note that Airbyte will still import tables that depend on these tables due
* to foreign keys, and will also import airbyte metaata tables.
*/
-export const relevantGitHubTableNames = `commits
+export const relevantGitHubTableNamesForImport = `commits
comments
pull_requests
pull_request_stats
@@ -17,12 +17,12 @@ issue_reactions`
/**
* List of "downstream" GitHub table names that will be imported by default by
- * the `airbyte-github` connector, given the list of `relevantGitHubTableNames`,
+ * the `airbyte-github` connector, given the list of `relevantGitHubTableNamesForImport`,
* because they're either an Airbyte meta table or a table that depends on
* one of the "relevant" tables.
*
* This is manually curated and might not be totally accurate. It's up to date
- * given the following list of `relevantGitHubTableNames`:
+ * given the following list of `relevantGitHubTableNamesForImport`:
*
* ```
* commits
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
new file mode 100644
index 0000000..099d4f2
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
@@ -0,0 +1,116 @@
+/**
+ * Return a a list of queries to export from Splitgraph to Seafowl, given the
+ * source repository (where the GitHub data was imported into), and the destination
+ * schema (where the data will be exported to at Seafowl).
+ */
+export const makeQueriesToExport = ({
+ splitgraphSourceRepository,
+ splitgraphSourceNamespace,
+ seafowlDestinationSchema,
+ splitgraphSourceImageHashOrTag = "latest",
+}: {
+ splitgraphSourceNamespace: string;
+ splitgraphSourceRepository: string;
+ seafowlDestinationSchema: string;
+ splitgraphSourceImageHashOrTag?: string;
+}): {
+ sourceQuery: string;
+ destinationSchema: string;
+ destinationTable: string;
+}[] => [
+ {
+ destinationSchema: seafowlDestinationSchema,
+ destinationTable: "monthly_user_stats",
+ sourceQuery: `
+ WITH
+
+ commits AS (
+ SELECT
+ date_trunc('month', created_at) AS created_at_month,
+ author->>'login' AS username,
+ count(*) as no_commits
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".commits
+ GROUP BY 1, 2
+ ),
+
+ comments AS (
+ SELECT
+ date_trunc('month', created_at) AS created_at_month,
+ "user"->>'login' AS username,
+ count(*) filter (where exists(select regexp_matches(issue_url, '.*/pull/.*'))) as no_pull_request_comments,
+ count(*) filter (where exists(select regexp_matches(issue_url, '.*/issue/.*'))) as no_issue_comments,
+ sum(length(body)) as total_comment_length
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".comments
+ GROUP BY 1, 2
+ ),
+
+ pull_requests AS (
+ WITH pull_request_creator AS (
+ SELECT id, "user"->>'login' AS username
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_requests
+ )
+
+ SELECT
+ date_trunc('month', updated_at) AS created_at_month,
+ username,
+ count(*) filter (where merged = true) AS merged_pull_requests,
+ count(*) AS total_pull_requests,
+ sum(additions::integer) filter (where merged = true) AS lines_added,
+ sum(deletions::integer) filter (where merged = true) AS lines_deleted
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_request_stats
+ INNER JOIN pull_request_creator USING (id)
+ GROUP BY 1, 2
+ ),
+
+ all_months_users AS (
+ SELECT DISTINCT created_at_month, username FROM commits
+ UNION SELECT DISTINCT created_at_month, username FROM comments
+ UNION SELECT DISTINCT created_at_month, username FROM pull_requests
+ ),
+
+ user_stats AS (
+ SELECT
+ amu.created_at_month,
+ amu.username,
+ COALESCE(cmt.no_commits, 0) AS no_commits,
+ COALESCE(cmnt.no_pull_request_comments, 0) AS no_pull_request_comments,
+ COALESCE(cmnt.no_issue_comments, 0) AS no_issue_comments,
+ COALESCE(cmnt.total_comment_length, 0) AS total_comment_length,
+ COALESCE(pr.merged_pull_requests, 0) AS merged_pull_requests,
+ COALESCE(pr.total_pull_requests, 0) AS total_pull_requests,
+ COALESCE(pr.lines_added, 0) AS lines_added,
+ COALESCE(pr.lines_deleted, 0) AS lines_deleted
+
+ FROM all_months_users amu
+ LEFT JOIN commits cmt ON amu.created_at_month = cmt.created_at_month AND amu.username = cmt.username
+ LEFT JOIN comments cmnt ON amu.created_at_month = cmnt.created_at_month AND amu.username = cmnt.username
+ LEFT JOIN pull_requests pr ON amu.created_at_month = pr.created_at_month AND amu.username = pr.username
+
+ ORDER BY created_at_month ASC, username ASC
+ )
+
+ SELECT * FROM user_stats;
+`,
+ },
+ {
+ destinationSchema: seafowlDestinationSchema,
+ destinationTable: "monthly_issue_stats",
+ sourceQuery: `
+SELECT
+ issue_number,
+ date_trunc('month', created_at::TIMESTAMP) as created_at_month,
+ COUNT(*) AS total_reacts,
+ COUNT(*) FILTER (WHERE content = '+1') AS no_plus_one,
+ COUNT(*) FILTER (WHERE content = '-1') AS no_minus_one,
+ COUNT(*) FILTER (WHERE content = 'laugh') AS no_laugh,
+ COUNT(*) FILTER (WHERE content = 'confused') AS no_confused,
+ COUNT(*) FILTER (WHERE content = 'heart') AS no_heart,
+ COUNT(*) FILTER (WHERE content = 'hooray') AS no_hooray,
+ COUNT(*) FILTER (WHERE content = 'rocket') AS no_rocket,
+ COUNT(*) FILTER (WHERE content = 'eyes') AS no_eyes
+FROM
+ "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}"."issue_reactions"
+GROUP BY 1, 2 ORDER BY 2, 3 DESC;
+`,
+ },
+];
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
index 4e95752..f616d37 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
@@ -1,16 +1,40 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db";
-type ResponseData =
+export type StartExportToSeafowlRequestShape =
+ | {
+ tables: ExportTableInput[];
+ }
+ | { queries: ExportQueryInput[] }
+ | { tables: ExportTableInput[]; queries: ExportQueryInput[] };
+
+export type StartExportToSeafowlResponseData =
| {
tables: {
- tableName: string;
+ destinationTable: string;
+ destinationSchema: string;
+ taskId: string;
+ }[];
+ queries: {
+ sourceQuery: string;
+ destinationSchema: string;
+ destinationTable: string;
taskId: string;
}[];
}
| { error: string };
-type TableInput = { namespace: string; repository: string; table: string };
+export type ExportTableInput = {
+ namespace: string;
+ repository: string;
+ table: string;
+};
+
+export type ExportQueryInput = {
+ sourceQuery: string;
+ destinationSchema: string;
+ destinationTable: string;
+};
/**
* To manually send a request, example:
@@ -23,16 +47,22 @@ curl -i \
*/
export default async function handler(
req: NextApiRequest,
- res: NextApiResponse<ResponseData>
+ res: NextApiResponse<StartExportToSeafowlResponseData>
) {
const db = makeAuthenticatedSplitgraphDb();
- const { tables } = req.body;
+ const { tables = [], queries = [] } = req.body;
+
+ if (tables.length === 0 && queries.length === 0) {
+ res.status(400).json({ error: "no tables or queries provided for export" });
+ return;
+ }
+
+ const errors = [];
if (
- !tables ||
- !tables.length ||
+ tables.length > 0 &&
!tables.every(
- (t: TableInput) =>
+ (t: ExportTableInput) =>
t.namespace &&
t.repository &&
t.table &&
@@ -41,17 +71,39 @@ export default async function handler(
typeof t.table === "string"
)
) {
- res.status(400).json({ error: "invalid tables input in request body" });
+ errors.push("invalid tables input in request body");
+ }
+
+ if (
+ queries.length > 0 &&
+ !queries.every(
+ (q: ExportQueryInput) =>
+ q.sourceQuery &&
+ q.destinationSchema &&
+ q.destinationTable &&
+ typeof q.sourceQuery === "string" &&
+ typeof q.destinationSchema === "string" &&
+ typeof q.destinationTable === "string"
+ )
+ ) {
+ errors.push("invalid queries input in request body");
+ }
+
+ if (errors.length > 0) {
+ res.status(400).json({ error: `Invalid request: ${errors.join(", ")}` });
return;
}
try {
- const exportingTables = await startExport({
- db,
- tables,
- });
+ const { tables: exportingTables, queries: exportingQueries } =
+ await startExport({
+ db,
+ tables,
+ queries,
+ });
res.status(200).json({
tables: exportingTables,
+ queries: exportingQueries,
});
} catch (err) {
res.status(400).json({
@@ -63,15 +115,26 @@ export default async function handler(
const startExport = async ({
db,
tables,
+ queries,
}: {
db: ReturnType<typeof makeAuthenticatedSplitgraphDb>;
- tables: TableInput[];
+ tables: ExportTableInput[];
+ queries: ExportQueryInput[];
}) => {
await db.fetchAccessToken();
const response = await db.exportData(
"export-to-seafowl",
{
+ queries: queries.map((query) => ({
+ source: {
+ query: query.sourceQuery,
+ },
+ destination: {
+ schema: query.destinationSchema,
+ table: query.destinationTable,
+ },
+ })),
tables: tables.map((splitgraphSource) => ({
source: {
repository: splitgraphSource.repository,
@@ -91,13 +154,33 @@ const startExport = async ({
throw new Error(JSON.stringify(response.error));
}
- const loadingTables: { taskId: string; tableName: string }[] =
- response.taskIds.tables.map(
- (t: { jobId: string; sourceTable: string }) => ({
- taskId: t.jobId,
- tableName: t.sourceTable,
- })
- );
+ const loadingTables = response.taskIds.tables.map(
+ (t: { jobId: string; sourceTable: string; sourceRepository: string }) => ({
+ taskId: t.jobId,
+ destinationTable: t.sourceTable,
+ destinationSchema: t.sourceRepository,
+ })
+ );
+
+ const loadingQueries = response.taskIds.queries.map(
+ (
+ queryJob: {
+ jobId: string;
+ destinationSchema: string;
+ destinationTable: string;
+ sourceQuery: string;
+ },
+ i: number
+ ) => ({
+ taskId: queryJob.jobId,
+ destinationSchema: queries[i].destinationSchema,
+ destinationTable: queries[i].destinationTable,
+ sourceQuery: queries[i].sourceQuery,
+ })
+ );
- return loadingTables;
+ return {
+ tables: loadingTables,
+ queries: loadingQueries,
+ };
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
index 506c529..5f7c079 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
@@ -3,7 +3,7 @@ import {
makeAuthenticatedSplitgraphDb,
claimsFromJWT,
} from "../../lib/backend/splitgraph-db";
-import { relevantGitHubTableNames } from "../../lib/config/github-tables";
+import { relevantGitHubTableNamesForImport } from "../../lib/config/github-tables";
const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET;
@@ -101,7 +101,7 @@ const startImport = async ({
namespace: splitgraphNamespace,
repository: splitgraphDestinationRepository,
tables: [
- ...relevantGitHubTableNames.map((t) => ({
+ ...relevantGitHubTableNamesForImport.map((t) => ({
name: t,
options: {},
schema: [],
From 17d8bc9eafdb0581a8f73f5abc8b7527f6a80115 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Mon, 29 May 2023 19:04:51 +0100
Subject: [PATCH 12/36] Track completed import/exports in a meta repository on
Splitgraph
After an import/export has completed, insert a row into the meta table,
which we will also use to fetch the previously imported repositories
from the client side when rendering the sidebar. We don't have transactional
guarantees on the DDN, so we can't do `INSERT ON CONFLICT`, so instead we
avoid duplicate rows by first selecting the existing row, and returning `204`
if it's already been inserted into the `completed_repositories` table.
However, I did notice that when I inserted the same row twice, it only
showed up once when I made a selection in the Console. I don't know if
this was due to a race condition, a bug, or because it's using the entire
row as a compound primary key and for some reason requiring that it be unique.
---
.../.env.test.local | 3 +
.../ImportExportStepper/ExportPanel.tsx | 9 +
.../ImportExportStepper/StepperContext.tsx | 2 +-
.../ImportExportStepper/stepper-states.ts | 100 +++++++++--
.../env-vars.d.ts | 53 ++++++
.../lib/backend/splitgraph-db.ts | 11 +-
.../pages/api/mark-import-export-complete.ts | 168 ++++++++++++++++++
7 files changed, 326 insertions(+), 20 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
index 473bf7d..dfb13b4 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
@@ -7,6 +7,9 @@
SPLITGRAPH_API_KEY="********************************"
SPLITGRAPH_API_SECRET="********************************"
+# This should match the username associated with the API key
+SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****"
+
# Create a GitHub token that can query the repositories you want to connect
# For example, a token with read-only access to public repos is sufficient
# CREATE ONE HERE: https://github.com/settings/personal-access-tokens/new
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index ad360be..bcfd800 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -45,6 +45,8 @@ export const ExportPanel = () => {
);
const handleStartExport = useCallback(async () => {
+ const abortController = new AbortController();
+
try {
const response = await fetch("/api/start-export-to-seafowl", {
method: "POST",
@@ -55,6 +57,7 @@ export const ExportPanel = () => {
headers: {
"Content-Type": "application/json",
},
+ signal: abortController.signal,
});
const data = (await response.json()) as StartExportToSeafowlResponseData;
@@ -87,8 +90,14 @@ export const ExportPanel = () => {
],
});
} catch (error) {
+ if (error.name === "AbortError") {
+ return;
+ }
+
dispatch({ type: "export_error", error: error.message });
}
+
+ return () => abortController.abort();
}, [queriesToExport, tablesToExport, dispatch]);
return (
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
index 0fdb7c0..f80ed7f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
@@ -1,5 +1,5 @@
// StepperContext.tsx
-import React, { useReducer, useContext, ReactNode } from "react";
+import React, { useContext, ReactNode } from "react";
import {
StepperState,
StepperAction,
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index 16ca39a..b1e5e51 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -45,11 +45,6 @@ export type StepperAction =
| { type: "reset" }
| { type: "initialize_from_url"; parsedFromUrl: StepperState };
-type ExtractStepperAction<T extends StepperAction["type"]> = Extract<
- StepperAction,
- { type: T }
->;
-
const initialState: StepperState = {
stepperState: "unstarted",
repository: null,
@@ -62,19 +57,6 @@ const initialState: StepperState = {
exportError: null,
};
-// FOR DEBUGGING: uncomment for hardcoded state initialization
-// export const initialState: StepperState = {
-// ...normalInitialState,
-// stepperState: "import_complete",
-// splitgraphNamespace: "miles",
-// splitgraphRepository: "import-via-nextjs",
-// };
-
-type ActionParams<T extends StepperAction["type"]> = Omit<
- ExtractStepperAction<T>,
- "type"
->;
-
const getQueryParamAsString = <T extends string = string>(
query: ParsedUrlQuery,
key: string
@@ -310,6 +292,86 @@ const urlNeedsChange = (state: StepperState, router: NextRouter) => {
);
};
+/**
+ * When the export has completed, send a request to /api/mark-import-export-complete
+ * which will insert the repository into the metadata table, which we query to
+ * render the sidebar
+ */
+const useMarkAsComplete = (
+ state: StepperState,
+ dispatch: React.Dispatch<StepperAction>
+) => {
+ useEffect(() => {
+ if (state.stepperState !== "export_complete") {
+ return;
+ }
+
+ const {
+ repository: {
+ namespace: githubSourceNamespace,
+ repository: githubSourceRepository,
+ },
+ splitgraphRepository: splitgraphDestinationRepository,
+ } = state;
+
+ // NOTE: Make sure to abort request so that in React 18 development mode,
+ // when effect runs twice, the second request is aborted and we don't have
+ // a race condition with two requests inserting into the table (where we have no transactional
+ // integrity and manually do a SELECT before the INSERT to check if the row already exists)
+ const abortController = new AbortController();
+
+ const markImportExportComplete = async () => {
+ try {
+ const response = await fetch("/api/mark-import-export-complete", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ githubSourceNamespace,
+ githubSourceRepository,
+ splitgraphDestinationRepository,
+ }),
+ signal: abortController.signal,
+ });
+
+ if (!response.ok) {
+ throw new Error("Failed to mark import/export as complete");
+ }
+
+ const data = await response.json();
+
+ if (!data.status) {
+ throw new Error(
+ "Got unexpected resposne shape when marking import/export complete"
+ );
+ }
+
+ if (data.error) {
+ throw new Error(
+ `Failed to mark import/export complete: ${data.error}`
+ );
+ }
+
+ console.log("Marked import/export as complete");
+ } catch (error) {
+ if (error.name === "AbortError") {
+ return;
+ }
+
+ dispatch({
+ type: "export_error",
+ error: error.message ?? error.toString(),
+ });
+ }
+ };
+
+ markImportExportComplete();
+
+ return () => abortController.abort();
+ }, [state, dispatch]);
+};
+
export const useStepperReducer = () => {
const router = useRouter();
const [state, dispatch] = useReducer(stepperReducer, {
@@ -317,6 +379,8 @@ export const useStepperReducer = () => {
stepperState: "uninitialized",
});
+ useMarkAsComplete(state, dispatch);
+
useEffect(() => {
dispatch({
type: "initialize_from_url",
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
index 5328009..2b0d888 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
@@ -37,5 +37,58 @@ namespace NodeJS {
* This is useful for debugging and development.
*/
MITMPROXY_ADDRESS?: string;
+
+ /**
+ * The namespace of the repository in Splitgraph where metadata is stored
+ * containing the state of imported GitHub repositories, which should contain
+ * the repository `SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY`.
+ *
+ * This should be defined in `.env.local`, since it's not checked into Git
+ * and can vary between users. It should match the username associated with
+ * the `SPLITGRAPH_API_KEY`
+ *
+ * Example:
+ *
+ * ```
+ * miles/splitgraph-github-analytics.completed_repositories
+ * ^^^^^
+ * SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE=miles
+ * ```
+ */
+ SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE: string;
+
+ /**
+ * The repository (no namespace) in Splitgraph where metadata is stored
+ * containing the state of imported GitHub repositories, which should be a
+ * repository contained inside `SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE`.
+ *
+ * This is defined by default in `.env` which is checked into Git.
+ *
+ * * Example:
+ *
+ * ```
+ * miles/splitgraph-github-analytics.completed_repositories
+ * ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ * SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=splitgraph-github-analytics
+ * ```
+ */
+ SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY: string;
+
+ /**
+ * The name of the table containing completed repositories, which are inserted
+ * when the import/export is complete, and which can be queried to render the
+ * sidebar containing previously imported github repositories.
+ *
+ * This is defined by default in `.env` which is checked into Git.
+ *
+ * Example:
+ *
+ * ```
+ * miles/splitgraph-github-analytics.completed_repositories
+ * ^^^^^^^^^^^^^^^^^^^^^^
+ * SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories
+ * ```
+ */
+ SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE: string;
}
}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
index 66ba751..ca1f2f2 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
@@ -1,4 +1,4 @@
-import { makeSplitgraphDb } from "@madatdata/core";
+import { makeSplitgraphDb, makeSplitgraphHTTPContext } from "@madatdata/core";
// TODO: fix plugin exports
import { makeDefaultPluginList } from "@madatdata/db-splitgraph";
@@ -33,6 +33,15 @@ export const makeAuthenticatedSplitgraphDb = () =>
}),
});
+export const makeAuthenticatedSplitgraphHTTPContext = () =>
+ makeSplitgraphHTTPContext({
+ authenticatedCredential,
+ plugins: makeDefaultPluginList({
+ graphqlEndpoint: defaultSplitgraphHost.baseUrls.gql,
+ authenticatedCredential,
+ }),
+ });
+
// TODO: export this utility function from the library
export const claimsFromJWT = (jwt?: string) => {
if (!jwt) {
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts
new file mode 100644
index 0000000..c981b03
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts
@@ -0,0 +1,168 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import {
+ makeAuthenticatedSplitgraphHTTPContext,
+ claimsFromJWT,
+} from "../../lib/backend/splitgraph-db";
+
+export type MarkImportExportCompleteRequestShape = {
+ githubSourceNamespace: string;
+ githubSourceRepository: string;
+ splitgraphDestinationRepository: string;
+};
+
+export type MarkImportExportCompleteSuccessResponse = {
+ status: "inserted";
+};
+
+export type MarkImportExportCompleteResponseData =
+ | MarkImportExportCompleteSuccessResponse
+ | { error: string };
+
+const META_NAMESPACE = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+const META_REPOSITORY = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY;
+const META_TABLE = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE;
+
+/**
+ * To manually send a request, example:
+
+```bash
+curl -i \
+ -H "Content-Type: application/json" http://localhost:3000/api/mark-import-export-complete \
+ -d@- <<EOF
+{"githubSourceNamespace": "splitgraph",
+"githubSourceRepository": "seafowl",
+"splitgraphDestinationRepository": "gh-import-splitgraph-seafowl" }
+EOF
+```
+ */
+export default async function handler(
+ req: NextApiRequest,
+ res: NextApiResponse<MarkImportExportCompleteResponseData>
+) {
+ if (!META_NAMESPACE) {
+ res.status(400).json({
+ error:
+ "Missing env var: SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE " +
+ "Is it in .env.local or Vercel secrets?",
+ });
+ return;
+ }
+
+ if (!META_REPOSITORY) {
+ res.status(400).json({
+ error:
+ "Missing env var: SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY " +
+ "Is it in .env or Vercel environment variables?",
+ });
+ return;
+ }
+
+ const missingOrInvalidKeys = [
+ "githubSourceNamespace",
+ "githubSourceRepository",
+ "splitgraphDestinationRepository",
+ ].filter(
+ (requiredKey) =>
+ !(requiredKey in req.body) ||
+ typeof req.body[requiredKey] !== "string" ||
+ !req.body[requiredKey] ||
+ !isSQLSafe(req.body[requiredKey])
+ );
+
+ if (missingOrInvalidKeys.length > 0) {
+ res.status(400).json({
+ error: `missing, non-string, empty or invalid keys: ${missingOrInvalidKeys.join(
+ ", "
+ )}`,
+ });
+ return;
+ }
+
+ try {
+ const { status } = await markImportExportAsComplete({
+ githubSourceNamespace: req.body.githubSourceNamespace,
+ githubSourceRepository: req.body.githubSourceRepository,
+ splitgraphDestinationRepository: req.body.splitgraphDestinationRepository,
+ });
+
+ if (status === "already exists") {
+ res.status(204).end();
+ return;
+ }
+
+ res.status(200).json({ status });
+ return;
+ } catch (err) {
+ res.status(400).json({ error: err });
+ return;
+ }
+}
+
+/**
+ * NOTE: We assume that this table already exists. If it does not exist, you can
+ * create it manually with a query like this in https://www.splitgraph.com/query :
+ *
+ * ```sql
+CREATE TABLE IF NOT EXISTS "miles/github-analytics-metadata".completed_repositories (
+ github_namespace VARCHAR NOT NULL,
+ github_repository VARCHAR NOT NULL,
+ splitgraph_namespace VARCHAR NOT NULL,
+ splitgraph_repository VARCHAR NOT NULL,
+ completed_at TIMESTAMP NOT NULL
+);
+```
+ */
+const markImportExportAsComplete = async ({
+ splitgraphDestinationRepository,
+ githubSourceNamespace,
+ githubSourceRepository,
+}: MarkImportExportCompleteRequestShape): Promise<{
+ status: "already exists" | "inserted";
+}> => {
+ const { db, client } = makeAuthenticatedSplitgraphHTTPContext();
+ const { username } = claimsFromJWT((await db.fetchAccessToken()).token);
+
+ // NOTE: We also assume that META_NAMESPACE is the same as destination namespace
+ if (!username || username !== META_NAMESPACE) {
+ throw new Error(
+ "Authenticated user does not match SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE"
+ );
+ }
+
+ // We don't want to insert the row if it already exists
+ // Note that Splitgraph doesn't support constraints so we can't use INSERT ON CONFLICT
+
+ const existingRows = await client.execute(`
+ SELECT splitgraph_repository FROM "${META_NAMESPACE}/${META_REPOSITORY}"."${META_TABLE}"
+ WHERE github_namespace = '${githubSourceNamespace}'
+ AND github_repository = '${githubSourceRepository}'
+ AND splitgraph_namespace = '${META_NAMESPACE}'
+ AND splitgraph_repository = '${splitgraphDestinationRepository}';
+ `);
+
+ if (existingRows.response && existingRows.response.rows.length > 0) {
+ return { status: "already exists" };
+ }
+
+ await client.execute(`INSERT INTO "${META_NAMESPACE}/${META_REPOSITORY}"."${META_TABLE}" (
+ github_namespace,
+ github_repository,
+ splitgraph_namespace,
+ splitgraph_repository,
+ completed_at
+) VALUES (
+ '${githubSourceNamespace}',
+ '${githubSourceRepository}',
+ '${META_NAMESPACE}',
+ '${splitgraphDestinationRepository}',
+ NOW()
+);`);
+
+ return { status: "inserted" };
+};
+
+/**
+ * Return `false` if the string contains any character other than alphanumeric,
+ * `-`, `_`, or `.`
+ */
+const isSQLSafe = (str: string) => !/[^a-z0-9\-_\.]/.test(str);
From 3ba9e97c30be6a1d0ab6d6cdc72054746d8aee0e Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Mon, 29 May 2023 22:21:19 +0100
Subject: [PATCH 13/36] Support ?debug=1 parameter in URL of stepper to render
DebugPane
---
.../components/ImportExportStepper/Stepper.tsx | 15 ++++++++++++---
.../ImportExportStepper/stepper-states.ts | 14 ++++++++++----
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
index 93f987f..101ad5f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
@@ -4,12 +4,22 @@ import { ImportPanel } from "./ImportPanel";
import { ExportPanel } from "./ExportPanel";
import styles from "./Stepper.module.css";
+import { useRouter } from "next/router";
const StepperOrLoading = ({ children }: { children: React.ReactNode }) => {
- const [{ stepperState }] = useStepper();
+ const [{ stepperState, debug }] = useStepper();
return (
- <>{stepperState === "uninitialized" ? <div>........</div> : children}</>
+ <>
+ {stepperState === "uninitialized" ? (
+ <div>........</div>
+ ) : (
+ <>
+ {debug && <DebugPanel />}
+ {children}
+ </>
+ )}
+ </>
);
};
@@ -18,7 +28,6 @@ export const Stepper = () => {
<StepperContextProvider>
<div className={styles.stepper}>
<StepperOrLoading>
- <DebugPanel />
<ImportPanel />
<ExportPanel />
</StepperOrLoading>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index b1e5e51..5895bdd 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -26,6 +26,7 @@ export type StepperState = {
exportedTablesLoading?: Set<ExportTable>;
exportedTablesCompleted?: Set<ExportTable>;
exportError?: string;
+ debug?: string | null;
};
export type StepperAction =
@@ -55,6 +56,7 @@ const initialState: StepperState = {
exportedTablesCompleted: new Set<ExportTable>(),
importError: null,
exportError: null,
+ debug: null,
};
const getQueryParamAsString = <T extends string = string>(
@@ -91,6 +93,7 @@ const queryParamParsers: {
getQueryParamAsString(query, "splitgraphNamespace"),
splitgraphRepository: (query) =>
getQueryParamAsString(query, "splitgraphRepository"),
+ debug: (query) => getQueryParamAsString(query, "debug"),
};
const requireKeys = <T extends Record<string, unknown>>(
@@ -151,6 +154,7 @@ const parseStateFromRouter = (router: NextRouter): StepperState => {
exportError: queryParamParsers.exportError(query),
splitgraphNamespace: queryParamParsers.splitgraphNamespace(query),
splitgraphRepository: queryParamParsers.splitgraphRepository(query),
+ debug: queryParamParsers.debug(query),
};
void stepperStateValidators[stepperState](stepper);
@@ -169,6 +173,7 @@ const serializeStateToQueryParams = (stepper: StepperState) => {
exportError: stepper.exportError ?? undefined,
splitgraphNamespace: stepper.splitgraphNamespace ?? undefined,
splitgraphRepository: stepper.splitgraphRepository ?? undefined,
+ debug: stepper.debug ?? undefined,
})
);
};
@@ -339,6 +344,11 @@ const useMarkAsComplete = (
throw new Error("Failed to mark import/export as complete");
}
+ if (response.status === 204) {
+ console.log("Repository already exists in metadata table");
+ return;
+ }
+
const data = await response.json();
if (!data.status) {
@@ -397,10 +407,6 @@ export const useStepperReducer = () => {
return;
}
- console.log("push", {
- pathname: router.pathname,
- query: serializeStateToQueryParams(state),
- });
router.push(
{
pathname: router.pathname,
From e6c592ed496dcc97adc9c08369696c780a6fd380 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Mon, 29 May 2023 22:22:28 +0100
Subject: [PATCH 14/36] Implement sidebar and stub out page for imported
repository to show the charts
The sidebar queries the DDN from the client-side with `useSql` from `@madatdata/react`,
using the default anonymous (thus read-only) credential to query the "metadata table"
that includes the list of repositories that have had a succesful import, and it links
to a page for each one, which is currently a stub but where we will show the chart(s)
with Observable Plot.
---
.../.env | 7 ++
.../.env.test.local | 2 +-
.../RepositoryAnalytics/Charts.module.css | 3 +
.../components/RepositoryAnalytics/Charts.tsx | 31 ++++++++
.../components/Sidebar.tsx | 73 ++++++++++++++++---
.../env-vars.d.ts | 16 ++--
.../[github_repository].tsx | 39 ++++++++++
.../pages/_app.tsx | 14 +++-
.../pages/api/mark-import-export-complete.ts | 15 ++--
.../pages/index.tsx | 51 ++-----------
.../types.ts | 6 ++
11 files changed, 184 insertions(+), 73 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/.env
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/types.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env b/examples/nextjs-import-airbyte-github-export-seafowl/.env
new file mode 100644
index 0000000..9ac0d11
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env
@@ -0,0 +1,7 @@
+# This file contains public environment variables and is therefore checked into the repo
+# For secret environment variables, see `.env.local` which is _not_ checked into the repo
+# Read env-vars.d.ts for expected variable names
+# See more: https://nextjs.org/docs/app/building-your-application/configuring/environment-variables
+
+NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=github-analytics-metadata
+NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
index dfb13b4..2d80ce0 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
@@ -8,7 +8,7 @@ SPLITGRAPH_API_KEY="********************************"
SPLITGRAPH_API_SECRET="********************************"
# This should match the username associated with the API key
-SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****"
+NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****"
# Create a GitHub token that can query the repositories you want to connect
# For example, a token with read-only access to public repos is sufficient
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css
new file mode 100644
index 0000000..2142170
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css
@@ -0,0 +1,3 @@
+.charts {
+ background: inherit;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
new file mode 100644
index 0000000..d5b3b3f
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
@@ -0,0 +1,31 @@
+import style from "./Charts.module.css";
+
+import type { ImportedRepository } from "../../types";
+
+export interface ChartsProps {
+ importedRepository: ImportedRepository;
+}
+
+export const Charts = ({
+ importedRepository: {
+ githubNamespace,
+ githubRepository,
+ splitgraphNamespace,
+ splitgraphRepository,
+ },
+}: ChartsProps) => {
+ return (
+ <div className={style.charts}>
+ Chart for{" "}
+ <a href={`https://github.com/${githubNamespace}/${githubRepository}`}>
+ github.com/{githubNamespace}/{githubRepository}
+ </a>
+ , based on{" "}
+ <a
+ href={`https://www.splitgraph.com/${splitgraphNamespace}/${splitgraphRepository}`}
+ >
+ splitgraph.com/{splitgraphNamespace}/{splitgraphRepository}
+ </a>
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
index f3ccb2f..c7574cb 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
@@ -1,29 +1,78 @@
-import React from "react";
+import React, { useMemo } from "react";
import Link from "next/link";
import styles from "./Sidebar.module.css";
+import { useSql } from "@madatdata/react";
-export interface GitHubRepository {
- namespace: string;
- repository: string;
-}
+import type { ImportedRepository } from "../types";
-interface SidebarProps {
- repositories: GitHubRepository[];
-}
+const META_REPOSITORY =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY;
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+const META_TABLE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE;
+
+const useImportedRepositories = (): ImportedRepository[] => {
+ const { response, error } = useSql<{
+ githubNamespace: string;
+ githubRepository: string;
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+ }>(
+ `
+ WITH ordered_repos AS (
+ SELECT
+ github_namespace,
+ github_repository,
+ splitgraph_namespace,
+ splitgraph_repository,
+ completed_at
+ FROM "${META_NAMESPACE}/${META_REPOSITORY}"."${META_TABLE}"
+ ORDER BY completed_at DESC
+ )
+ SELECT DISTINCT
+ github_namespace AS "githubNamespace",
+ github_repository AS "githubRepository",
+ splitgraph_namespace AS "splitgraphNamespace",
+ splitgraph_repository AS "splitgraphRepository"
+ FROM ordered_repos;
+ `
+ );
+
+ const repositories = useMemo(() => {
+ if (error) {
+ console.warn("Error fetching repositories:", error);
+ return [];
+ }
+
+ if (!response) {
+ console.warn("No response received");
+ return [];
+ }
+
+ return response.rows ?? [];
+ }, [error, response]);
+
+ return repositories;
+};
+
+export const Sidebar = () => {
+ const repositories = useImportedRepositories();
-export const Sidebar = ({ repositories }: React.PropsWithRef<SidebarProps>) => {
return (
<aside className={styles.sidebar}>
<div className={styles.importButtonContainer}>
- <Link href="/start-import" className={styles.importButton}>
+ <Link href="/" className={styles.importButton}>
Import Your Repository
</Link>
</div>
<ul className={styles.repoList}>
{repositories.map((repo, index) => (
<li key={index}>
- <Link href={`/${repo.namespace}/${repo.repository}`}>
- {repo.namespace}/{repo.repository}
+ <Link
+ href={`/${repo.githubNamespace}/${repo.githubRepository}?splitgraphNamespace=${repo.splitgraphNamespace}&splitgraphRepository=${repo.splitgraphRepository}`}
+ >
+ {repo.githubNamespace}/{repo.githubRepository}
</Link>
</li>
))}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
index 2b0d888..650db4e 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
@@ -41,7 +41,7 @@ namespace NodeJS {
/**
* The namespace of the repository in Splitgraph where metadata is stored
* containing the state of imported GitHub repositories, which should contain
- * the repository `SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY`.
+ * the repository `NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY`.
*
* This should be defined in `.env.local`, since it's not checked into Git
* and can vary between users. It should match the username associated with
@@ -52,15 +52,15 @@ namespace NodeJS {
* ```
* miles/splitgraph-github-analytics.completed_repositories
* ^^^^^
- * SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE=miles
+ * NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE=miles
* ```
*/
- SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE: string;
+ NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE: string;
/**
* The repository (no namespace) in Splitgraph where metadata is stored
* containing the state of imported GitHub repositories, which should be a
- * repository contained inside `SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE`.
+ * repository contained inside `NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE`.
*
* This is defined by default in `.env` which is checked into Git.
*
@@ -69,10 +69,10 @@ namespace NodeJS {
* ```
* miles/splitgraph-github-analytics.completed_repositories
* ^^^^^^^^^^^^^^^^^^^^^^^^^^^
- * SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=splitgraph-github-analytics
+ * NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=splitgraph-github-analytics
* ```
*/
- SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY: string;
+ NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY: string;
/**
* The name of the table containing completed repositories, which are inserted
@@ -86,9 +86,9 @@ namespace NodeJS {
* ```
* miles/splitgraph-github-analytics.completed_repositories
* ^^^^^^^^^^^^^^^^^^^^^^
- * SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories
+ * NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories
* ```
*/
- SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE: string;
+ NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE: string;
}
}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
new file mode 100644
index 0000000..6447a43
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
@@ -0,0 +1,39 @@
+import { BaseLayout } from "../../components/BaseLayout";
+
+import { Sidebar } from "../../components/Sidebar";
+import { Charts } from "../../components/RepositoryAnalytics/Charts";
+import { useRouter } from "next/router";
+
+import type { ImportedRepository } from "../../types";
+
+const useImportedRepoFromURL = () => {
+ const { query } = useRouter();
+
+ return (
+ [
+ ["github_namespace", "githubNamespace"],
+ ["github_repository", "githubRepository"],
+ ["splitgraphNamespace", "splitgraphNamespace"],
+ ["splitgraphRepository", "splitgraphRepository"],
+ ] as [string, keyof ImportedRepository][]
+ ).reduce((acc, [queryParam, repoKey]) => {
+ if (!query[queryParam] || Array.isArray(query[queryParam])) {
+ throw new Error(`Invalid query params: unexpected type of ${queryParam}`);
+ }
+
+ return {
+ ...acc,
+ [repoKey]: query[queryParam] as string,
+ };
+ }, {} as ImportedRepository);
+};
+
+const RepositoryAnalyticsPage = () => {
+ return (
+ <BaseLayout sidebar={<Sidebar />}>
+ <Charts importedRepository={useImportedRepoFromURL()} />
+ </BaseLayout>
+ );
+};
+
+export default RepositoryAnalyticsPage;
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
index 3a6568a..13b6201 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
@@ -1,7 +1,19 @@
import type { AppProps } from "next/app";
+import { SqlProvider, makeSplitgraphHTTPContext } from "@madatdata/react";
+import { useMemo } from "react";
+
import "../components/global-styles/reset.css";
import "../components/global-styles/theme.css";
export default function GitHubAnalyticsApp({ Component, pageProps }: AppProps) {
- return <Component {...pageProps} />;
+ const splitgraphDataContext = useMemo(
+ () => makeSplitgraphHTTPContext({ credential: null }),
+ []
+ );
+
+ return (
+ <SqlProvider dataContext={splitgraphDataContext}>
+ <Component {...pageProps} />
+ </SqlProvider>
+ );
}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts
index c981b03..de85325 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts
@@ -18,9 +18,12 @@ export type MarkImportExportCompleteResponseData =
| MarkImportExportCompleteSuccessResponse
| { error: string };
-const META_NAMESPACE = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
-const META_REPOSITORY = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY;
-const META_TABLE = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE;
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+const META_REPOSITORY =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY;
+const META_TABLE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE;
/**
* To manually send a request, example:
@@ -42,7 +45,7 @@ export default async function handler(
if (!META_NAMESPACE) {
res.status(400).json({
error:
- "Missing env var: SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE " +
+ "Missing env var: NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE " +
"Is it in .env.local or Vercel secrets?",
});
return;
@@ -51,7 +54,7 @@ export default async function handler(
if (!META_REPOSITORY) {
res.status(400).json({
error:
- "Missing env var: SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY " +
+ "Missing env var: NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY " +
"Is it in .env or Vercel environment variables?",
});
return;
@@ -125,7 +128,7 @@ const markImportExportAsComplete = async ({
// NOTE: We also assume that META_NAMESPACE is the same as destination namespace
if (!username || username !== META_NAMESPACE) {
throw new Error(
- "Authenticated user does not match SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE"
+ "Authenticated user does not match NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE"
);
}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
index 4218207..fb52bfe 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx
@@ -1,53 +1,14 @@
-import { SqlProvider, makeSplitgraphHTTPContext } from "@madatdata/react";
-import { useMemo } from "react";
-
import { BaseLayout } from "../components/BaseLayout";
-import { Sidebar, type GitHubRepository } from "../components/Sidebar";
+import { Sidebar } from "../components/Sidebar";
import { Stepper } from "../components/ImportExportStepper/Stepper";
-const SplitgraphSampleQuery = () => {
- const splitgraphDataContext = useMemo(
- () => makeSplitgraphHTTPContext({ credential: null }),
- []
- );
-
- // Uses splitgraph.com by default (anon access supported for public data)
+const ImportPage = () => {
return (
- <SqlProvider dataContext={splitgraphDataContext}>
- <BaseLayout sidebar={<Sidebar repositories={sampleRepositories} />}>
- <Stepper />
- </BaseLayout>
- </SqlProvider>
+ <BaseLayout sidebar={<Sidebar />}>
+ <Stepper />
+ </BaseLayout>
);
};
-export default SplitgraphSampleQuery;
-
-const sampleRepositories: GitHubRepository[] = [
- { namespace: "OpenTech", repository: "data-structures" },
- { namespace: "AiSolutions", repository: "machine-learning-api" },
- { namespace: "DevToolsInc", repository: "react-components" },
- { namespace: "QuantumComputing", repository: "quantum-algorithms" },
- { namespace: "GlobalNetworks", repository: "network-optimization" },
- { namespace: "CyberSec", repository: "firewall-config" },
- { namespace: "DataSci", repository: "data-analysis" },
- { namespace: "WebDevCo", repository: "responsive-templates" },
- { namespace: "CloudNet", repository: "cloud-infrastructure" },
- { namespace: "AiData", repository: "neural-networks" },
- { namespace: "DistributedSys", repository: "microservices-arch" },
- { namespace: "KernelDev", repository: "os-development" },
- { namespace: "FrontEndMagic", repository: "vue-utilities" },
- { namespace: "BackEndLogix", repository: "nodejs-server" },
- { namespace: "Securitech", repository: "encryption-utils" },
- { namespace: "FullStack", repository: "end-to-end-app" },
- { namespace: "DBMasters", repository: "database-design" },
- { namespace: "MobileApps", repository: "android-development" },
- { namespace: "GameFactory", repository: "game-engine" },
- { namespace: "WebAssembly", repository: "wasm-runtime" },
- { namespace: "RoboLogic", repository: "robot-navigation" },
- { namespace: "IoTDesign", repository: "iot-devices" },
- { namespace: "BlockchainTech", repository: "blockchain-network" },
- { namespace: "CryptoCoins", repository: "cryptocurrency" },
- { namespace: "VRWorld", repository: "vr-applications" },
-];
+export default ImportPage;
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
new file mode 100644
index 0000000..4c6cbc5
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
@@ -0,0 +1,6 @@
+export interface ImportedRepository {
+ githubNamespace: string;
+ githubRepository: string;
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+}
From 613a77462e3f55a4b33b644bd09f3e070aceb9cd Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Mon, 29 May 2023 22:27:23 +0100
Subject: [PATCH 15/36] Install `@observablehq/plot` in
`example-nextjs-import-airbyte-github-export-seafowl`
---
.../package.json | 1 +
examples/yarn.lock | 392 +++++++++++++++++-
2 files changed, 392 insertions(+), 1 deletion(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/package.json b/examples/nextjs-import-airbyte-github-export-seafowl/package.json
index ef04766..9add33f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/package.json
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/package.json
@@ -9,6 +9,7 @@
"dependencies": {
"@madatdata/core": "latest",
"@madatdata/react": "latest",
+ "@observablehq/plot": "0.6.7",
"next": "latest",
"react": "18.2.0",
"react-dom": "18.2.0"
diff --git a/examples/yarn.lock b/examples/yarn.lock
index 3cbd334..0b1d996 100644
--- a/examples/yarn.lock
+++ b/examples/yarn.lock
@@ -701,6 +701,17 @@ __metadata:
languageName: node
linkType: hard
+"@observablehq/plot@npm:0.6.7":
+ version: 0.6.7
+ resolution: "@observablehq/plot@npm:0.6.7"
+ dependencies:
+ d3: ^7.8.0
+ interval-tree-1d: ^1.0.0
+ isoformat: ^0.2.0
+ checksum: bd9ce3a6ad2073a0b17d2e71cf3e6f1867057de360ffa14cb46cc85b0b02e3b34c73aa121b8231f78b7ce152c71b127c084f68893105871aac4ea551f0252655
+ languageName: node
+ linkType: hard
+
"@swc/helpers@npm:0.4.14":
version: 0.4.14
resolution: "@swc/helpers@npm:0.4.14"
@@ -877,6 +888,13 @@ __metadata:
languageName: node
linkType: hard
+"binary-search-bounds@npm:^2.0.0":
+ version: 2.0.5
+ resolution: "binary-search-bounds@npm:2.0.5"
+ checksum: e073e265570ad09fe7520835c620f1e95036c7e9696c4f2135c9b20f4b4a44e0306b38977e057b049dab60fea4ab53ed4ad2ee19d9bf44cb6b652aa081788b89
+ languageName: node
+ linkType: hard
+
"brace-expansion@npm:^1.1.7":
version: 1.1.11
resolution: "brace-expansion@npm:1.1.11"
@@ -1050,6 +1068,13 @@ __metadata:
languageName: node
linkType: hard
+"commander@npm:7":
+ version: 7.2.0
+ resolution: "commander@npm:7.2.0"
+ checksum: 53501cbeee61d5157546c0bef0fedb6cdfc763a882136284bed9a07225f09a14b82d2a84e7637edfd1a679fb35ed9502fd58ef1d091e6287f60d790147f68ddc
+ languageName: node
+ linkType: hard
+
"concat-map@npm:0.0.1":
version: 0.0.1
resolution: "concat-map@npm:0.0.1"
@@ -1087,6 +1112,324 @@ __metadata:
languageName: node
linkType: hard
+"d3-array@npm:2 - 3, d3-array@npm:2.10.0 - 3, d3-array@npm:2.5.0 - 3, d3-array@npm:3, d3-array@npm:^3.2.0":
+ version: 3.2.3
+ resolution: "d3-array@npm:3.2.3"
+ dependencies:
+ internmap: 1 - 2
+ checksum: 41d6a4989b73e0d2649a880b2f29a7e7cc059db0eba36cd29a79e0118ebdf6b78922a84cde0733cd54cb4072f3442ec44f3563902e00ea42892442d60e99f961
+ languageName: node
+ linkType: hard
+
+"d3-axis@npm:3":
+ version: 3.0.0
+ resolution: "d3-axis@npm:3.0.0"
+ checksum: 227ddaa6d4bad083539c1ec245e2228b4620cca941997a8a650cb0af239375dc20271993127eedac66f0543f331027aca09385e1e16eed023f93eac937cddf0b
+ languageName: node
+ linkType: hard
+
+"d3-brush@npm:3":
+ version: 3.0.0
+ resolution: "d3-brush@npm:3.0.0"
+ dependencies:
+ d3-dispatch: 1 - 3
+ d3-drag: 2 - 3
+ d3-interpolate: 1 - 3
+ d3-selection: 3
+ d3-transition: 3
+ checksum: 1d042167769a02ac76271c71e90376d7184206e489552b7022a8ec2860209fe269db55e0a3430f3dcbe13b6fec2ff65b1adeaccba3218991b38e022390df72e3
+ languageName: node
+ linkType: hard
+
+"d3-chord@npm:3":
+ version: 3.0.1
+ resolution: "d3-chord@npm:3.0.1"
+ dependencies:
+ d3-path: 1 - 3
+ checksum: ddf35d41675e0f8738600a8a2f05bf0858def413438c12cba357c5802ecc1014c80a658acbbee63cbad2a8c747912efb2358455d93e59906fe37469f1dc6b78b
+ languageName: node
+ linkType: hard
+
+"d3-color@npm:1 - 3, d3-color@npm:3":
+ version: 3.1.0
+ resolution: "d3-color@npm:3.1.0"
+ checksum: 4931fbfda5d7c4b5cfa283a13c91a954f86e3b69d75ce588d06cde6c3628cebfc3af2069ccf225e982e8987c612aa7948b3932163ce15eb3c11cd7c003f3ee3b
+ languageName: node
+ linkType: hard
+
+"d3-contour@npm:4":
+ version: 4.0.2
+ resolution: "d3-contour@npm:4.0.2"
+ dependencies:
+ d3-array: ^3.2.0
+ checksum: 56aa082c1acf62a45b61c8d29fdd307041785aa17d9a07de7d1d848633769887a33fb6823888afa383f31c460d0f21d24756593e84e334ddb92d774214d32f1b
+ languageName: node
+ linkType: hard
+
+"d3-delaunay@npm:6":
+ version: 6.0.4
+ resolution: "d3-delaunay@npm:6.0.4"
+ dependencies:
+ delaunator: 5
+ checksum: ce6d267d5ef21a8aeadfe4606329fc80a22ab6e7748d47bc220bcc396ee8be84b77a5473033954c5ac4aa522d265ddc45d4165d30fe4787dd60a15ea66b9bbb4
+ languageName: node
+ linkType: hard
+
+"d3-dispatch@npm:1 - 3, d3-dispatch@npm:3":
+ version: 3.0.1
+ resolution: "d3-dispatch@npm:3.0.1"
+ checksum: fdfd4a230f46463e28e5b22a45dd76d03be9345b605e1b5dc7d18bd7ebf504e6c00ae123fd6d03e23d9e2711e01f0e14ea89cd0632545b9f0c00b924ba4be223
+ languageName: node
+ linkType: hard
+
+"d3-drag@npm:2 - 3, d3-drag@npm:3":
+ version: 3.0.0
+ resolution: "d3-drag@npm:3.0.0"
+ dependencies:
+ d3-dispatch: 1 - 3
+ d3-selection: 3
+ checksum: d297231e60ecd633b0d076a63b4052b436ddeb48b5a3a11ff68c7e41a6774565473a6b064c5e9256e88eca6439a917ab9cea76032c52d944ddbf4fd289e31111
+ languageName: node
+ linkType: hard
+
+"d3-dsv@npm:1 - 3, d3-dsv@npm:3":
+ version: 3.0.1
+ resolution: "d3-dsv@npm:3.0.1"
+ dependencies:
+ commander: 7
+ iconv-lite: 0.6
+ rw: 1
+ bin:
+ csv2json: bin/dsv2json.js
+ csv2tsv: bin/dsv2dsv.js
+ dsv2dsv: bin/dsv2dsv.js
+ dsv2json: bin/dsv2json.js
+ json2csv: bin/json2dsv.js
+ json2dsv: bin/json2dsv.js
+ json2tsv: bin/json2dsv.js
+ tsv2csv: bin/dsv2dsv.js
+ tsv2json: bin/dsv2json.js
+ checksum: 5fc0723647269d5dccd181d74f2265920ab368a2868b0b4f55ffa2fecdfb7814390ea28622cd61ee5d9594ab262879509059544e9f815c54fe76fbfb4ffa4c8a
+ languageName: node
+ linkType: hard
+
+"d3-ease@npm:1 - 3, d3-ease@npm:3":
+ version: 3.0.1
+ resolution: "d3-ease@npm:3.0.1"
+ checksum: 06e2ee5326d1e3545eab4e2c0f84046a123dcd3b612e68858219aa034da1160333d9ce3da20a1d3486d98cb5c2a06f7d233eee1bc19ce42d1533458bd85dedcd
+ languageName: node
+ linkType: hard
+
+"d3-fetch@npm:3":
+ version: 3.0.1
+ resolution: "d3-fetch@npm:3.0.1"
+ dependencies:
+ d3-dsv: 1 - 3
+ checksum: 382dcea06549ef82c8d0b719e5dc1d96286352579e3b51b20f71437f5800323315b09cf7dcfd4e1f60a41e1204deb01758470cea257d2285a7abd9dcec806984
+ languageName: node
+ linkType: hard
+
+"d3-force@npm:3":
+ version: 3.0.0
+ resolution: "d3-force@npm:3.0.0"
+ dependencies:
+ d3-dispatch: 1 - 3
+ d3-quadtree: 1 - 3
+ d3-timer: 1 - 3
+ checksum: 6c7e96438cab62fa32aeadb0ade3297b62b51f81b1b38b0a60a5ec9fd627d74090c1189654d92df2250775f31b06812342f089f1d5947de9960a635ee3581def
+ languageName: node
+ linkType: hard
+
+"d3-format@npm:1 - 3, d3-format@npm:3":
+ version: 3.1.0
+ resolution: "d3-format@npm:3.1.0"
+ checksum: f345ec3b8ad3cab19bff5dead395bd9f5590628eb97a389b1dd89f0b204c7c4fc1d9520f13231c2c7cf14b7c9a8cf10f8ef15bde2befbab41454a569bd706ca2
+ languageName: node
+ linkType: hard
+
+"d3-geo@npm:3":
+ version: 3.1.0
+ resolution: "d3-geo@npm:3.1.0"
+ dependencies:
+ d3-array: 2.5.0 - 3
+ checksum: adf82b0c105c0c5951ae0a833d4dfc479a563791ad7938579fa14e1cffd623b469d8aa7a37dc413a327fb6ac56880f3da3f6c43d4abe3c923972dd98f34f37d1
+ languageName: node
+ linkType: hard
+
+"d3-hierarchy@npm:3":
+ version: 3.1.2
+ resolution: "d3-hierarchy@npm:3.1.2"
+ checksum: 0fd946a8c5fd4686d43d3e11bbfc2037a145fda29d2261ccd0e36f70b66af6d7638e2c0c7112124d63fc3d3127197a00a6aecf676bd5bd392a94d7235a214263
+ languageName: node
+ linkType: hard
+
+"d3-interpolate@npm:1 - 3, d3-interpolate@npm:1.2.0 - 3, d3-interpolate@npm:3":
+ version: 3.0.1
+ resolution: "d3-interpolate@npm:3.0.1"
+ dependencies:
+ d3-color: 1 - 3
+ checksum: a42ba314e295e95e5365eff0f604834e67e4a3b3c7102458781c477bd67e9b24b6bb9d8e41ff5521050a3f2c7c0c4bbbb6e187fd586daa3980943095b267e78b
+ languageName: node
+ linkType: hard
+
+"d3-path@npm:1 - 3, d3-path@npm:3, d3-path@npm:^3.1.0":
+ version: 3.1.0
+ resolution: "d3-path@npm:3.1.0"
+ checksum: 2306f1bd9191e1eac895ec13e3064f732a85f243d6e627d242a313f9777756838a2215ea11562f0c7630c7c3b16a19ec1fe0948b1c82f3317fac55882f6ee5d8
+ languageName: node
+ linkType: hard
+
+"d3-polygon@npm:3":
+ version: 3.0.1
+ resolution: "d3-polygon@npm:3.0.1"
+ checksum: 0b85c532517895544683849768a2c377cee3801ef8ccf3fa9693c8871dd21a0c1a2a0fc75ff54192f0ba2c562b0da2bc27f5bf959dfafc7fa23573b574865d2c
+ languageName: node
+ linkType: hard
+
+"d3-quadtree@npm:1 - 3, d3-quadtree@npm:3":
+ version: 3.0.1
+ resolution: "d3-quadtree@npm:3.0.1"
+ checksum: 5469d462763811475f34a7294d984f3eb100515b0585ca5b249656f6b1a6e99b20056a2d2e463cc9944b888896d2b1d07859c50f9c0cf23438df9cd2e3146066
+ languageName: node
+ linkType: hard
+
+"d3-random@npm:3":
+ version: 3.0.1
+ resolution: "d3-random@npm:3.0.1"
+ checksum: a70ad8d1cabe399ebeb2e482703121ac8946a3b336830b518da6848b9fdd48a111990fc041dc716f16885a72176ffa2898f2a250ca3d363ecdba5ef92b18e131
+ languageName: node
+ linkType: hard
+
+"d3-scale-chromatic@npm:3":
+ version: 3.0.0
+ resolution: "d3-scale-chromatic@npm:3.0.0"
+ dependencies:
+ d3-color: 1 - 3
+ d3-interpolate: 1 - 3
+ checksum: a8ce4cb0267a17b28ebbb929f5e3071d985908a9c13b6fcaa2a198e1e018f275804d691c5794b970df0049725b7944f32297b31603d235af6414004f0c7f82c0
+ languageName: node
+ linkType: hard
+
+"d3-scale@npm:4":
+ version: 4.0.2
+ resolution: "d3-scale@npm:4.0.2"
+ dependencies:
+ d3-array: 2.10.0 - 3
+ d3-format: 1 - 3
+ d3-interpolate: 1.2.0 - 3
+ d3-time: 2.1.1 - 3
+ d3-time-format: 2 - 4
+ checksum: a9c770d283162c3bd11477c3d9d485d07f8db2071665f1a4ad23eec3e515e2cefbd369059ec677c9ac849877d1a765494e90e92051d4f21111aa56791c98729e
+ languageName: node
+ linkType: hard
+
+"d3-selection@npm:2 - 3, d3-selection@npm:3":
+ version: 3.0.0
+ resolution: "d3-selection@npm:3.0.0"
+ checksum: f4e60e133309115b99f5b36a79ae0a19d71ee6e2d5e3c7216ef3e75ebd2cb1e778c2ed2fa4c01bef35e0dcbd96c5428f5bd6ca2184fe2957ed582fde6841cbc5
+ languageName: node
+ linkType: hard
+
+"d3-shape@npm:3":
+ version: 3.2.0
+ resolution: "d3-shape@npm:3.2.0"
+ dependencies:
+ d3-path: ^3.1.0
+ checksum: de2af5fc9a93036a7b68581ca0bfc4aca2d5a328aa7ba7064c11aedd44d24f310c20c40157cb654359d4c15c3ef369f95ee53d71221017276e34172c7b719cfa
+ languageName: node
+ linkType: hard
+
+"d3-time-format@npm:2 - 4, d3-time-format@npm:4":
+ version: 4.1.0
+ resolution: "d3-time-format@npm:4.1.0"
+ dependencies:
+ d3-time: 1 - 3
+ checksum: 7342bce28355378152bbd4db4e275405439cabba082d9cd01946d40581140481c8328456d91740b0fe513c51ec4a467f4471ffa390c7e0e30ea30e9ec98fcdf4
+ languageName: node
+ linkType: hard
+
+"d3-time@npm:1 - 3, d3-time@npm:2.1.1 - 3, d3-time@npm:3":
+ version: 3.1.0
+ resolution: "d3-time@npm:3.1.0"
+ dependencies:
+ d3-array: 2 - 3
+ checksum: 613b435352a78d9f31b7f68540788186d8c331b63feca60ad21c88e9db1989fe888f97f242322ebd6365e45ec3fb206a4324cd4ca0dfffa1d9b5feb856ba00a7
+ languageName: node
+ linkType: hard
+
+"d3-timer@npm:1 - 3, d3-timer@npm:3":
+ version: 3.0.1
+ resolution: "d3-timer@npm:3.0.1"
+ checksum: 1cfddf86d7bca22f73f2c427f52dfa35c49f50d64e187eb788dcad6e927625c636aa18ae4edd44d084eb9d1f81d8ca4ec305dae7f733c15846a824575b789d73
+ languageName: node
+ linkType: hard
+
+"d3-transition@npm:2 - 3, d3-transition@npm:3":
+ version: 3.0.1
+ resolution: "d3-transition@npm:3.0.1"
+ dependencies:
+ d3-color: 1 - 3
+ d3-dispatch: 1 - 3
+ d3-ease: 1 - 3
+ d3-interpolate: 1 - 3
+ d3-timer: 1 - 3
+ peerDependencies:
+ d3-selection: 2 - 3
+ checksum: cb1e6e018c3abf0502fe9ff7b631ad058efb197b5e14b973a410d3935aead6e3c07c67d726cfab258e4936ef2667c2c3d1cd2037feb0765f0b4e1d3b8788c0ea
+ languageName: node
+ linkType: hard
+
+"d3-zoom@npm:3":
+ version: 3.0.0
+ resolution: "d3-zoom@npm:3.0.0"
+ dependencies:
+ d3-dispatch: 1 - 3
+ d3-drag: 2 - 3
+ d3-interpolate: 1 - 3
+ d3-selection: 2 - 3
+ d3-transition: 2 - 3
+ checksum: 8056e3527281cfd1ccbcbc458408f86973b0583e9dac00e51204026d1d36803ca437f970b5736f02fafed9f2b78f145f72a5dbc66397e02d4d95d4c594b8ff54
+ languageName: node
+ linkType: hard
+
+"d3@npm:^7.8.0":
+ version: 7.8.4
+ resolution: "d3@npm:7.8.4"
+ dependencies:
+ d3-array: 3
+ d3-axis: 3
+ d3-brush: 3
+ d3-chord: 3
+ d3-color: 3
+ d3-contour: 4
+ d3-delaunay: 6
+ d3-dispatch: 3
+ d3-drag: 3
+ d3-dsv: 3
+ d3-ease: 3
+ d3-fetch: 3
+ d3-force: 3
+ d3-format: 3
+ d3-geo: 3
+ d3-hierarchy: 3
+ d3-interpolate: 3
+ d3-path: 3
+ d3-polygon: 3
+ d3-quadtree: 3
+ d3-random: 3
+ d3-scale: 4
+ d3-scale-chromatic: 3
+ d3-selection: 3
+ d3-shape: 3
+ d3-time: 3
+ d3-time-format: 4
+ d3-timer: 3
+ d3-transition: 3
+ d3-zoom: 3
+ checksum: 8dfea4d026e5597ab9c46035df7f0ca4f3891b2b52b21b181bd8660fc61d85f0bbe3cc2b8f1e922978084156cb017cbbfa350a8e42349310642023a9f1517471
+ languageName: node
+ linkType: hard
+
"debug@npm:4, debug@npm:^4.1.0, debug@npm:^4.3.3":
version: 4.3.4
resolution: "debug@npm:4.3.4"
@@ -1106,6 +1449,15 @@ __metadata:
languageName: node
linkType: hard
+"delaunator@npm:5":
+ version: 5.0.0
+ resolution: "delaunator@npm:5.0.0"
+ dependencies:
+ robust-predicates: ^3.0.0
+ checksum: d6764188442b7f7c6bcacebd96edc00e35f542a96f1af3ef600e586bfb9849a3682c489c0ab423440c90bc4c7cac77f28761babff76fa29e193e1cf50a95b860
+ languageName: node
+ linkType: hard
+
"delayed-stream@npm:~1.0.0":
version: 1.0.0
resolution: "delayed-stream@npm:1.0.0"
@@ -1477,7 +1829,7 @@ __metadata:
languageName: node
linkType: hard
-"iconv-lite@npm:^0.6.2":
+"iconv-lite@npm:0.6, iconv-lite@npm:^0.6.2":
version: 0.6.3
resolution: "iconv-lite@npm:0.6.3"
dependencies:
@@ -1524,6 +1876,22 @@ __metadata:
languageName: node
linkType: hard
+"internmap@npm:1 - 2":
+ version: 2.0.3
+ resolution: "internmap@npm:2.0.3"
+ checksum: 7ca41ec6aba8f0072fc32fa8a023450a9f44503e2d8e403583c55714b25efd6390c38a87161ec456bf42d7bc83aab62eb28f5aef34876b1ac4e60693d5e1d241
+ languageName: node
+ linkType: hard
+
+"interval-tree-1d@npm:^1.0.0":
+ version: 1.0.4
+ resolution: "interval-tree-1d@npm:1.0.4"
+ dependencies:
+ binary-search-bounds: ^2.0.0
+ checksum: 7cac7a4ea99ba2d03e3ebaeaf73ad35eaeebc63b5dca1e126601474c7e41c09b158acc31423595c384a88ea7c1ce8e0b340cfde41a85fd6dbef552d12f35ba69
+ languageName: node
+ linkType: hard
+
"ip@npm:^2.0.0":
version: 2.0.0
resolution: "ip@npm:2.0.0"
@@ -1561,6 +1929,13 @@ __metadata:
languageName: node
linkType: hard
+"isoformat@npm:^0.2.0":
+ version: 0.2.1
+ resolution: "isoformat@npm:0.2.1"
+ checksum: 28487777526c93360c2f49abbf03d45778ad2c55bfccb3a6bf04905b2ddfafcb9f29a68d6ab5251f2919afb47e0e018fe25f815fd68180f4117161c508878558
+ languageName: node
+ linkType: hard
+
"js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0":
version: 4.0.0
resolution: "js-tokens@npm:4.0.0"
@@ -1900,6 +2275,7 @@ __metadata:
dependencies:
"@madatdata/core": latest
"@madatdata/react": latest
+ "@observablehq/plot": 0.6.7
"@types/node": ^18.0.0
"@types/react": ^18.0.14
next: latest
@@ -2237,6 +2613,13 @@ __metadata:
languageName: node
linkType: hard
+"robust-predicates@npm:^3.0.0":
+ version: 3.0.2
+ resolution: "robust-predicates@npm:3.0.2"
+ checksum: 36854c1321548ceca96d36ad9d6e0a5a512986029ec6929ad6ed3ec1612c22cc8b46cc72d2c5674af42e8074a119d793f6f0ea3a5b51373e3ab926c64b172d7a
+ languageName: node
+ linkType: hard
+
"rollup@npm:^3.18.0":
version: 3.20.2
resolution: "rollup@npm:3.20.2"
@@ -2260,6 +2643,13 @@ __metadata:
languageName: unknown
linkType: soft
+"rw@npm:1":
+ version: 1.3.3
+ resolution: "rw@npm:1.3.3"
+ checksum: c20d82421f5a71c86a13f76121b751553a99cd4a70ea27db86f9b23f33db941f3f06019c30f60d50c356d0bd674c8e74764ac146ea55e217c091bde6fba82aa3
+ languageName: node
+ linkType: hard
+
"safe-buffer@npm:~5.2.0":
version: 5.2.1
resolution: "safe-buffer@npm:5.2.1"
From 309203df7b76baf2cade750f237cfc8894357b20 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Tue, 30 May 2023 03:31:17 +0100
Subject: [PATCH 16/36] Render stargazers line chart with Observable Plot
querying Seafowl
---
.../components/RepositoryAnalytics/Charts.tsx | 107 ++++++++++++---
.../ImportedRepoMetadata.module.css | 3 +
.../ImportedRepoMetadata.tsx | 129 ++++++++++++++++++
.../RepositoryAnalytics/sql-queries.ts | 54 ++++++++
.../components/Sidebar.tsx | 39 ++++--
.../[github_repository].tsx | 53 ++++---
.../pages/_app.tsx | 13 +-
7 files changed, 337 insertions(+), 61 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
index d5b3b3f..0e0f2f3 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
@@ -1,31 +1,102 @@
import style from "./Charts.module.css";
+import { useEffect, useRef } from "react";
import type { ImportedRepository } from "../../types";
+import { SqlProvider, makeSeafowlHTTPContext, useSql } from "@madatdata/react";
+
+import * as Plot from "@observablehq/plot";
+import { useMemo } from "react";
+
+import {
+ stargazersLineChartQuery,
+ type StargazersLineChartRow,
+} from "./sql-queries";
export interface ChartsProps {
importedRepository: ImportedRepository;
}
-export const Charts = ({
- importedRepository: {
- githubNamespace,
- githubRepository,
- splitgraphNamespace,
- splitgraphRepository,
- },
-}: ChartsProps) => {
+// Assume meta namespace contains both the meta tables, and all imported repositories and tables
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+
+export const Charts = ({ importedRepository }: ChartsProps) => {
+ const seafowlDataContext = useMemo(
+ () =>
+ makeSeafowlHTTPContext("https://demo.seafowl.cloud", {
+ dbname: META_NAMESPACE,
+ }),
+ []
+ );
+
return (
<div className={style.charts}>
- Chart for{" "}
- <a href={`https://github.com/${githubNamespace}/${githubRepository}`}>
- github.com/{githubNamespace}/{githubRepository}
- </a>
- , based on{" "}
- <a
- href={`https://www.splitgraph.com/${splitgraphNamespace}/${splitgraphRepository}`}
- >
- splitgraph.com/{splitgraphNamespace}/{splitgraphRepository}
- </a>
+ <SqlProvider dataContext={seafowlDataContext}>
+ <StargazersChart {...importedRepository} />
+ </SqlProvider>
</div>
);
};
+
+const StargazersChart = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+}: ImportedRepository) => {
+ const containerRef = useRef<HTMLDivElement>();
+
+ const { response, error } = useSql<StargazersLineChartRow>(
+ stargazersLineChartQuery({ splitgraphNamespace, splitgraphRepository })
+ );
+
+ const stargazers = useMemo(() => {
+ return !response || error
+ ? []
+ : (response.rows ?? []).map((r) => ({
+ ...r,
+ starred_at: new Date(r.starred_at),
+ }));
+ }, [response, error]);
+
+ useEffect(() => {
+ if (stargazers === undefined) {
+ return;
+ }
+
+ const plot = Plot.plot({
+ y: { grid: true },
+ color: { scheme: "burd" },
+ marks: [
+ Plot.lineY(stargazers, {
+ x: "starred_at",
+ y: "cumulative_stars",
+ }),
+ // NOTE: We don't have username when querying Seafowl because it's within a JSON object,
+ // and seafowl doesn't support querying inside JSON objects
+ // Plot.tip(
+ // stargazers,
+ // Plot.pointer({
+ // x: "starred_at",
+ // y: "cumulative_stars",
+ // title: (d) => `${d.username} was stargazer #${d.cumulative_stars}`,
+ // })
+ // ),
+ ],
+ });
+
+ // There is a bug(?) in useSql where, since we can't give it dependencies, it
+ // will re-run even with splitgraphNamespace and splitgraphRepository are undefined,
+ // which results in an error querying Seafowl. So just don't render the chart in that case.
+ if (splitgraphNamespace && splitgraphRepository) {
+ containerRef.current.append(plot);
+ }
+
+ return () => plot.remove();
+ }, [stargazers]);
+
+ return (
+ <>
+ <h3>Stargazers</h3>
+ <div ref={containerRef} />
+ </>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.module.css
new file mode 100644
index 0000000..9c4f1c7
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.module.css
@@ -0,0 +1,3 @@
+.importedRepoMetadata {
+ background: inherit;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
new file mode 100644
index 0000000..b48d71b
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
@@ -0,0 +1,129 @@
+import type { ImportedRepository } from "../../types";
+import style from "./ImportedRepoMetadata.module.css";
+
+import { makeStargazersTableQuery } from "./sql-queries";
+
+// Assume meta namespace contains both the meta tables, and all imported repositories and tables
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+
+interface ImportedRepoMetadataProps {
+ importedRepository: ImportedRepository;
+}
+
+export const ImportedRepoMetadata = ({
+ importedRepository,
+}: ImportedRepoMetadataProps) => {
+ return (
+ <div className={style.importedRepoMetadata}>
+ <h1>
+ <GitHubRepoLink {...importedRepository} />
+ </h1>
+ <h2>GitHub Analytics</h2>
+
+ <ul>
+ <li>
+ Browse the data: <SplitgraphRepoLink {...importedRepository} />
+ </li>
+ <li>
+ <SplitgraphQueryLink
+ importedRepository={importedRepository}
+ tableName={"stargazers"}
+ makeQuery={makeStargazersTableQuery}
+ />
+ </li>
+ <li>
+ <SeafowlQueryLink
+ importedRepository={importedRepository}
+ tableName={"stargazers"}
+ makeQuery={makeStargazersTableQuery}
+ />
+ </li>
+ </ul>
+ </div>
+ );
+};
+
+const SplitgraphRepoLink = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+}: ImportedRepository) => {
+ return (
+ <a
+ target="_blank"
+ href={`https://www.splitgraph.com/${splitgraphNamespace}/${splitgraphRepository}`}
+ >
+ {`splitgraph.com/${splitgraphNamespace}/${splitgraphRepository}`}
+ </a>
+ );
+};
+
+const GitHubRepoLink = ({
+ githubNamespace,
+ githubRepository,
+}: ImportedRepository) => {
+ return (
+ <a
+ target="_blank"
+ href={`https://github.com/${githubNamespace}/${githubRepository}`}
+ >{`github.com/${githubNamespace}/${githubRepository}`}</a>
+ );
+};
+
+const SplitgraphQueryLink = ({
+ importedRepository,
+ makeQuery,
+ tableName,
+}: {
+ importedRepository: ImportedRepository;
+ makeQuery: (repo: ImportedRepository) => string;
+ tableName: string;
+}) => {
+ return (
+ <a
+ href={makeSplitgraphQueryHref(makeQuery(importedRepository))}
+ target="_blank"
+ >
+ Query {tableName} in the Splitgraph Console
+ </a>
+ );
+};
+
+const SeafowlQueryLink = ({
+ importedRepository,
+ makeQuery,
+ tableName,
+}: {
+ importedRepository: ImportedRepository;
+ makeQuery: (repo: ImportedRepository) => string;
+ tableName: string;
+}) => {
+ return (
+ <a
+ href={makeSeafowlQueryHref(makeQuery(importedRepository))}
+ target="_blank"
+ >
+ Query Seafowl table {tableName} using the Splitgraph Console
+ </a>
+ );
+};
+
+/** Return the URL to Splitgraph Console pointing to Splitgraph DDN */
+const makeSplitgraphQueryHref = (sqlQuery: string) => {
+ const url = `https://www.splitgraph.com/query?${new URLSearchParams({
+ sqlQuery: sqlQuery,
+ flavor: "splitgraph",
+ }).toString()}`;
+
+ return url;
+};
+
+/** Return the URL to Splitgraph Console pointing to Seafowl db where we export tables */
+const makeSeafowlQueryHref = (sqlQuery: string) => {
+ return `https://www.splitgraph.com/query?${new URLSearchParams({
+ sqlQuery: sqlQuery,
+ flavor: "seafowl",
+ // Splitgraph exports to Seafowl dbname matching the username of the exporting user
+ "database-name": META_NAMESPACE,
+ })}`;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
new file mode 100644
index 0000000..47e1962
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
@@ -0,0 +1,54 @@
+import type { ImportedRepository } from "../../types";
+
+// Assume meta namespace contains both the meta tables, and all imported repositories and tables
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+
+type TargetSplitgraphRepo = {
+ splitgraphNamespace?: string;
+ splitgraphRepository: string;
+};
+
+/**
+ * Raw query to select all columns in the stargazers table, which can be
+ * run on both Splitgraph and Seafowl.
+ *
+ * This is meant for linking to the query editor, not for rendering charts.
+ */
+export const makeStargazersTableQuery = ({
+ splitgraphNamespace = META_NAMESPACE,
+ splitgraphRepository,
+}: TargetSplitgraphRepo) => {
+ return `SELECT
+ "repository",
+ "user_id",
+ "starred_at",
+ "user",
+ "_airbyte_ab_id",
+ "_airbyte_emitted_at",
+ "_airbyte_normalized_at",
+ "_airbyte_stargazers_hashid"
+FROM
+ "${splitgraphNamespace}/${splitgraphRepository}"."stargazers"
+LIMIT 100;`;
+};
+
+/** Shape of row returned by {@link stargazersLineChartQuery} */
+export type StargazersLineChartRow = {
+ username: string;
+ cumulative_stars: number;
+ starred_at: string;
+};
+
+/** Time series of GitHub stargazers for the given repository */
+export const stargazersLineChartQuery = ({
+ splitgraphNamespace = META_NAMESPACE,
+ splitgraphRepository,
+}: TargetSplitgraphRepo) => {
+ return `SELECT
+ COUNT(*) OVER (ORDER BY starred_at) AS cumulative_stars,
+ starred_at
+FROM
+ "${splitgraphNamespace}/${splitgraphRepository}"."stargazers"
+ORDER BY starred_at;`;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
index c7574cb..1ab5bf9 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
@@ -1,6 +1,7 @@
import React, { useMemo } from "react";
import Link from "next/link";
import styles from "./Sidebar.module.css";
+import { SqlProvider, makeSplitgraphHTTPContext } from "@madatdata/react";
import { useSql } from "@madatdata/react";
import type { ImportedRepository } from "../types";
@@ -46,7 +47,6 @@ const useImportedRepositories = (): ImportedRepository[] => {
}
if (!response) {
- console.warn("No response received");
return [];
}
@@ -56,9 +56,30 @@ const useImportedRepositories = (): ImportedRepository[] => {
return repositories;
};
-export const Sidebar = () => {
+const RepositoriesList = () => {
const repositories = useImportedRepositories();
+ return (
+ <ul className={styles.repoList}>
+ {repositories.map((repo, index) => (
+ <li key={index}>
+ <Link
+ href={`/${repo.githubNamespace}/${repo.githubRepository}?splitgraphNamespace=${repo.splitgraphNamespace}&splitgraphRepository=${repo.splitgraphRepository}`}
+ >
+ {repo.githubNamespace}/{repo.githubRepository}
+ </Link>
+ </li>
+ ))}
+ </ul>
+ );
+};
+
+export const Sidebar = () => {
+ const splitgraphDataContext = useMemo(
+ () => makeSplitgraphHTTPContext({ credential: null }),
+ []
+ );
+
return (
<aside className={styles.sidebar}>
<div className={styles.importButtonContainer}>
@@ -66,17 +87,9 @@ export const Sidebar = () => {
Import Your Repository
</Link>
</div>
- <ul className={styles.repoList}>
- {repositories.map((repo, index) => (
- <li key={index}>
- <Link
- href={`/${repo.githubNamespace}/${repo.githubRepository}?splitgraphNamespace=${repo.splitgraphNamespace}&splitgraphRepository=${repo.splitgraphRepository}`}
- >
- {repo.githubNamespace}/{repo.githubRepository}
- </Link>
- </li>
- ))}
- </ul>
+ <SqlProvider dataContext={splitgraphDataContext}>
+ <RepositoriesList />
+ </SqlProvider>
</aside>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
index 6447a43..c8baaa3 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
@@ -6,32 +6,49 @@ import { useRouter } from "next/router";
import type { ImportedRepository } from "../../types";
+import { ImportedRepoMetadata } from "../../components/RepositoryAnalytics/ImportedRepoMetadata";
+import { useMemo } from "react";
+
const useImportedRepoFromURL = () => {
const { query } = useRouter();
- return (
- [
- ["github_namespace", "githubNamespace"],
- ["github_repository", "githubRepository"],
- ["splitgraphNamespace", "splitgraphNamespace"],
- ["splitgraphRepository", "splitgraphRepository"],
- ] as [string, keyof ImportedRepository][]
- ).reduce((acc, [queryParam, repoKey]) => {
- if (!query[queryParam] || Array.isArray(query[queryParam])) {
- throw new Error(`Invalid query params: unexpected type of ${queryParam}`);
- }
-
- return {
- ...acc,
- [repoKey]: query[queryParam] as string,
- };
- }, {} as ImportedRepository);
+ console.log("query:", query);
+
+ const queryParams = useMemo(
+ () =>
+ (
+ [
+ ["github_namespace", "githubNamespace"],
+ ["github_repository", "githubRepository"],
+ ["splitgraphNamespace", "splitgraphNamespace"],
+ ["splitgraphRepository", "splitgraphRepository"],
+ ] as [string, keyof ImportedRepository][]
+ ).reduce((acc, [queryParam, repoKey]) => {
+ if (!query[queryParam] || Array.isArray(query[queryParam])) {
+ // throw new Error(
+ // `Invalid query params: unexpected type of ${queryParam}: ${query[queryParam]}}`
+ // );
+ return acc;
+ }
+
+ return {
+ ...acc,
+ [repoKey]: query[queryParam] as string,
+ };
+ }, {} as ImportedRepository),
+ [query]
+ );
+
+ return queryParams;
};
const RepositoryAnalyticsPage = () => {
+ const importedRepository = useImportedRepoFromURL();
+
return (
<BaseLayout sidebar={<Sidebar />}>
- <Charts importedRepository={useImportedRepoFromURL()} />
+ <ImportedRepoMetadata importedRepository={importedRepository} />
+ <Charts importedRepository={importedRepository} />
</BaseLayout>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
index 13b6201..2ae46c6 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx
@@ -1,19 +1,8 @@
import type { AppProps } from "next/app";
-import { SqlProvider, makeSplitgraphHTTPContext } from "@madatdata/react";
-import { useMemo } from "react";
import "../components/global-styles/reset.css";
import "../components/global-styles/theme.css";
export default function GitHubAnalyticsApp({ Component, pageProps }: AppProps) {
- const splitgraphDataContext = useMemo(
- () => makeSplitgraphHTTPContext({ credential: null }),
- []
- );
-
- return (
- <SqlProvider dataContext={splitgraphDataContext}>
- <Component {...pageProps} />
- </SqlProvider>
- );
+ return <Component {...pageProps} />;
}
From 9b581899e5006bae7105c7b897b49a51bc33f548 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Thu, 1 Jun 2023 16:08:16 -0400
Subject: [PATCH 17/36] Refactor Observable plot: Add `useSqlPlot` hook and
make file per plot
---
.../components/RepositoryAnalytics/Charts.tsx | 73 +--------------
.../charts/StargazersChart.tsx | 57 ++++++++++++
.../RepositoryAnalytics/sql-queries.ts | 27 +-----
.../RepositoryAnalytics/useSqlPlot.tsx | 93 +++++++++++++++++++
.../types.ts | 5 +
5 files changed, 159 insertions(+), 96 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
index 0e0f2f3..6eea242 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
@@ -1,16 +1,11 @@
import style from "./Charts.module.css";
-import { useEffect, useRef } from "react";
import type { ImportedRepository } from "../../types";
-import { SqlProvider, makeSeafowlHTTPContext, useSql } from "@madatdata/react";
+import { SqlProvider, makeSeafowlHTTPContext } from "@madatdata/react";
-import * as Plot from "@observablehq/plot";
import { useMemo } from "react";
-import {
- stargazersLineChartQuery,
- type StargazersLineChartRow,
-} from "./sql-queries";
+import { StargazersChart } from "./charts/StargazersChart";
export interface ChartsProps {
importedRepository: ImportedRepository;
@@ -32,71 +27,9 @@ export const Charts = ({ importedRepository }: ChartsProps) => {
return (
<div className={style.charts}>
<SqlProvider dataContext={seafowlDataContext}>
+ <h3>Stargazers</h3>
<StargazersChart {...importedRepository} />
</SqlProvider>
</div>
);
};
-
-const StargazersChart = ({
- splitgraphNamespace,
- splitgraphRepository,
-}: ImportedRepository) => {
- const containerRef = useRef<HTMLDivElement>();
-
- const { response, error } = useSql<StargazersLineChartRow>(
- stargazersLineChartQuery({ splitgraphNamespace, splitgraphRepository })
- );
-
- const stargazers = useMemo(() => {
- return !response || error
- ? []
- : (response.rows ?? []).map((r) => ({
- ...r,
- starred_at: new Date(r.starred_at),
- }));
- }, [response, error]);
-
- useEffect(() => {
- if (stargazers === undefined) {
- return;
- }
-
- const plot = Plot.plot({
- y: { grid: true },
- color: { scheme: "burd" },
- marks: [
- Plot.lineY(stargazers, {
- x: "starred_at",
- y: "cumulative_stars",
- }),
- // NOTE: We don't have username when querying Seafowl because it's within a JSON object,
- // and seafowl doesn't support querying inside JSON objects
- // Plot.tip(
- // stargazers,
- // Plot.pointer({
- // x: "starred_at",
- // y: "cumulative_stars",
- // title: (d) => `${d.username} was stargazer #${d.cumulative_stars}`,
- // })
- // ),
- ],
- });
-
- // There is a bug(?) in useSql where, since we can't give it dependencies, it
- // will re-run even with splitgraphNamespace and splitgraphRepository are undefined,
- // which results in an error querying Seafowl. So just don't render the chart in that case.
- if (splitgraphNamespace && splitgraphRepository) {
- containerRef.current.append(plot);
- }
-
- return () => plot.remove();
- }, [stargazers]);
-
- return (
- <>
- <h3>Stargazers</h3>
- <div ref={containerRef} />
- </>
- );
-};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx
new file mode 100644
index 0000000..72789c7
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx
@@ -0,0 +1,57 @@
+import * as Plot from "@observablehq/plot";
+import { useSqlPlot } from "../useSqlPlot";
+import type { ImportedRepository, TargetSplitgraphRepo } from "../../../types";
+
+// Assume meta namespace contains both the meta tables, and all imported repositories and tables
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+
+/**
+ * A simple line graph showing the number of stargazers over time
+ */
+export const StargazersChart = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+}: ImportedRepository) => {
+ const renderPlot = useSqlPlot({
+ sqlParams: { splitgraphNamespace, splitgraphRepository },
+ buildQuery: stargazersLineChartQuery,
+ mapRows: (r: StargazersLineChartRow) => ({
+ ...r,
+ starred_at: new Date(r.starred_at),
+ }),
+ isRenderable: (p) => !!p.splitgraphRepository,
+ makePlotOptions: (stargazers) => ({
+ y: { grid: true },
+ color: { scheme: "burd" },
+ marks: [
+ Plot.lineY(stargazers, {
+ x: "starred_at",
+ y: "cumulative_stars",
+ }),
+ ],
+ }),
+ });
+
+ return renderPlot();
+};
+
+/** Shape of row returned by {@link stargazersLineChartQuery} */
+export type StargazersLineChartRow = {
+ username: string;
+ cumulative_stars: number;
+ starred_at: string;
+};
+
+/** Time series of GitHub stargazers for the given repository */
+export const stargazersLineChartQuery = ({
+ splitgraphNamespace = META_NAMESPACE,
+ splitgraphRepository,
+}: TargetSplitgraphRepo) => {
+ return `SELECT
+ COUNT(*) OVER (ORDER BY starred_at) AS cumulative_stars,
+ starred_at
+FROM
+ "${splitgraphNamespace}/${splitgraphRepository}"."stargazers"
+ORDER BY starred_at;`;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
index 47e1962..11acec8 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
@@ -1,14 +1,9 @@
-import type { ImportedRepository } from "../../types";
+import type { ImportedRepository, TargetSplitgraphRepo } from "../../types";
// Assume meta namespace contains both the meta tables, and all imported repositories and tables
const META_NAMESPACE =
process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
-type TargetSplitgraphRepo = {
- splitgraphNamespace?: string;
- splitgraphRepository: string;
-};
-
/**
* Raw query to select all columns in the stargazers table, which can be
* run on both Splitgraph and Seafowl.
@@ -32,23 +27,3 @@ FROM
"${splitgraphNamespace}/${splitgraphRepository}"."stargazers"
LIMIT 100;`;
};
-
-/** Shape of row returned by {@link stargazersLineChartQuery} */
-export type StargazersLineChartRow = {
- username: string;
- cumulative_stars: number;
- starred_at: string;
-};
-
-/** Time series of GitHub stargazers for the given repository */
-export const stargazersLineChartQuery = ({
- splitgraphNamespace = META_NAMESPACE,
- splitgraphRepository,
-}: TargetSplitgraphRepo) => {
- return `SELECT
- COUNT(*) OVER (ORDER BY starred_at) AS cumulative_stars,
- starred_at
-FROM
- "${splitgraphNamespace}/${splitgraphRepository}"."stargazers"
-ORDER BY starred_at;`;
-};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
new file mode 100644
index 0000000..65dff18
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
@@ -0,0 +1,93 @@
+import { useCallback, useEffect, useRef } from "react";
+
+import { UnknownObjectShape, useSql } from "@madatdata/react";
+
+import * as Plot from "@observablehq/plot";
+import { useMemo } from "react";
+
+/**
+ * A hook that returns a render function for a Plot chart built from the
+ * results of a SQL query. All of the generic parameters should be inferrable
+ * based on the parameters passed to the `sqlParams` parameter.
+ *
+ * @returns A render function which returns a value that can be returned from a Component
+ */
+export const useSqlPlot = <
+ RowShape extends UnknownObjectShape,
+ SqlParams extends object,
+ MappedRow extends UnknownObjectShape
+>({
+ sqlParams,
+ mapRows,
+ buildQuery,
+ makePlotOptions,
+ isRenderable,
+}: {
+ /**
+ * The input parameters, an object that should match the first and only parameter
+ * of the `buildQuery` callback
+ * */
+ sqlParams: SqlParams;
+ /**
+ * An optional function to map the rows returned by the SQL query to a different
+ * row shape, which is most often useful for things like converting a string column
+ * to a `Date` object.
+ */
+ mapRows?: (row: RowShape) => MappedRow;
+ /**
+ * A builder function that returns a SQL query given a set of parameters, which
+ * will be the parameters passed as the `sqlParams` parameter.
+ */
+ buildQuery: (sqlParams: SqlParams) => string;
+ /**
+ * A function to call after receiving the result of the SQL query (and mapping
+ * its rows if applicable), to create the options given to Observable {@link Plot.plot}
+ */
+ makePlotOptions: (rows: MappedRow[]) => Plot.PlotOptions;
+ /**
+ * A function to call to determine if the chart is renderable. This is helpful
+ * during server side rendering, when Observable Plot doesn't typically work well,
+ * and also when the response from the query is empty, for example because the `useSql`
+ * hook executed before its parameters were set (this works around an inconvenience in
+ * `useSql` where it does not take any parameters and so always executes on first render)
+ */
+ isRenderable?: (sqlParams: SqlParams) => boolean;
+}) => {
+ const containerRef = useRef<HTMLDivElement>();
+
+ const { response, error } = useSql<RowShape>(buildQuery(sqlParams));
+
+ const mappedRows = useMemo(() => {
+ return !response || error
+ ? []
+ : (response.rows ?? []).map(
+ mapRows ?? ((r) => r as unknown as MappedRow)
+ );
+ }, [response, error]);
+
+ const plotOptions = useMemo(() => makePlotOptions(mappedRows), [mappedRows]);
+
+ useEffect(() => {
+ if (mappedRows === undefined) {
+ return;
+ }
+
+ const plot = Plot.plot(plotOptions);
+
+ // There is a bug(?) in useSql where, since we can't give it dependencies, it
+ // will re-run even with splitgraphNamespace and splitgraphRepository are undefined,
+ // which results in an error querying Seafowl. So just don't render the chart in that case.
+ if (!isRenderable || isRenderable(sqlParams)) {
+ containerRef.current.append(plot);
+ }
+
+ return () => plot.remove();
+ }, [mappedRows]);
+
+ const renderPlot = useCallback(
+ () => <div ref={containerRef} />,
+ [containerRef]
+ );
+
+ return renderPlot;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
index 4c6cbc5..8183d8d 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
@@ -4,3 +4,8 @@ export interface ImportedRepository {
splitgraphNamespace: string;
splitgraphRepository: string;
}
+
+export interface TargetSplitgraphRepo {
+ splitgraphNamespace?: string;
+ splitgraphRepository: string;
+}
From 5f4d6b5db54787ee8c6699f49b9ea81cc9308769 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 7 Jun 2023 22:04:33 -0400
Subject: [PATCH 18/36] Refactor: use better name than `acc` in reduce function
---
.../pages/[github_namespace]/[github_repository].tsx | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
index c8baaa3..06cec7d 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
@@ -12,8 +12,6 @@ import { useMemo } from "react";
const useImportedRepoFromURL = () => {
const { query } = useRouter();
- console.log("query:", query);
-
const queryParams = useMemo(
() =>
(
@@ -23,16 +21,16 @@ const useImportedRepoFromURL = () => {
["splitgraphNamespace", "splitgraphNamespace"],
["splitgraphRepository", "splitgraphRepository"],
] as [string, keyof ImportedRepository][]
- ).reduce((acc, [queryParam, repoKey]) => {
+ ).reduce((parsedQueryParams, [queryParam, repoKey]) => {
if (!query[queryParam] || Array.isArray(query[queryParam])) {
// throw new Error(
// `Invalid query params: unexpected type of ${queryParam}: ${query[queryParam]}}`
// );
- return acc;
+ return parsedQueryParams;
}
return {
- ...acc,
+ ...parsedQueryParams,
[repoKey]: query[queryParam] as string,
};
}, {} as ImportedRepository),
From b362bc22abdc076c30b7e273c1358dcaccdd0fd3 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Sat, 17 Jun 2023 02:27:41 +0100
Subject: [PATCH 19/36] Add styling, bells and whistles to stepper (text,
buttons, loading bars, etc.)
---
.../ImportExportStepper/ChartsPanel.tsx | 53 ++++++++
.../ExportPanel.module.css | 29 ++++-
.../ImportExportStepper/ExportPanel.tsx | 95 +++++++++++++--
.../ImportExportStepper/ImportLoadingBar.tsx | 41 ++++++-
.../ImportPanel.module.css | 28 +++++
.../ImportExportStepper/ImportPanel.tsx | 107 +++++++++++++++--
.../StepDescription.module.css | 12 ++
.../ImportExportStepper/StepDescription.tsx | 19 +++
.../ImportExportStepper/StepTitle.module.css | 20 ++++
.../ImportExportStepper/StepTitle.tsx | 27 +++++
.../ImportExportStepper/Stepper.tsx | 3 +-
.../components/LoadingBar.module.css | 92 ++++++++++++++
.../components/LoadingBar.tsx | 68 +++++++++++
.../ImportedRepoMetadata.tsx | 113 ++++++++++++++++--
.../charts/StargazersChart.tsx | 3 +-
.../RepositoryAnalytics/sql-queries.ts | 2 +-
.../RepositoryAnalytics/useSqlPlot.tsx | 10 +-
.../components/Sidebar.module.css | 2 +-
.../components/global-styles/theme.css | 18 +++
.../pages/api/start-export-to-seafowl.ts | 24 ++--
20 files changed, 716 insertions(+), 50 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx
new file mode 100644
index 0000000..1d2dcf4
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx
@@ -0,0 +1,53 @@
+import { Charts } from "../RepositoryAnalytics/Charts";
+import { StepDescription } from "./StepDescription";
+import { StepTitle } from "./StepTitle";
+import { useStepper } from "./StepperContext";
+
+export const ChartsPanel = () => {
+ const [
+ {
+ repository: { namespace: githubNamespace, repository: githubRepository },
+ splitgraphNamespace,
+ splitgraphRepository,
+ stepperState,
+ },
+ ] = useStepper();
+
+ const stepStatus = (() => {
+ switch (stepperState) {
+ case "export_complete":
+ return "active";
+ default:
+ return "unstarted";
+ }
+ })();
+
+ return (
+ <div>
+ <StepTitle
+ status={stepStatus}
+ stepNumber={3}
+ stepTitle="Render charts with Observable Plot"
+ />
+ <StepDescription status={stepStatus}>
+ Once the data is loaded into Seafowl, we can query it with{" "}
+ <a href="https://www.github.com/splitgraph/madatdata" target="_blank">
+ madatdata
+ </a>{" "}
+ and render some charts using{" "}
+ <a href="https://observablehq.com/plot/" target="_blank">
+ Observable Plot
+ </a>
+ .
+ </StepDescription>
+ <Charts
+ importedRepository={{
+ githubNamespace,
+ githubRepository,
+ splitgraphNamespace,
+ splitgraphRepository,
+ }}
+ />
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
index c82f97c..087130c 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
@@ -1,11 +1,7 @@
.exportPanel {
/* Styles for the export panel container */
background: inherit;
-}
-
-.startExportButton {
- /* Styles for the start export button */
- background: inherit;
+ margin-top: 2rem;
}
.querySeafowlButton {
@@ -17,3 +13,26 @@
/* Styles for the view report button */
background: inherit;
}
+
+.startExportButton {
+ color: var(--background);
+ background-color: var(--secondary);
+ padding: 8px;
+ border-radius: 8px;
+ text-decoration: none;
+ font-weight: bold;
+ border-style: none;
+}
+
+.startExportButton:hover {
+ text-shadow: 0 0 5px rgba(43, 0, 255, 0.5);
+ cursor: pointer;
+}
+
+.exportCompleteInfo p {
+ margin-bottom: 1rem;
+}
+
+.exportInfo p {
+ margin-bottom: 1rem;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index bcfd800..9298a42 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -4,7 +4,7 @@ import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
import { ExportLoadingBars } from "./ExportLoadingBars";
-import { relevantGitHubTableNamesForImport } from "../../lib/config/github-tables";
+import { splitgraphTablesToExportToSeafowl } from "../../lib/config/github-tables";
import { makeQueriesToExport } from "../../lib/config/queries-to-export";
import type {
ExportQueryInput,
@@ -13,10 +13,29 @@ import type {
StartExportToSeafowlResponseData,
} from "../../pages/api/start-export-to-seafowl";
import { useMemo, useCallback } from "react";
+import { StepTitle } from "./StepTitle";
+import { StepDescription } from "./StepDescription";
+import {
+ SeafowlEmbeddedQuery,
+ SeafowlStargazersQueryLink,
+} from "../RepositoryAnalytics/ImportedRepoMetadata";
+import {
+ GitHubRepoLink,
+ SplitgraphStargazersQueryLink,
+ SplitgraphEmbeddedQuery,
+} from "../RepositoryAnalytics/ImportedRepoMetadata";
+import { makeStargazersTableQuery } from "../RepositoryAnalytics/sql-queries";
export const ExportPanel = () => {
const [
- { stepperState, exportError, splitgraphRepository, splitgraphNamespace },
+ {
+ stepperState,
+ exportError,
+ splitgraphRepository,
+ splitgraphNamespace,
+ exportedTablesCompleted,
+ repository: githubRepositoryFromStepper,
+ },
dispatch,
] = useStepper();
@@ -32,7 +51,7 @@ export const ExportPanel = () => {
const tablesToExport = useMemo<ExportTableInput[]>(
() =>
- relevantGitHubTableNamesForImport.map((tableName) => ({
+ splitgraphTablesToExportToSeafowl.map((tableName) => ({
namespace: splitgraphNamespace,
repository: splitgraphRepository,
table: tableName,
@@ -40,7 +59,7 @@ export const ExportPanel = () => {
[
splitgraphNamespace,
splitgraphRepository,
- relevantGitHubTableNamesForImport,
+ splitgraphTablesToExportToSeafowl,
]
);
@@ -100,26 +119,76 @@ export const ExportPanel = () => {
return () => abortController.abort();
}, [queriesToExport, tablesToExport, dispatch]);
+ const stepStatus = (() => {
+ switch (stepperState) {
+ case "import_complete":
+ return "active";
+ case "awaiting_export":
+ return "loading";
+ case "export_complete":
+ return "completed";
+ default:
+ return "unstarted";
+ }
+ })();
+
return (
<div className={styles.exportPanel}>
+ <StepTitle
+ stepNumber={2}
+ stepTitle={"Export Data from Splitgraph to Seafowl"}
+ status={stepStatus}
+ />
+ <StepDescription status={stepStatus}>
+ {stepStatus === "completed" ? (
+ <div className={styles.exportCompleteInfo}>
+ <p>
+ ✓ Export complete! We successfully imported tables and
+ queries from Splitgraph to our{" "}
+ <a href="https://seafowl.io" target="_blank">
+ Seafowl
+ </a>{" "}
+ instance running at <code>https://demo.seafowl.cloud</code>. Now
+ we can query it and get cache-optimized responses for rendering
+ charts and analytics.
+ </p>
+ <p>
+ <strong>Query Data: </strong>
+ <SeafowlStargazersQueryLink
+ splitgraphNamespace={splitgraphNamespace}
+ splitgraphRepository={splitgraphRepository}
+ />
+ </p>
+ <SeafowlEmbeddedQuery
+ importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+ tableName={"stargazers"}
+ makeQuery={makeStargazersTableQuery}
+ />
+ </div>
+ ) : (
+ <div className={styles.exportInfo}>
+ Now let's export some tables and pre-made queries from our staging
+ area in Splitgraph to our cache-optimized{" "}
+ <a href="https://seafowl.io" target="_blank">
+ Seafowl
+ </a>{" "}
+ instance running at <code>https://demo.seafowl.cloud</code>.{" "}
+ {stepStatus === "active" && (
+ <> Click the button to start the export.</>
+ )}
+ </div>
+ )}
+ </StepDescription>
{exportError && <p className={styles.error}>{exportError}</p>}
{stepperState === "import_complete" && (
<button
className={styles.startExportButton}
onClick={handleStartExport}
>
- Start Export
+ Start Export of Tables and Queries from Splitgraph to Seafowl
</button>
)}
{stepperState === "awaiting_export" && <ExportLoadingBars />}
- {stepperState === "export_complete" && (
- <>
- <button className={styles.querySeafowlButton}>
- Query Seafowl in Splitgraph Console
- </button>
- <button className={styles.viewReportButton}>View Report</button>
- </>
- )}
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
index 3795a61..443210c 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx
@@ -1,16 +1,21 @@
import { useEffect } from "react";
import { useStepper } from "./StepperContext";
+import { LoadingBar } from "../LoadingBar";
type ImportLoadingBarProps = {
taskId: string;
splitgraphNamespace: string;
splitgraphRepository: string;
+ githubNamespace: string;
+ githubRepository: string;
};
export const ImportLoadingBar: React.FC<ImportLoadingBarProps> = ({
taskId,
splitgraphNamespace,
splitgraphRepository,
+ githubNamespace,
+ githubRepository,
}) => {
const [{ stepperState }, dispatch] = useStepper();
@@ -69,5 +74,39 @@ export const ImportLoadingBar: React.FC<ImportLoadingBarProps> = ({
dispatch,
]);
- return <div>Loading...</div>;
+ return (
+ <div>
+ <LoadingBar
+ title={
+ <div style={{ textAlign: "center" }}>
+ <p>
+ Importing tables from{" "}
+ <a
+ href={`https://github.com/${githubNamespace}/${githubRepository}`}
+ target="_blank"
+ >
+ github.com/{`${githubNamespace}/${githubRepository}`}
+ </a>
+ </p>
+
+ <p>
+ into:{" "}
+ <a
+ href={`https://www.splitgraph.com/${splitgraphNamespace}/${splitgraphRepository}`}
+ target="_blank"
+ >
+ splitgraph.com/
+ {`${splitgraphNamespace}/${splitgraphRepository}`}
+ </a>
+ </p>
+ </div>
+ }
+ >
+ <p>
+ This might take 5-10 minutes depending on the size of the GitHub
+ repository.
+ </p>
+ </LoadingBar>
+ </div>
+ );
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css
index ea3961d..ecca0bc 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css
@@ -8,3 +8,31 @@
border: 1px solid var(--sidebar);
margin-bottom: 8px;
}
+
+.repoNameInput {
+ margin-right: 0.25rem;
+ width: 50ch;
+}
+
+.repoNameForm {
+ display: flex;
+}
+
+.startImportButton {
+ color: var(--background);
+ background-color: var(--secondary);
+ padding: 8px;
+ border-radius: 8px;
+ text-decoration: none;
+ font-weight: bold;
+ border-style: none;
+}
+
+.startImportButton:hover {
+ text-shadow: 0 0 5px rgba(43, 0, 255, 0.5);
+ cursor: pointer;
+}
+
+.importCompleteInfo p {
+ margin-bottom: 1rem;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
index b57f2ef..44ae8b4 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
@@ -1,8 +1,16 @@
import { useState } from "react";
import { useStepper } from "./StepperContext";
import { ImportLoadingBar } from "./ImportLoadingBar";
+import { StepTitle } from "./StepTitle";
import styles from "./ImportPanel.module.css";
+import { StepDescription } from "./StepDescription";
+import {
+ GitHubRepoLink,
+ SplitgraphStargazersQueryLink,
+ SplitgraphEmbeddedQuery,
+} from "../RepositoryAnalytics/ImportedRepoMetadata";
+import { makeStargazersTableQuery } from "../RepositoryAnalytics/sql-queries";
export const ImportPanel = () => {
const [
@@ -12,6 +20,7 @@ export const ImportPanel = () => {
importError,
splitgraphNamespace,
splitgraphRepository,
+ repository: githubRepositoryFromStepper,
},
dispatch,
] = useStepper();
@@ -78,19 +87,100 @@ export const ImportPanel = () => {
return /^[\w-.]+\/[\w-.]+$/.test(repoName);
};
+ const stepStatus: React.ComponentProps<typeof StepTitle>["status"] = (() => {
+ switch (stepperState) {
+ case "import_complete":
+ case "awaiting_export":
+ case "export_complete":
+ return "completed";
+ case "awaiting_import":
+ return "loading";
+ default:
+ return "active";
+ }
+ })();
+
return (
<div className={styles.importPanel}>
+ <StepTitle
+ stepTitle="Import GitHub repository data to Splitgraph"
+ stepNumber={1}
+ status={stepStatus}
+ />
+ <StepDescription status={stepStatus}>
+ {stepStatus === "completed" ? (
+ <div className={styles.importCompleteInfo}>
+ <p>
+ ✓ Import complete! We successfully imported data from{" "}
+ <GitHubRepoLink
+ githubNamespace={githubRepositoryFromStepper.namespace}
+ githubRepository={githubRepositoryFromStepper.repository}
+ />{" "}
+ into Splitgraph.
+ </p>
+ <p>
+ <strong>Browse Data:</strong>{" "}
+ <a
+ href={`https://www.splitgraph.com/${splitgraphNamespace}/${splitgraphRepository}`}
+ target="_blank"
+ >
+ splitgraph.com/
+ {`${splitgraphNamespace}/${splitgraphRepository}`}
+ </a>
+ </p>
+ <p>
+ <strong>Query Data:</strong>
+ {" "}
+ <SplitgraphStargazersQueryLink
+ splitgraphNamespace={splitgraphNamespace}
+ splitgraphRepository={splitgraphRepository}
+ />
+ </p>
+ <SplitgraphEmbeddedQuery
+ importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+ tableName={"stargazers"}
+ makeQuery={makeStargazersTableQuery}
+ />
+ </div>
+ ) : (
+ <>
+ We'll use the{" "}
+ <a
+ href="https://www.splitgraph.com/connect/data/github"
+ target="_blank"
+ >
+ airbyte-github
+ </a>{" "}
+ plugin to import data about this GitHub repository to the Splitgraph
+ Data Delivery Network (DDN). Then you'll be able to browse the data
+ in the Splitgraph catalog, or query it with{" "}
+ <a href="https://www.splitgraph.com/connect/query" target="_blank">
+ your favorite Postgres Client
+ </a>{" "}
+ or with the{" "}
+ <a href="https://www.splitgraph.com/query" target="_blank">
+ Splitgraph Query Console
+ </a>
+ .
+ </>
+ )}
+ </StepDescription>
{stepperState === "unstarted" && (
<>
{importError && <p className={styles.error}>{importError}</p>}
- <form onSubmit={handleInputSubmit}>
+ <form onSubmit={handleInputSubmit} className={styles.repoNameForm}>
<input
type="text"
- placeholder="Enter repository name"
+ placeholder="GitHub repository name, e.g. splitgraph/seafowl"
value={inputValue}
onChange={(e) => setInputValue(e.target.value)}
+ className={styles.repoNameInput}
+ tabIndex={1}
+ autoFocus={true}
/>
- <button type="submit">Start Import</button>
+ <button type="submit" className={styles.startImportButton}>
+ Start Import
+ </button>
</form>
</>
)}
@@ -99,13 +189,14 @@ export const ImportPanel = () => {
taskId={importTaskId}
splitgraphNamespace={splitgraphNamespace}
splitgraphRepository={splitgraphRepository}
+ githubNamespace={
+ githubRepositoryFromStepper.namespace ?? inputValue.split("/")[0]
+ }
+ githubRepository={
+ githubRepositoryFromStepper.repository ?? inputValue.split("/")[1]
+ }
/>
)}
- {stepperState === "import_complete" && (
- <div>
- <p>Import Complete</p>
- </div>
- )}
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.module.css
new file mode 100644
index 0000000..4b64378
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.module.css
@@ -0,0 +1,12 @@
+.stepDescriptionContainer {
+ margin-bottom: 1rem;
+}
+
+.stepDescriptionContainer a {
+ color: var(--primary);
+ text-decoration: none;
+}
+
+.stepDescriptionContainer a:hover {
+ text-decoration: underline;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.tsx
new file mode 100644
index 0000000..c13aac8
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepDescription.tsx
@@ -0,0 +1,19 @@
+import styles from "./StepDescription.module.css";
+
+export const StepDescription = ({
+ children,
+ status,
+}: React.PropsWithChildren<{
+ status: "active" | "completed" | "unstarted" | "loading";
+}>) => {
+ return (
+ <div
+ className={[
+ styles.stepDescriptionContainer,
+ styles[`step-description-status-${status}`],
+ ].join(" ")}
+ >
+ {typeof children === "string" ? <p>{children}</p> : children}
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.module.css
new file mode 100644
index 0000000..2659538
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.module.css
@@ -0,0 +1,20 @@
+div.stepTitleContainer {
+ display: flex;
+ align-items: baseline;
+ border-bottom: 1px solid var(--muted);
+ margin-bottom: 1rem;
+}
+
+h2.stepNumber {
+ color: var(--sidebar);
+ opacity: 50%;
+ margin-right: 1rem;
+}
+
+h2.stepTitle {
+ color: var(--subtext);
+}
+
+.step-status-active .stepTitle {
+ color: var(--text);
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.tsx
new file mode 100644
index 0000000..3255bc4
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepTitle.tsx
@@ -0,0 +1,27 @@
+import styles from "./StepTitle.module.css";
+
+export const StepTitle = ({
+ stepNumber,
+ stepTitle,
+ status,
+}: {
+ stepNumber: number;
+ stepTitle: string;
+ status: "active" | "completed" | "unstarted" | "loading";
+}) => {
+ return (
+ <div
+ className={[
+ styles.stepTitleContainer,
+ styles[`step-status-${status}`],
+ ].join(" ")}
+ >
+ <h2 key="step-number" className={styles.stepNumber}>
+ {stepNumber.toString()}
+ </h2>
+ <h2 key="step-title" className={styles.stepTitle}>
+ {stepTitle}
+ </h2>
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
index 101ad5f..504eca4 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx
@@ -4,7 +4,7 @@ import { ImportPanel } from "./ImportPanel";
import { ExportPanel } from "./ExportPanel";
import styles from "./Stepper.module.css";
-import { useRouter } from "next/router";
+import { ChartsPanel } from "./ChartsPanel";
const StepperOrLoading = ({ children }: { children: React.ReactNode }) => {
const [{ stepperState, debug }] = useStepper();
@@ -30,6 +30,7 @@ export const Stepper = () => {
<StepperOrLoading>
<ImportPanel />
<ExportPanel />
+ <ChartsPanel />
</StepperOrLoading>
</div>
</StepperContextProvider>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.module.css
new file mode 100644
index 0000000..9cb9c33
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.module.css
@@ -0,0 +1,92 @@
+.loaderContainer {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ width: 100%;
+}
+
+.loadingBox {
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ height: 100%;
+ border: 1px dashed var(--primary);
+ padding: 2rem;
+}
+
+.loadingTitle {
+ font-size: medium;
+}
+
+.loaderInBox {
+ margin-bottom: 2rem !important;
+ margin-top: 2rem !important;
+}
+
+.loaderIsolated {
+ margin: 0 auto;
+}
+
+.timeElapsed {
+ font-size: small;
+ color: var(--muted);
+}
+
+/*
+ FOLLOWING CODE IS COPIED FROM: https://codepen.io/stoepke/pen/QOOqGW
+*/
+
+.loader {
+ width: 100%;
+ margin: 0 auto;
+ border-radius: 10px;
+ border: 4px solid transparent;
+ position: relative;
+ padding: 1px;
+}
+.loader:before {
+ content: "";
+ border: 1px solid var(--primary);
+ border-radius: 10px;
+ position: absolute;
+ top: -4px;
+ right: -4px;
+ bottom: -4px;
+ left: -4px;
+}
+.loader .loaderBar {
+ position: absolute;
+ border-radius: 10px;
+ top: 0;
+ right: 100%;
+ bottom: 0;
+ left: 0;
+ background: var(--primary);
+ width: 0;
+ animation: borealisBar 2s linear infinite;
+}
+
+@keyframes borealisBar {
+ 0% {
+ left: 0%;
+ right: 100%;
+ width: 0%;
+ }
+ 10% {
+ left: 0%;
+ right: 75%;
+ width: 25%;
+ }
+ 90% {
+ right: 0%;
+ left: 75%;
+ width: 25%;
+ }
+ 100% {
+ left: 100%;
+ right: 0%;
+ width: 0%;
+ }
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx
new file mode 100644
index 0000000..29d53f3
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx
@@ -0,0 +1,68 @@
+import styles from "./LoadingBar.module.css";
+import { useState, useEffect } from "react";
+
+export const LoadingBar = ({
+ children,
+ title,
+}: {
+ children?: React.ReactNode;
+ title?: React.ReactNode;
+}) => {
+ return children || title ? (
+ <LoadingBarWithText title={title}>{children}</LoadingBarWithText>
+ ) : (
+ <JustLoadingBar inBox={false} />
+ );
+};
+
+export const JustLoadingBar = ({ inBox }: { inBox: boolean }) => {
+ return (
+ <div
+ className={[
+ styles.loaderContainer,
+ inBox ? styles.loaderInBox : styles.loaderIsolated,
+ ].join(" ")}
+ >
+ <div className={styles.loader}>
+ <div className={styles.loaderBar}></div>
+ </div>
+ <TimeElapsed />
+ </div>
+ );
+};
+
+export const LoadingBarWithText = ({
+ children,
+ title,
+}: {
+ children?: React.ReactNode;
+ title?: React.ReactNode;
+}) => {
+ return (
+ <div className={styles.loadingBox}>
+ {title ? <h2 className={styles.loadingTitle}>{title}</h2> : null}
+ <JustLoadingBar inBox={true} />
+ {children}
+ </div>
+ );
+};
+
+const TimeElapsed = () => {
+ const [seconds, setSeconds] = useState<number>(0);
+
+ useEffect(() => {
+ const intervalId = setInterval(() => {
+ setSeconds((prevSeconds) => prevSeconds + 1);
+ }, 1000);
+
+ return () => {
+ clearInterval(intervalId);
+ };
+ }, []);
+
+ return (
+ <div className={styles.timeElapsed}>
+ {seconds < 2 ? <> </> : <>Time elapsed: {seconds} seconds...</>}
+ </div>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
index b48d71b..59bddc1 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
@@ -1,3 +1,4 @@
+import { ComponentProps } from "react";
import type { ImportedRepository } from "../../types";
import style from "./ImportedRepoMetadata.module.css";
@@ -11,6 +12,11 @@ interface ImportedRepoMetadataProps {
importedRepository: ImportedRepository;
}
+type SplitgraphRepository = Pick<
+ ImportedRepository,
+ "splitgraphNamespace" | "splitgraphRepository"
+>;
+
export const ImportedRepoMetadata = ({
importedRepository,
}: ImportedRepoMetadataProps) => {
@@ -40,14 +46,22 @@ export const ImportedRepoMetadata = ({
/>
</li>
</ul>
+ <SeafowlEmbeddedQuery
+ importedRepository={importedRepository}
+ tableName={"stargazers"}
+ makeQuery={makeStargazersTableQuery}
+ />
</div>
);
};
-const SplitgraphRepoLink = ({
+export const SplitgraphRepoLink = ({
splitgraphNamespace,
splitgraphRepository,
-}: ImportedRepository) => {
+}: Pick<
+ ImportedRepository,
+ "splitgraphNamespace" | "splitgraphRepository"
+>) => {
return (
<a
target="_blank"
@@ -58,10 +72,10 @@ const SplitgraphRepoLink = ({
);
};
-const GitHubRepoLink = ({
+export const GitHubRepoLink = ({
githubNamespace,
githubRepository,
-}: ImportedRepository) => {
+}: Pick<ImportedRepository, "githubNamespace" | "githubRepository">) => {
return (
<a
target="_blank"
@@ -70,13 +84,13 @@ const GitHubRepoLink = ({
);
};
-const SplitgraphQueryLink = ({
+export const SplitgraphQueryLink = ({
importedRepository,
makeQuery,
tableName,
}: {
- importedRepository: ImportedRepository;
- makeQuery: (repo: ImportedRepository) => string;
+ importedRepository: SplitgraphRepository;
+ makeQuery: (repo: SplitgraphRepository) => string;
tableName: string;
}) => {
return (
@@ -84,18 +98,42 @@ const SplitgraphQueryLink = ({
href={makeSplitgraphQueryHref(makeQuery(importedRepository))}
target="_blank"
>
- Query {tableName} in the Splitgraph Console
+ Query the {tableName} table in the Splitgraph Console
</a>
);
};
+export const SplitgraphStargazersQueryLink = ({
+ ...importedRepository
+}: SplitgraphRepository) => {
+ return (
+ <SplitgraphQueryLink
+ importedRepository={importedRepository}
+ tableName={"stargazers"}
+ makeQuery={makeStargazersTableQuery}
+ />
+ );
+};
+
+export const SeafowlStargazersQueryLink = ({
+ ...importedRepository
+}: SplitgraphRepository) => {
+ return (
+ <SeafowlQueryLink
+ importedRepository={importedRepository}
+ tableName={"stargazers"}
+ makeQuery={makeStargazersTableQuery}
+ />
+ );
+};
+
const SeafowlQueryLink = ({
importedRepository,
makeQuery,
tableName,
}: {
- importedRepository: ImportedRepository;
- makeQuery: (repo: ImportedRepository) => string;
+ importedRepository: SplitgraphRepository;
+ makeQuery: (repo: SplitgraphRepository) => string;
tableName: string;
}) => {
return (
@@ -127,3 +165,58 @@ const makeSeafowlQueryHref = (sqlQuery: string) => {
"database-name": META_NAMESPACE,
})}`;
};
+
+export const SplitgraphEmbeddedQuery = ({
+ importedRepository,
+ makeQuery,
+}: {
+ importedRepository: SplitgraphRepository;
+ makeQuery: (repo: SplitgraphRepository) => string;
+ tableName: string;
+}) => {
+ return (
+ <iframe
+ src={makeSplitgraphEmbeddableQueryHref(makeQuery(importedRepository))}
+ allowFullScreen={false}
+ style={{ border: "none" }}
+ height={"400px"}
+ width={"80%"}
+ />
+ );
+};
+
+export const SeafowlEmbeddedQuery = ({
+ importedRepository,
+ makeQuery,
+}: {
+ importedRepository: SplitgraphRepository;
+ makeQuery: (repo: SplitgraphRepository) => string;
+ tableName: string;
+}) => {
+ return (
+ <iframe
+ src={makeSeafowlEmbeddableQueryHref(makeQuery(importedRepository))}
+ style={{ border: "none" }}
+ height={"400px"}
+ width={"80%"}
+ />
+ );
+};
+
+/** Return the URL to Splitgraph Console pointing to Seafowl db where we export tables */
+const makeSeafowlEmbeddableQueryHref = (sqlQuery: string) => {
+ return `https://www.splitgraph.com/embeddable-seafowl-console/query-editor?${new URLSearchParams(
+ {
+ "sql-query": sqlQuery,
+ // Splitgraph exports to Seafowl dbname matching the username of the exporting user
+ database: META_NAMESPACE,
+ }
+ )}`;
+};
+
+const makeSplitgraphEmbeddableQueryHref = (sqlQuery: string) => {
+ return `https://www.splitgraph.com/embed/workspace/ddn?${new URLSearchParams({
+ layout: "hsplit",
+ query: sqlQuery,
+ })}`;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx
index 72789c7..f45f928 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/StargazersChart.tsx
@@ -51,7 +51,6 @@ export const stargazersLineChartQuery = ({
return `SELECT
COUNT(*) OVER (ORDER BY starred_at) AS cumulative_stars,
starred_at
-FROM
- "${splitgraphNamespace}/${splitgraphRepository}"."stargazers"
+FROM "${splitgraphNamespace}/${splitgraphRepository}"."stargazers"
ORDER BY starred_at;`;
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
index 11acec8..3463d3b 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/sql-queries.ts
@@ -1,4 +1,4 @@
-import type { ImportedRepository, TargetSplitgraphRepo } from "../../types";
+import type { TargetSplitgraphRepo } from "../../types";
// Assume meta namespace contains both the meta tables, and all imported repositories and tables
const META_NAMESPACE =
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
index 65dff18..3a2fc5d 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
@@ -55,7 +55,15 @@ export const useSqlPlot = <
}) => {
const containerRef = useRef<HTMLDivElement>();
- const { response, error } = useSql<RowShape>(buildQuery(sqlParams));
+ const sqlQueryIfReady = () => {
+ if (isRenderable && !isRenderable(sqlParams)) {
+ return null;
+ }
+
+ return buildQuery(sqlParams);
+ };
+
+ const { response, error } = useSql<RowShape>(sqlQueryIfReady());
const mappedRows = useMemo(() => {
return !response || error
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css
index a39db81..5846221 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css
@@ -16,7 +16,7 @@
}
.importButton {
- color: var(--background);
+ color: var(--background) !important;
background-color: var(--secondary);
padding: 16px;
border-radius: 16px;
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
index 66c4514..70f1893 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css
@@ -8,8 +8,26 @@
--text: #1a202c;
--subtext: #718096;
--danger: #eb8585;
+ --muted: rgba(113, 128, 150, 0.5);
}
body {
font-family: Consolas, Monaco, "Andale Mono", "Ubuntu Mono", monospace;
}
+
+body a {
+ color: var(--primary);
+ text-decoration: none;
+}
+
+body a:hover {
+ text-decoration: underline;
+}
+
+body a:visited {
+ color: var(--primary);
+}
+
+code {
+ font-size: 1rem;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
index f616d37..9b9f8cf 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
@@ -154,13 +154,23 @@ const startExport = async ({
throw new Error(JSON.stringify(response.error));
}
- const loadingTables = response.taskIds.tables.map(
- (t: { jobId: string; sourceTable: string; sourceRepository: string }) => ({
- taskId: t.jobId,
- destinationTable: t.sourceTable,
- destinationSchema: t.sourceRepository,
- })
- );
+ const loadingTableTaskId =
+ response.taskIds.tables.length === 1
+ ? response.taskIds.tables[0].jobId
+ : null;
+
+ // NOTE: We return a list of tables, and duplicate the taskId in each of them,
+ // even though there is only one taskId for all tables. It's up to the frontend
+ // how many requests it wants to send when polling for status updates.
+ const loadingTables = loadingTableTaskId
+ ? response.taskIds.tables[0].tables.map(
+ (t: { sourceTable: string; sourceRepository: string }) => ({
+ taskId: loadingTableTaskId,
+ destinationTable: t.sourceTable,
+ destinationSchema: t.sourceRepository,
+ })
+ )
+ : [];
const loadingQueries = response.taskIds.queries.map(
(
From 4454199863de83ba6c11ca382bdee444634e4eb6 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Mon, 19 Jun 2023 21:16:45 +0100
Subject: [PATCH 20/36] Refactor
---
.../ImportExportStepper/ExportPanel.tsx | 128 +++++++++++++++++-
.../ImportedRepoMetadata.tsx | 2 +-
2 files changed, 123 insertions(+), 7 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index 9298a42..b910b07 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -1,5 +1,7 @@
// components/ImportExportStepper/ExportPanel.tsx
+import { ComponentProps, Fragment } from "react";
+
import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
import { ExportLoadingBars } from "./ExportLoadingBars";
@@ -181,14 +183,128 @@ export const ExportPanel = () => {
</StepDescription>
{exportError && <p className={styles.error}>{exportError}</p>}
{stepperState === "import_complete" && (
- <button
- className={styles.startExportButton}
- onClick={handleStartExport}
- >
- Start Export of Tables and Queries from Splitgraph to Seafowl
- </button>
+ <ExportPreview
+ handleStartExport={handleStartExport}
+ tablesToExport={tablesToExport}
+ queriesToExport={queriesToExport}
+ splitgraphRepository={splitgraphRepository}
+ splitgraphNamespace={splitgraphNamespace}
+ />
)}
{stepperState === "awaiting_export" && <ExportLoadingBars />}
</div>
);
};
+
+const ExportPreview = ({
+ handleStartExport,
+ tablesToExport,
+ queriesToExport,
+ splitgraphRepository,
+ splitgraphNamespace,
+}: {
+ handleStartExport: () => Promise<() => void>;
+ tablesToExport: ExportTableInput[];
+ queriesToExport: ExportQueryInput[];
+ splitgraphRepository: string;
+ splitgraphNamespace: string;
+}) => {
+ return (
+ <>
+ <button className={styles.startExportButton} onClick={handleStartExport}>
+ Start Export of Tables and Queries from Splitgraph to Seafowl
+ </button>
+ {/*
+ <h3>Debugging</h3>
+
+ <SplitgraphEmbeddedQuery
+ importedRepository={{
+ splitgraphNamespace,
+ splitgraphRepository,
+ }}
+ tableName={"Debugging"}
+ makeQuery={() => "select 1;"}
+ /> */}
+
+ <h3>Tables to Export</h3>
+ {tablesToExport
+ .filter((_) => true)
+ .map((exportTable) => (
+ <ExportTablePreview
+ key={`export-table-preview-${exportTable.table}`}
+ splitgraphNamespace={splitgraphNamespace}
+ splitgraphRepository={splitgraphRepository}
+ {...exportTable}
+ />
+ ))}
+
+ <h3>Queries to Export</h3>
+ {queriesToExport
+ .filter((_) => true)
+ .map((exportQuery) => (
+ <ExportQueryPreview
+ key={`export-query-preview-${exportQuery.destinationTable}-${exportQuery.destinationSchema}`}
+ splitgraphNamespace={splitgraphNamespace}
+ splitgraphRepository={splitgraphRepository}
+ {...exportQuery}
+ />
+ ))}
+ </>
+ );
+};
+
+const ExportQueryPreview = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+ destinationSchema,
+ destinationTable,
+ sourceQuery,
+}: ExportQueryInput & {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+}) => {
+ return (
+ <>
+ <h4>
+ <code>
+ {destinationSchema}.{destinationTable}
+ </code>
+ </h4>
+ <SplitgraphEmbeddedQuery
+ importedRepository={{
+ splitgraphNamespace,
+ splitgraphRepository,
+ }}
+ tableName={destinationTable}
+ makeQuery={() => sourceQuery}
+ />
+ </>
+ );
+};
+
+const ExportTablePreview = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+ table,
+}: {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+} & ExportTableInput) => {
+ return (
+ <>
+ <h4>
+ <code>{table}</code>
+ </h4>
+ <SplitgraphEmbeddedQuery
+ importedRepository={{
+ splitgraphNamespace,
+ splitgraphRepository,
+ }}
+ tableName={table}
+ makeQuery={({ splitgraphNamespace, splitgraphRepository }) =>
+ `SELECT * FROM "${splitgraphNamespace}/${splitgraphRepository}"."${table}";`
+ }
+ />
+ </>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
index 59bddc1..6903ecd 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
@@ -216,7 +216,7 @@ const makeSeafowlEmbeddableQueryHref = (sqlQuery: string) => {
const makeSplitgraphEmbeddableQueryHref = (sqlQuery: string) => {
return `https://www.splitgraph.com/embed/workspace/ddn?${new URLSearchParams({
- layout: "hsplit",
+ layout: "query",
query: sqlQuery,
})}`;
};
From cb716043854a21c0eda5d6ab4948e3e33a39da85 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Mon, 19 Jun 2023 23:44:02 +0100
Subject: [PATCH 21/36] Display preview table for each loading/completed:
either embed Splitgraph, or Seafowl
---
.../ImportExportStepper/ExportPanel.tsx | 163 +++++++++++-------
.../ImportExportStepper/stepper-states.ts | 2 +-
.../ImportedRepoMetadata.tsx | 18 +-
3 files changed, 107 insertions(+), 76 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index b910b07..e89de0d 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -18,6 +18,7 @@ import { useMemo, useCallback } from "react";
import { StepTitle } from "./StepTitle";
import { StepDescription } from "./StepDescription";
import {
+ type EmbeddedQueryProps,
SeafowlEmbeddedQuery,
SeafowlStargazersQueryLink,
} from "../RepositoryAnalytics/ImportedRepoMetadata";
@@ -27,6 +28,9 @@ import {
SplitgraphEmbeddedQuery,
} from "../RepositoryAnalytics/ImportedRepoMetadata";
import { makeStargazersTableQuery } from "../RepositoryAnalytics/sql-queries";
+import type { ExportTable } from "./stepper-states";
+
+import type { TargetSplitgraphRepo } from "../../types";
export const ExportPanel = () => {
const [
@@ -182,7 +186,9 @@ export const ExportPanel = () => {
)}
</StepDescription>
{exportError && <p className={styles.error}>{exportError}</p>}
- {stepperState === "import_complete" && (
+ {["import_complete", "awaiting_export", "export_complete"].includes(
+ stepperState
+ ) && (
<ExportPreview
handleStartExport={handleStartExport}
tablesToExport={tablesToExport}
@@ -214,27 +220,23 @@ const ExportPreview = ({
<button className={styles.startExportButton} onClick={handleStartExport}>
Start Export of Tables and Queries from Splitgraph to Seafowl
</button>
- {/*
- <h3>Debugging</h3>
-
- <SplitgraphEmbeddedQuery
- importedRepository={{
- splitgraphNamespace,
- splitgraphRepository,
- }}
- tableName={"Debugging"}
- makeQuery={() => "select 1;"}
- /> */}
-
<h3>Tables to Export</h3>
{tablesToExport
.filter((_) => true)
.map((exportTable) => (
- <ExportTablePreview
+ <ExportEmbedPreviewTableOrQuery
key={`export-table-preview-${exportTable.table}`}
- splitgraphNamespace={splitgraphNamespace}
- splitgraphRepository={splitgraphRepository}
- {...exportTable}
+ exportInput={exportTable}
+ importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+ makeQuery={({ splitgraphNamespace, splitgraphRepository, table }) =>
+ `SELECT * FROM "${splitgraphNamespace}/${splitgraphRepository}"."${table}";`
+ }
+ makeMatchInputToExported={(exportTableInput) => (exportTable) => {
+ return (
+ exportTable.destinationSchema === exportTableInput.repository &&
+ exportTable.destinationTable === exportTableInput.table
+ );
+ }}
/>
))}
@@ -242,69 +244,100 @@ const ExportPreview = ({
{queriesToExport
.filter((_) => true)
.map((exportQuery) => (
- <ExportQueryPreview
+ <ExportEmbedPreviewTableOrQuery
key={`export-query-preview-${exportQuery.destinationTable}-${exportQuery.destinationSchema}`}
- splitgraphNamespace={splitgraphNamespace}
- splitgraphRepository={splitgraphRepository}
- {...exportQuery}
+ exportInput={exportQuery}
+ importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+ makeQuery={({ sourceQuery }) => sourceQuery}
+ makeMatchInputToExported={(exportQueryInput) =>
+ (exportTable: ExportTable) => {
+ return (
+ exportTable.destinationSchema ===
+ exportQueryInput.destinationSchema &&
+ exportTable.destinationTable ===
+ exportQueryInput.destinationTable
+ );
+ }}
/>
))}
</>
);
};
-const ExportQueryPreview = ({
- splitgraphNamespace,
- splitgraphRepository,
- destinationSchema,
- destinationTable,
- sourceQuery,
-}: ExportQueryInput & {
- splitgraphNamespace: string;
- splitgraphRepository: string;
-}) => {
- return (
- <>
- <h4>
- <code>
- {destinationSchema}.{destinationTable}
- </code>
- </h4>
- <SplitgraphEmbeddedQuery
- importedRepository={{
- splitgraphNamespace,
- splitgraphRepository,
- }}
- tableName={destinationTable}
- makeQuery={() => sourceQuery}
- />
- </>
+const useFindMatchingExportTable = (
+ isMatch: (candidateTable: ExportTable) => boolean
+) => {
+ const [{ exportedTablesLoading, exportedTablesCompleted }] = useStepper();
+
+ const matchingCompletedTable = useMemo(
+ () => Array.from(exportedTablesCompleted).find(isMatch),
+ [exportedTablesCompleted, isMatch]
+ );
+ const matchingLoadingTable = useMemo(
+ () => Array.from(exportedTablesLoading).find(isMatch),
+ [exportedTablesLoading, isMatch]
);
+
+ const completed = matchingCompletedTable ?? false;
+ const loading = matchingLoadingTable ?? false;
+ const unstarted = !completed && !loading;
+
+ return {
+ completed,
+ loading,
+ unstarted,
+ };
};
-const ExportTablePreview = ({
- splitgraphNamespace,
- splitgraphRepository,
- table,
+const ExportEmbedPreviewTableOrQuery = <
+ ExportInputShape extends ExportQueryInput | ExportTableInput
+>({
+ importedRepository,
+ exportInput,
+ makeQuery,
+ makeMatchInputToExported,
}: {
- splitgraphNamespace: string;
- splitgraphRepository: string;
-} & ExportTableInput) => {
+ exportInput: ExportInputShape;
+ makeQuery: (
+ tableOrQueryInput: ExportInputShape & {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+ }
+ ) => string;
+ makeMatchInputToExported: (
+ tableOrQueryInput: ExportInputShape
+ ) => (exported: ExportTable) => boolean;
+ importedRepository: {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+ };
+}) => {
+ const embedProps = {
+ importedRepository,
+ tableName:
+ "destinationTable" in exportInput
+ ? exportInput.destinationTable
+ : exportInput.table,
+ makeQuery: () => makeQuery({ ...exportInput, ...importedRepository }),
+ };
+
+ const { unstarted, loading, completed } = useFindMatchingExportTable(
+ makeMatchInputToExported(exportInput)
+ );
+
+ const heading =
+ "table" in exportInput
+ ? exportInput.table
+ : `${exportInput.destinationSchema}.${exportInput.destinationTable}`;
+
return (
<>
<h4>
- <code>{table}</code>
+ <code>{heading}</code>
</h4>
- <SplitgraphEmbeddedQuery
- importedRepository={{
- splitgraphNamespace,
- splitgraphRepository,
- }}
- tableName={table}
- makeQuery={({ splitgraphNamespace, splitgraphRepository }) =>
- `SELECT * FROM "${splitgraphNamespace}/${splitgraphRepository}"."${table}";`
- }
- />
+ <pre>{JSON.stringify({ completed, loading }, null, 2)}</pre>
+ {(unstarted || loading) && <SplitgraphEmbeddedQuery {...embedProps} />}
+ {completed && <SeafowlEmbeddedQuery {...embedProps} />}
</>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index 5895bdd..0030053 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -3,7 +3,7 @@ import { ParsedUrlQuery } from "querystring";
import { useEffect, useReducer } from "react";
export type GitHubRepository = { namespace: string; repository: string };
-type ExportTable = {
+export type ExportTable = {
destinationSchema: string;
destinationTable: string;
taskId: string;
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
index 6903ecd..bb75a28 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
@@ -166,14 +166,16 @@ const makeSeafowlQueryHref = (sqlQuery: string) => {
})}`;
};
-export const SplitgraphEmbeddedQuery = ({
- importedRepository,
- makeQuery,
-}: {
+export interface EmbeddedQueryProps {
importedRepository: SplitgraphRepository;
makeQuery: (repo: SplitgraphRepository) => string;
tableName: string;
-}) => {
+}
+
+export const SplitgraphEmbeddedQuery = ({
+ importedRepository,
+ makeQuery,
+}: EmbeddedQueryProps) => {
return (
<iframe
src={makeSplitgraphEmbeddableQueryHref(makeQuery(importedRepository))}
@@ -188,11 +190,7 @@ export const SplitgraphEmbeddedQuery = ({
export const SeafowlEmbeddedQuery = ({
importedRepository,
makeQuery,
-}: {
- importedRepository: SplitgraphRepository;
- makeQuery: (repo: SplitgraphRepository) => string;
- tableName: string;
-}) => {
+}: EmbeddedQueryProps) => {
return (
<iframe
src={makeSeafowlEmbeddableQueryHref(makeQuery(importedRepository))}
From cc8cde155038825a5b14b7bbde431ac176d5d5ae Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Mon, 19 Jun 2023 23:51:51 +0100
Subject: [PATCH 22/36] Allow toggling between Splitgraph/Seafowl embeds (once
export has completed)
---
.../ImportExportStepper/ExportPanel.tsx | 43 +++++++++++++++++--
1 file changed, 40 insertions(+), 3 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index e89de0d..13dae13 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -1,6 +1,6 @@
// components/ImportExportStepper/ExportPanel.tsx
-import { ComponentProps, Fragment } from "react";
+import { ComponentProps, Fragment, useState } from "react";
import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
@@ -330,14 +330,51 @@ const ExportEmbedPreviewTableOrQuery = <
? exportInput.table
: `${exportInput.destinationSchema}.${exportInput.destinationTable}`;
+ const [selectedTab, setSelectedTab] = useState<"splitgraph" | "seafowl">(
+ "splitgraph"
+ );
+
return (
<>
<h4>
<code>{heading}</code>
</h4>
+ <div style={{ display: "flex", flexDirection: "row" }}>
+ <button
+ onClick={() => setSelectedTab("splitgraph")}
+ disabled={selectedTab === "splitgraph"}
+ style={{ marginRight: "1rem" }}
+ >
+ Splitgraph Query
+ </button>
+ <button
+ onClick={() => setSelectedTab("seafowl")}
+ disabled={selectedTab === "seafowl" || !completed}
+ >
+ Seafowl Query
+ </button>
+ </div>
<pre>{JSON.stringify({ completed, loading }, null, 2)}</pre>
- {(unstarted || loading) && <SplitgraphEmbeddedQuery {...embedProps} />}
- {completed && <SeafowlEmbeddedQuery {...embedProps} />}
+ {
+ <div
+ style={{
+ visibility: selectedTab === "splitgraph" ? "visible" : "hidden",
+ display: selectedTab === "seafowl" ? "none" : "block",
+ }}
+ >
+ <SplitgraphEmbeddedQuery {...embedProps} />
+ </div>
+ }
+ {completed && (
+ <div
+ style={{
+ visibility: selectedTab === "seafowl" ? "visible" : "hidden",
+ display: selectedTab === "splitgraph" ? "none" : "block",
+ }}
+ >
+ <SeafowlEmbeddedQuery {...embedProps} />
+ </div>
+ )}
</>
);
};
From 2b51141902b11b6314d5e75cf5a29df37aa0b810 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Tue, 20 Jun 2023 03:13:44 +0100
Subject: [PATCH 23/36] Use `<a>` tag for "Import Your Repository" link, to
completely reset state (force SSR)
---
.../components/Sidebar.tsx | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
index 1ab5bf9..65f7c2c 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx
@@ -83,9 +83,10 @@ export const Sidebar = () => {
return (
<aside className={styles.sidebar}>
<div className={styles.importButtonContainer}>
- <Link href="/" className={styles.importButton}>
+ {/* NOTE: Use <a> (not <Link />) to force SSR to clear state if an import was already in progress */}
+ <a href="/" className={styles.importButton}>
Import Your Repository
- </Link>
+ </a>
</div>
<SqlProvider dataContext={splitgraphDataContext}>
<RepositoriesList />
From f43d0e8d5c4319ecbe4cc1311a888d80c03604c9 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Tue, 20 Jun 2023 03:14:29 +0100
Subject: [PATCH 24/36] Add `formatTimeElapsed` function prop to `LoadingBar`
for custom formatting of time elapsed message
---
.../components/LoadingBar.tsx | 37 +++++++++++++++----
1 file changed, 29 insertions(+), 8 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx
index 29d53f3..70f2777 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/LoadingBar.tsx
@@ -1,21 +1,31 @@
import styles from "./LoadingBar.module.css";
-import { useState, useEffect } from "react";
+import { useState, useEffect, ComponentProps } from "react";
export const LoadingBar = ({
children,
title,
+ formatTimeElapsed,
}: {
children?: React.ReactNode;
title?: React.ReactNode;
+ formatTimeElapsed?: FormatTimeElapsed;
}) => {
return children || title ? (
- <LoadingBarWithText title={title}>{children}</LoadingBarWithText>
+ <LoadingBarWithText formatTimeElapsed={formatTimeElapsed} title={title}>
+ {children}
+ </LoadingBarWithText>
) : (
- <JustLoadingBar inBox={false} />
+ <JustLoadingBar formatTimeElapsed={formatTimeElapsed} inBox={false} />
);
};
-export const JustLoadingBar = ({ inBox }: { inBox: boolean }) => {
+export const JustLoadingBar = ({
+ inBox,
+ formatTimeElapsed,
+}: {
+ inBox: boolean;
+ formatTimeElapsed?: FormatTimeElapsed;
+}) => {
return (
<div
className={[
@@ -26,7 +36,7 @@ export const JustLoadingBar = ({ inBox }: { inBox: boolean }) => {
<div className={styles.loader}>
<div className={styles.loaderBar}></div>
</div>
- <TimeElapsed />
+ <TimeElapsed formatTimeElapsed={formatTimeElapsed} />
</div>
);
};
@@ -34,20 +44,31 @@ export const JustLoadingBar = ({ inBox }: { inBox: boolean }) => {
export const LoadingBarWithText = ({
children,
title,
+ formatTimeElapsed,
}: {
children?: React.ReactNode;
title?: React.ReactNode;
+ formatTimeElapsed?: FormatTimeElapsed;
}) => {
return (
<div className={styles.loadingBox}>
{title ? <h2 className={styles.loadingTitle}>{title}</h2> : null}
- <JustLoadingBar inBox={true} />
+ <JustLoadingBar inBox={true} formatTimeElapsed={formatTimeElapsed} />
{children}
</div>
);
};
-const TimeElapsed = () => {
+type FormatTimeElapsed = (seconds: number) => React.ReactNode;
+
+const defaultFormatTimeElapsed: FormatTimeElapsed = (seconds) =>
+ `Time elapsed: ${seconds} seconds...`;
+
+const TimeElapsed = ({
+ formatTimeElapsed = defaultFormatTimeElapsed,
+}: {
+ formatTimeElapsed?: FormatTimeElapsed;
+}) => {
const [seconds, setSeconds] = useState<number>(0);
useEffect(() => {
@@ -62,7 +83,7 @@ const TimeElapsed = () => {
return (
<div className={styles.timeElapsed}>
- {seconds < 2 ? <> </> : <>Time elapsed: {seconds} seconds...</>}
+ {seconds < 2 ? <> </> : <>{formatTimeElapsed(seconds)}</>}
</div>
);
};
From cf97067330c918caf1bc27b1709285aa7481f290 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Tue, 20 Jun 2023 03:15:02 +0100
Subject: [PATCH 25/36] Productionize the export step: add lots of nice
styling, bells and whistles, etc.
Render each export(able|ed) query/table in a Splitgraph embed, with
a tabbed conainer for switching to a Seafowl embed when it's ready,
i.e. show each table/query individually, inline with its loading state.
---
.../ImportExportStepper/ChartsPanel.tsx | 18 +-
.../EmbeddedQuery.module.css | 45 +++
.../ExportPanel.module.css | 21 ++
.../ImportExportStepper/ExportPanel.tsx | 340 +++++++++++++++---
.../ExportTableLoadingBar.tsx | 4 +
.../ImportExportStepper/ImportPanel.tsx | 13 +-
.../ImportExportStepper/TabButton.module.css | 22 ++
.../ImportedRepoMetadata.tsx | 6 +-
8 files changed, 388 insertions(+), 81 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/EmbeddedQuery.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/TabButton.module.css
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx
index 1d2dcf4..3939286 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ChartsPanel.tsx
@@ -40,14 +40,16 @@ export const ChartsPanel = () => {
</a>
.
</StepDescription>
- <Charts
- importedRepository={{
- githubNamespace,
- githubRepository,
- splitgraphNamespace,
- splitgraphRepository,
- }}
- />
+ {stepStatus === "active" && (
+ <Charts
+ importedRepository={{
+ githubNamespace,
+ githubRepository,
+ splitgraphNamespace,
+ splitgraphRepository,
+ }}
+ />
+ )}
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/EmbeddedQuery.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/EmbeddedQuery.module.css
new file mode 100644
index 0000000..3708609
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/EmbeddedQuery.module.css
@@ -0,0 +1,45 @@
+.embeddedQuery {
+ border-left: 4px solid var(--muted);
+ padding-left: 10px;
+ margin-bottom: 2rem;
+}
+
+.topBar {
+ display: flex;
+ flex-direction: row;
+ justify-content: space-between;
+ align-items: center;
+ margin-bottom: 8px;
+}
+
+.consoleFlavorButtonsAndLoadingBar {
+ display: inline-flex;
+ flex-direction: row;
+}
+
+.embeddedQuery iframe {
+ width: 100%;
+}
+
+.heading {
+ font-size: 1.5rem;
+ font-weight: bold;
+ display: inline-flex;
+ align-items: center;
+ padding-top: 10px;
+ padding-bottom: 10px;
+}
+
+.embedControls {
+ background: inherit;
+}
+
+.openInConsoleLink {
+ display: inline-flex;
+ align-items: center;
+ font-size: small;
+}
+
+.openInConsoleLink svg {
+ margin-right: 4px;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
index 087130c..d99dfe7 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
@@ -22,6 +22,13 @@
text-decoration: none;
font-weight: bold;
border-style: none;
+ margin-bottom: 1rem;
+}
+
+.startExportButtonLoading {
+ /* color: var(--muted); */
+ /* background-color: transparent; */
+ opacity: 0.5;
}
.startExportButton:hover {
@@ -36,3 +43,17 @@
.exportInfo p {
margin-bottom: 1rem;
}
+
+.exportPreviewHeading {
+ margin-bottom: 0;
+}
+
+.exportPreviewDescription {
+ margin-bottom: 1rem;
+}
+
+.exportNote {
+ font-size: small;
+ /* color: red !important; */
+ display: block;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index 13dae13..a790ceb 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -1,13 +1,14 @@
-// components/ImportExportStepper/ExportPanel.tsx
-
-import { ComponentProps, Fragment, useState } from "react";
+import { ButtonHTMLAttributes, useState } from "react";
import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
import { ExportLoadingBars } from "./ExportLoadingBars";
import { splitgraphTablesToExportToSeafowl } from "../../lib/config/github-tables";
-import { makeQueriesToExport } from "../../lib/config/queries-to-export";
+import {
+ genericDemoQuery,
+ makeQueriesToExport,
+} from "../../lib/config/queries-to-export";
import type {
ExportQueryInput,
ExportTableInput,
@@ -18,20 +19,13 @@ import { useMemo, useCallback } from "react";
import { StepTitle } from "./StepTitle";
import { StepDescription } from "./StepDescription";
import {
- type EmbeddedQueryProps,
SeafowlEmbeddedQuery,
- SeafowlStargazersQueryLink,
-} from "../RepositoryAnalytics/ImportedRepoMetadata";
-import {
- GitHubRepoLink,
- SplitgraphStargazersQueryLink,
SplitgraphEmbeddedQuery,
+ makeSplitgraphQueryHref,
+ makeSeafowlQueryHref,
} from "../RepositoryAnalytics/ImportedRepoMetadata";
-import { makeStargazersTableQuery } from "../RepositoryAnalytics/sql-queries";
import type { ExportTable } from "./stepper-states";
-import type { TargetSplitgraphRepo } from "../../types";
-
export const ExportPanel = () => {
const [
{
@@ -156,41 +150,89 @@ export const ExportPanel = () => {
</a>{" "}
instance running at <code>https://demo.seafowl.cloud</code>. Now
we can query it and get cache-optimized responses for rendering
- charts and analytics.
- </p>
- <p>
- <strong>Query Data: </strong>
- <SeafowlStargazersQueryLink
- splitgraphNamespace={splitgraphNamespace}
- splitgraphRepository={splitgraphRepository}
- />
+ charts and analytics.{" "}
</p>
- <SeafowlEmbeddedQuery
- importedRepository={{ splitgraphNamespace, splitgraphRepository }}
- tableName={"stargazers"}
- makeQuery={makeStargazersTableQuery}
- />
</div>
) : (
<div className={styles.exportInfo}>
- Now let's export some tables and pre-made queries from our staging
- area in Splitgraph to our cache-optimized{" "}
+ {["uninitialized", "unstarted", "awaiting_import"].includes(
+ stepperState
+ )
+ ? "Next we'll "
+ : "Now let's "}
+ export some tables and pre-made queries from our staging area in
+ Splitgraph to our cache-optimized{" "}
<a href="https://seafowl.io" target="_blank">
Seafowl
</a>{" "}
- instance running at <code>https://demo.seafowl.cloud</code>.{" "}
+ instance running at <code>https://demo.seafowl.cloud</code>. This
+ demo exports them programatically with{" "}
+ <a target="_blank" href="https://github.com/splitgraph/madatdata">
+ madatdata
+ </a>{" "}
+ calling the Splitgraph API from a Next.js API route, but you can
+ write your own queries and manually export them from the{" "}
+ <a
+ href={makeSplitgraphQueryHref(`--- Splitgraph is a public data platform that allows you to query and share data sets
+--- You can write any query you want here, and run it against data.splitgraph.com
+--- If you're logged in, you can also export the query to a Seafowl instance (defaulting to demo.seafowl.cloud)
+
+${genericDemoQuery}
+`)}
+ target="_blank"
+ >
+ Splitgraph Console
+ </a>{" "}
+ (once you've created an account and logged into Splitgraph).
{stepStatus === "active" && (
- <> Click the button to start the export.</>
+ <>
+ <br />
+ <br /> <b>Click the button to start the export.</b> While it's
+ running, you can use the embedded query editors to play with the
+ imported Splitgraph data, and when it's complete, you can run
+ the same queries in Seafowl.
+ </>
)}
</div>
)}
</StepDescription>
+ {["import_complete", "awaiting_export", "export_complete"].includes(
+ stepperState
+ ) && (
+ <button
+ className={[
+ styles.startExportButton,
+ ...(stepperState === "awaiting_export"
+ ? [styles.startExportButtonLoading]
+ : []),
+ ].join(" ")}
+ onClick={handleStartExport}
+ title={
+ stepperState === "export_complete"
+ ? "Trigger another export job, which will overwrite the data in Seafowl"
+ : stepperState === "awaiting_export"
+ ? "Exporting tables and queries to Seafowl..."
+ : "Trigger an export job from Splitgraph to Seafowl"
+ }
+ >
+ {stepperState === "awaiting_export"
+ ? "Exporting Tables and Queries to Seafowl..."
+ : stepperState === "export_complete"
+ ? "Restart Export of Tables and Queries to Seafowl"
+ : "Start Export of Tables and Queries to Seafowl"}
+ </button>
+ )}
{exportError && <p className={styles.error}>{exportError}</p>}
{["import_complete", "awaiting_export", "export_complete"].includes(
stepperState
) && (
<ExportPreview
- handleStartExport={handleStartExport}
+ stepperState={
+ stepperState as
+ | "import_complete"
+ | "awaiting_export"
+ | "export_complete"
+ }
tablesToExport={tablesToExport}
queriesToExport={queriesToExport}
splitgraphRepository={splitgraphRepository}
@@ -203,13 +245,13 @@ export const ExportPanel = () => {
};
const ExportPreview = ({
- handleStartExport,
+ stepperState,
tablesToExport,
queriesToExport,
splitgraphRepository,
splitgraphNamespace,
}: {
- handleStartExport: () => Promise<() => void>;
+ stepperState: "import_complete" | "awaiting_export" | "export_complete";
tablesToExport: ExportTableInput[];
queriesToExport: ExportQueryInput[];
splitgraphRepository: string;
@@ -217,10 +259,25 @@ const ExportPreview = ({
}) => {
return (
<>
- <button className={styles.startExportButton} onClick={handleStartExport}>
- Start Export of Tables and Queries from Splitgraph to Seafowl
- </button>
- <h3>Tables to Export</h3>
+ {stepperState !== "export_complete" ? (
+ <>
+ <h2 className={styles.exportPreviewHeading}>Tables to Export</h2>
+ <p className={styles.exportPreviewDescription}>
+ These are the tables that we'll export from Splitgraph to Seafowl.
+ You can query them in Splitgraph now, and then when the export is
+ complete, you'll be able to query them in Seafowl too.
+ </p>
+ </>
+ ) : (
+ <>
+ <h2 className={styles.exportPreviewHeading}>Exported Tables</h2>
+ <p className={styles.exportPreviewDescription}>
+ We successfully exported the tables to Seafowl, so now you can query
+ them in Seafowl too.
+ </p>
+ </>
+ )}
+
{tablesToExport
.filter((_) => true)
.map((exportTable) => (
@@ -240,7 +297,31 @@ const ExportPreview = ({
/>
))}
- <h3>Queries to Export</h3>
+ {stepperState !== "export_complete" ? (
+ <>
+ <h2 className={styles.exportPreviewHeading}>Queries to Export</h2>
+ <p className={styles.exportPreviewDescription}>
+ We've prepared a few queries to export from Splitgraph to Seafowl,
+ so that we can use them to render the charts that we want.
+ Splitgraph will execute the query and insert its result into
+ Seafowl. You can query them in Splitgraph now, and then when the
+ export is complete, you'll be able to query them in Seafowl too.
+ </p>
+ </>
+ ) : (
+ <>
+ <h2 className={styles.exportPreviewHeading}>Exported Queries</h2>
+ <p className={styles.exportPreviewDescription}>
+ We successfully exported these queries from Splitgraph to Seafowl,
+ so now you can query them in Seafowl too.{" "}
+ <em className={styles.exportNote}>
+ Note: If some queries failed to export, it's probably because they
+ had empty result sets (e.g. the table of issue reactions)
+ </em>
+ </p>
+ </>
+ )}
+
{queriesToExport
.filter((_) => true)
.map((exportQuery) => (
@@ -264,10 +345,24 @@ const ExportPreview = ({
);
};
+/**
+ * Given a function to match a candidate `ExportTable` to (presumably) an `ExportTableInput`,
+ * determine if the table (which could also be a query - it's keyed by `destinationSchema`
+ * and `destinationTable`) is currently exporting (`loading`) or has exported (`completed`).
+ *
+ * Return `{ loading, completed, unstarted }`, where:
+ *
+ * * `loading` is `true` if there is a match in the `exportedTablesLoading` set,
+ * * `completed` is `true` if there is a match in the `exportedTablesCompleted` set
+ * (or if `stepperState` is `export_complete`),
+ * * `unstarted` is `true` if there is no match in either set.
+ *
+ */
const useFindMatchingExportTable = (
isMatch: (candidateTable: ExportTable) => boolean
) => {
- const [{ exportedTablesLoading, exportedTablesCompleted }] = useStepper();
+ const [{ stepperState, exportedTablesLoading, exportedTablesCompleted }] =
+ useStepper();
const matchingCompletedTable = useMemo(
() => Array.from(exportedTablesCompleted).find(isMatch),
@@ -278,7 +373,11 @@ const useFindMatchingExportTable = (
[exportedTablesLoading, isMatch]
);
- const completed = matchingCompletedTable ?? false;
+ // If the state is export_complete, we might have loaded the page directly
+ // and thus we don't have the sets of exportedTablesCompleted, but we know they exist
+ const exportFullyCompleted = stepperState === "export_complete";
+
+ const completed = matchingCompletedTable ?? (exportFullyCompleted || false);
const loading = matchingLoadingTable ?? false;
const unstarted = !completed && !loading;
@@ -289,6 +388,10 @@ const useFindMatchingExportTable = (
};
};
+const useStepperDebug = () => useStepper()[0].debug;
+
+import EmbeddedQueryStyles from "./EmbeddedQuery.module.css";
+
const ExportEmbedPreviewTableOrQuery = <
ExportInputShape extends ExportQueryInput | ExportTableInput
>({
@@ -312,6 +415,8 @@ const ExportEmbedPreviewTableOrQuery = <
splitgraphRepository: string;
};
}) => {
+ const debug = useStepperDebug();
+
const embedProps = {
importedRepository,
tableName:
@@ -321,7 +426,7 @@ const ExportEmbedPreviewTableOrQuery = <
makeQuery: () => makeQuery({ ...exportInput, ...importedRepository }),
};
- const { unstarted, loading, completed } = useFindMatchingExportTable(
+ const { loading, completed } = useFindMatchingExportTable(
makeMatchInputToExported(exportInput)
);
@@ -334,27 +439,82 @@ const ExportEmbedPreviewTableOrQuery = <
"splitgraph"
);
+ const linkToConsole = useMemo(() => {
+ switch (selectedTab) {
+ case "splitgraph":
+ return {
+ anchor: "Open in Console",
+ href: makeSplitgraphQueryHref(
+ makeQuery({ ...exportInput, ...importedRepository })
+ ),
+ };
+
+ case "seafowl":
+ return {
+ anchor: "Open in Console",
+ href: makeSeafowlQueryHref(
+ makeQuery({ ...exportInput, ...importedRepository })
+ ),
+ };
+ }
+ }, [selectedTab]);
+
return (
- <>
- <h4>
+ <div className={EmbeddedQueryStyles.embeddedQuery}>
+ <h4 className={EmbeddedQueryStyles.heading}>
<code>{heading}</code>
</h4>
- <div style={{ display: "flex", flexDirection: "row" }}>
- <button
- onClick={() => setSelectedTab("splitgraph")}
- disabled={selectedTab === "splitgraph"}
- style={{ marginRight: "1rem" }}
- >
- Splitgraph Query
- </button>
- <button
- onClick={() => setSelectedTab("seafowl")}
- disabled={selectedTab === "seafowl" || !completed}
- >
- Seafowl Query
- </button>
+ <div className={EmbeddedQueryStyles.topBar}>
+ <div className={EmbeddedQueryStyles.consoleFlavorButtonsAndLoadingBar}>
+ <TabButton
+ onClick={() => setSelectedTab("splitgraph")}
+ active={selectedTab === "splitgraph"}
+ style={{ marginRight: "1rem" }}
+ title={
+ selectedTab === "splitgraph"
+ ? ""
+ : "Query the imported data in Splitgraph"
+ }
+ >
+ data.splitgraph.com
+ </TabButton>
+ <TabButton
+ onClick={() => setSelectedTab("seafowl")}
+ active={selectedTab === "seafowl"}
+ disabled={!completed}
+ style={{ marginRight: "1rem" }}
+ title={
+ selectedTab === "seafowl"
+ ? ""
+ : completed
+ ? "Query the exported data in Seafowl"
+ : "Once you export the data to Seafowl, you can send the same query to Seafowl"
+ }
+ >
+ demo.seafowl.cloud
+ </TabButton>
+ {loading && (
+ <LoadingBar
+ formatTimeElapsed={(seconds) =>
+ `Export to Seafowl: Started ${seconds} seconds ago...`
+ }
+ />
+ )}
+ </div>
+ <div className={EmbeddedQueryStyles.embedControls}>
+ <a
+ href={linkToConsole.href}
+ target="_blank"
+ rel="noopener"
+ className={EmbeddedQueryStyles.openInConsoleLink}
+ >
+ <IconOpenInConsole size={14} />
+ {linkToConsole.anchor}
+ </a>
+ </div>
</div>
- <pre>{JSON.stringify({ completed, loading }, null, 2)}</pre>
+
+ {debug && <pre>{JSON.stringify({ completed, loading }, null, 2)}</pre>}
{
<div
style={{
@@ -375,6 +535,68 @@ const ExportEmbedPreviewTableOrQuery = <
<SeafowlEmbeddedQuery {...embedProps} />
</div>
)}
- </>
+ </div>
);
};
+
+import TabButtonStyle from "./TabButton.module.css";
+import { LoadingBar } from "../LoadingBar";
+
+interface TabButtonProps extends ButtonHTMLAttributes<HTMLButtonElement> {
+ active: boolean;
+ onClick: () => void;
+}
+
+const TabButton = ({
+ active,
+ onClick,
+ disabled: alwaysDisabled,
+ children,
+ ...rest
+}: React.PropsWithChildren<TabButtonProps>) => {
+ const className = [
+ TabButtonStyle["tab-button"],
+ ...(active
+ ? [TabButtonStyle["tab-button-active"]]
+ : [TabButtonStyle["tab-button-inactive"]]),
+ ...(alwaysDisabled ? [TabButtonStyle["tab-button-disabled"]] : []),
+ ].join(" ");
+
+ return (
+ <button
+ onClick={onClick}
+ disabled={active || alwaysDisabled}
+ className={className}
+ {...rest}
+ >
+ {children}
+ </button>
+ );
+};
+
+export const IconOpenInConsole = ({ size }: { size: number | string }) => (
+ <svg
+ width={size}
+ height={size}
+ viewBox="0 0 20 20"
+ fill="none"
+ xmlns="http://www.w3.org/2000/svg"
+ >
+ <path
+ d="M10.625 2.5H3.75C2.36929 2.5 1.25 3.61929 1.25 5V15C1.25 16.3807 2.36929 17.5 3.75 17.5H16.25C17.6307 17.5 18.75 16.3807 18.75 15V10.625"
+ stroke="currentColor"
+ strokeWidth="1.25"
+ />
+ <path
+ d="M18.7501 8.2291L18.7501 2.49946M13.213 2.49961L18.7501 2.49946M18.7501 2.49946L12.5712 8.67797"
+ stroke="currentColor"
+ strokeWidth="1.25"
+ />
+ <path
+ d="M5 6.875L8.125 10L5 13.125"
+ stroke="currentColor"
+ strokeWidth="1.25"
+ />
+ <path d="M10.625 13.125H15" stroke="currentColor" strokeWidth="1.25" />
+ </svg>
+);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
index 945fbd3..db8bb8f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
@@ -80,6 +80,10 @@ export const ExportTableLoadingBar = ({
(t) => t.taskId === taskId
);
+ console.log("----- exportedTablesLoading ----");
+ console.log(exportedTablesLoading);
+ console.log("----- ----");
+
return (
<div className={styles.exportTableLoadingBar}>
<div className={styles.loadingBar}>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
index 44ae8b4..9693305 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx
@@ -8,9 +8,7 @@ import { StepDescription } from "./StepDescription";
import {
GitHubRepoLink,
SplitgraphStargazersQueryLink,
- SplitgraphEmbeddedQuery,
} from "../RepositoryAnalytics/ImportedRepoMetadata";
-import { makeStargazersTableQuery } from "../RepositoryAnalytics/sql-queries";
export const ImportPanel = () => {
const [
@@ -136,11 +134,6 @@ export const ImportPanel = () => {
splitgraphRepository={splitgraphRepository}
/>
</p>
- <SplitgraphEmbeddedQuery
- importedRepository={{ splitgraphNamespace, splitgraphRepository }}
- tableName={"stargazers"}
- makeQuery={makeStargazersTableQuery}
- />
</div>
) : (
<>
@@ -151,9 +144,9 @@ export const ImportPanel = () => {
>
airbyte-github
</a>{" "}
- plugin to import data about this GitHub repository to the Splitgraph
- Data Delivery Network (DDN). Then you'll be able to browse the data
- in the Splitgraph catalog, or query it with{" "}
+ plugin to import data about this GitHub repository into the
+ Splitgraph Data Delivery Network (DDN). Then you'll be able to
+ browse the data in the Splitgraph catalog, or query it with{" "}
<a href="https://www.splitgraph.com/connect/query" target="_blank">
your favorite Postgres Client
</a>{" "}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/TabButton.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/TabButton.module.css
new file mode 100644
index 0000000..56ed32a
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/TabButton.module.css
@@ -0,0 +1,22 @@
+.tab-button {
+ border-width: 3px;
+ border-radius: 10px;
+ border-style: solid;
+ background: transparent;
+ color: var(--text);
+}
+
+.tab-button-active {
+ border-color: var(--primary);
+ color: var(--primary);
+}
+
+.tab-button-inactive:not(.tab-button-disabled):hover {
+ border-color: var(--text);
+ cursor: pointer;
+}
+
+.tab-button-disabled {
+ color: var(--muted);
+ border-color: var(--muted);
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
index bb75a28..bc05767 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
@@ -147,7 +147,7 @@ const SeafowlQueryLink = ({
};
/** Return the URL to Splitgraph Console pointing to Splitgraph DDN */
-const makeSplitgraphQueryHref = (sqlQuery: string) => {
+export const makeSplitgraphQueryHref = (sqlQuery: string) => {
const url = `https://www.splitgraph.com/query?${new URLSearchParams({
sqlQuery: sqlQuery,
flavor: "splitgraph",
@@ -157,7 +157,7 @@ const makeSplitgraphQueryHref = (sqlQuery: string) => {
};
/** Return the URL to Splitgraph Console pointing to Seafowl db where we export tables */
-const makeSeafowlQueryHref = (sqlQuery: string) => {
+export const makeSeafowlQueryHref = (sqlQuery: string) => {
return `https://www.splitgraph.com/query?${new URLSearchParams({
sqlQuery: sqlQuery,
flavor: "seafowl",
@@ -182,7 +182,6 @@ export const SplitgraphEmbeddedQuery = ({
allowFullScreen={false}
style={{ border: "none" }}
height={"400px"}
- width={"80%"}
/>
);
};
@@ -196,7 +195,6 @@ export const SeafowlEmbeddedQuery = ({
src={makeSeafowlEmbeddableQueryHref(makeQuery(importedRepository))}
style={{ border: "none" }}
height={"400px"}
- width={"80%"}
/>
);
};
From ecd1e43bba609ee78b7e45b7c58c075e0e2a43e7 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Tue, 20 Jun 2023 15:55:19 +0100
Subject: [PATCH 26/36] Refactor
---
.../EmbeddedQuery.module.css | 0
.../EmbeddedQuery/EmbeddedQuery.tsx | 195 +++++++++++++
.../TabButton.module.css | 0
.../components/EmbeddedQuery/TabButton.tsx | 35 +++
.../ImportExportStepper/ExportPanel.tsx | 262 +-----------------
.../ImportExportStepper/StepperContext.tsx | 55 +++-
.../pages/api/start-export-to-seafowl.ts | 40 +--
.../types.ts | 35 +++
8 files changed, 325 insertions(+), 297 deletions(-)
rename examples/nextjs-import-airbyte-github-export-seafowl/components/{ImportExportStepper => EmbeddedQuery}/EmbeddedQuery.module.css (100%)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
rename examples/nextjs-import-airbyte-github-export-seafowl/components/{ImportExportStepper => EmbeddedQuery}/TabButton.module.css (100%)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/EmbeddedQuery.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.module.css
similarity index 100%
rename from examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/EmbeddedQuery.module.css
rename to examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.module.css
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
new file mode 100644
index 0000000..f79d798
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
@@ -0,0 +1,195 @@
+import EmbeddedQueryStyles from "./EmbeddedQuery.module.css";
+import type { ExportQueryInput, ExportTableInput } from "../../types";
+import { useState, useMemo } from "react";
+import {
+ makeSplitgraphQueryHref,
+ makeSeafowlQueryHref,
+} from "../RepositoryAnalytics/ImportedRepoMetadata";
+import {
+ SplitgraphEmbeddedQuery,
+ SeafowlEmbeddedQuery,
+} from "../RepositoryAnalytics/ImportedRepoMetadata";
+import {
+ useStepperDebug,
+ useFindMatchingExportTable,
+} from "../ImportExportStepper/StepperContext";
+
+import type { ExportTable } from "../ImportExportStepper/stepper-states";
+
+import { LoadingBar } from "../LoadingBar";
+
+import { TabButton } from "./TabButton";
+
+export const ExportEmbedPreviewTableOrQuery = <
+ ExportInputShape extends ExportQueryInput | ExportTableInput
+>({
+ importedRepository,
+ exportInput,
+ makeQuery,
+ makeMatchInputToExported,
+}: {
+ exportInput: ExportInputShape;
+ makeQuery: (
+ tableOrQueryInput: ExportInputShape & {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+ }
+ ) => string;
+ makeMatchInputToExported: (
+ tableOrQueryInput: ExportInputShape
+ ) => (exported: ExportTable) => boolean;
+ importedRepository: {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+ };
+}) => {
+ const debug = useStepperDebug();
+
+ const embedProps = {
+ importedRepository,
+ tableName:
+ "destinationTable" in exportInput
+ ? exportInput.destinationTable
+ : exportInput.table,
+ makeQuery: () => makeQuery({ ...exportInput, ...importedRepository }),
+ };
+
+ const { loading, completed } = useFindMatchingExportTable(
+ makeMatchInputToExported(exportInput)
+ );
+
+ const heading =
+ "table" in exportInput
+ ? exportInput.table
+ : `${exportInput.destinationSchema}.${exportInput.destinationTable}`;
+
+ const [selectedTab, setSelectedTab] = useState<"splitgraph" | "seafowl">(
+ "splitgraph"
+ );
+
+ const linkToConsole = useMemo(() => {
+ switch (selectedTab) {
+ case "splitgraph":
+ return {
+ anchor: "Open in Console",
+ href: makeSplitgraphQueryHref(
+ makeQuery({ ...exportInput, ...importedRepository })
+ ),
+ };
+
+ case "seafowl":
+ return {
+ anchor: "Open in Console",
+ href: makeSeafowlQueryHref(
+ makeQuery({ ...exportInput, ...importedRepository })
+ ),
+ };
+ }
+ }, [selectedTab]);
+
+ return (
+ <div className={EmbeddedQueryStyles.embeddedQuery}>
+ <h4 className={EmbeddedQueryStyles.heading}>
+ <code>{heading}</code>
+ </h4>
+ <div className={EmbeddedQueryStyles.topBar}>
+ <div className={EmbeddedQueryStyles.consoleFlavorButtonsAndLoadingBar}>
+ <TabButton
+ onClick={() => setSelectedTab("splitgraph")}
+ active={selectedTab === "splitgraph"}
+ style={{ marginRight: "1rem" }}
+ title={
+ selectedTab === "splitgraph"
+ ? ""
+ : "Query the imported data in Splitgraph"
+ }
+ >
+ data.splitgraph.com
+ </TabButton>
+ <TabButton
+ onClick={() => setSelectedTab("seafowl")}
+ active={selectedTab === "seafowl"}
+ disabled={!completed}
+ style={{ marginRight: "1rem" }}
+ title={
+ selectedTab === "seafowl"
+ ? ""
+ : completed
+ ? "Query the exported data in Seafowl"
+ : "Once you export the data to Seafowl, you can send the same query to Seafowl"
+ }
+ >
+ demo.seafowl.cloud
+ </TabButton>
+ {loading && (
+ <LoadingBar
+ formatTimeElapsed={(seconds) =>
+ `Export to Seafowl: Started ${seconds} seconds ago...`
+ }
+ />
+ )}
+ </div>
+ <div className={EmbeddedQueryStyles.embedControls}>
+ <a
+ href={linkToConsole.href}
+ target="_blank"
+ rel="noopener"
+ className={EmbeddedQueryStyles.openInConsoleLink}
+ >
+ <IconOpenInConsole size={14} />
+ {linkToConsole.anchor}
+ </a>
+ </div>
+ </div>
+
+ {debug && <pre>{JSON.stringify({ completed, loading }, null, 2)}</pre>}
+ {
+ <div
+ style={{
+ visibility: selectedTab === "splitgraph" ? "visible" : "hidden",
+ display: selectedTab === "seafowl" ? "none" : "block",
+ }}
+ >
+ <SplitgraphEmbeddedQuery {...embedProps} />
+ </div>
+ }
+ {completed && (
+ <div
+ style={{
+ visibility: selectedTab === "seafowl" ? "visible" : "hidden",
+ display: selectedTab === "splitgraph" ? "none" : "block",
+ }}
+ >
+ <SeafowlEmbeddedQuery {...embedProps} />
+ </div>
+ )}
+ </div>
+ );
+};
+
+export const IconOpenInConsole = ({ size }: { size: number | string }) => (
+ <svg
+ width={size}
+ height={size}
+ viewBox="0 0 20 20"
+ fill="none"
+ xmlns="http://www.w3.org/2000/svg"
+ >
+ <path
+ d="M10.625 2.5H3.75C2.36929 2.5 1.25 3.61929 1.25 5V15C1.25 16.3807 2.36929 17.5 3.75 17.5H16.25C17.6307 17.5 18.75 16.3807 18.75 15V10.625"
+ stroke="currentColor"
+ strokeWidth="1.25"
+ />
+ <path
+ d="M18.7501 8.2291L18.7501 2.49946M13.213 2.49961L18.7501 2.49946M18.7501 2.49946L12.5712 8.67797"
+ stroke="currentColor"
+ strokeWidth="1.25"
+ />
+ <path
+ d="M5 6.875L8.125 10L5 13.125"
+ stroke="currentColor"
+ strokeWidth="1.25"
+ />
+ <path d="M10.625 13.125H15" stroke="currentColor" strokeWidth="1.25" />
+ </svg>
+);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/TabButton.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.module.css
similarity index 100%
rename from examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/TabButton.module.css
rename to examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.module.css
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx
new file mode 100644
index 0000000..7f4c9e9
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx
@@ -0,0 +1,35 @@
+import type { ButtonHTMLAttributes } from "react";
+
+import TabButtonStyle from "./TabButton.module.css";
+
+interface TabButtonProps extends ButtonHTMLAttributes<HTMLButtonElement> {
+ active: boolean;
+ onClick: () => void;
+}
+
+export const TabButton = ({
+ active,
+ onClick,
+ disabled: alwaysDisabled,
+ children,
+ ...rest
+}: React.PropsWithChildren<TabButtonProps>) => {
+ const className = [
+ TabButtonStyle["tab-button"],
+ ...(active
+ ? [TabButtonStyle["tab-button-active"]]
+ : [TabButtonStyle["tab-button-inactive"]]),
+ ...(alwaysDisabled ? [TabButtonStyle["tab-button-disabled"]] : []),
+ ].join(" ");
+
+ return (
+ <button
+ onClick={onClick}
+ disabled={active || alwaysDisabled}
+ className={className}
+ {...rest}
+ >
+ {children}
+ </button>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index a790ceb..d1ae5b4 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -1,5 +1,3 @@
-import { ButtonHTMLAttributes, useState } from "react";
-
import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
import { ExportLoadingBars } from "./ExportLoadingBars";
@@ -14,7 +12,7 @@ import type {
ExportTableInput,
StartExportToSeafowlRequestShape,
StartExportToSeafowlResponseData,
-} from "../../pages/api/start-export-to-seafowl";
+} from "../../types";
import { useMemo, useCallback } from "react";
import { StepTitle } from "./StepTitle";
import { StepDescription } from "./StepDescription";
@@ -26,6 +24,8 @@ import {
} from "../RepositoryAnalytics/ImportedRepoMetadata";
import type { ExportTable } from "./stepper-states";
+import { ExportEmbedPreviewTableOrQuery } from "../EmbeddedQuery/EmbeddedQuery";
+
export const ExportPanel = () => {
const [
{
@@ -344,259 +344,3 @@ const ExportPreview = ({
</>
);
};
-
-/**
- * Given a function to match a candidate `ExportTable` to (presumably) an `ExportTableInput`,
- * determine if the table (which could also be a query - it's keyed by `destinationSchema`
- * and `destinationTable`) is currently exporting (`loading`) or has exported (`completed`).
- *
- * Return `{ loading, completed, unstarted }`, where:
- *
- * * `loading` is `true` if there is a match in the `exportedTablesLoading` set,
- * * `completed` is `true` if there is a match in the `exportedTablesCompleted` set
- * (or if `stepperState` is `export_complete`),
- * * `unstarted` is `true` if there is no match in either set.
- *
- */
-const useFindMatchingExportTable = (
- isMatch: (candidateTable: ExportTable) => boolean
-) => {
- const [{ stepperState, exportedTablesLoading, exportedTablesCompleted }] =
- useStepper();
-
- const matchingCompletedTable = useMemo(
- () => Array.from(exportedTablesCompleted).find(isMatch),
- [exportedTablesCompleted, isMatch]
- );
- const matchingLoadingTable = useMemo(
- () => Array.from(exportedTablesLoading).find(isMatch),
- [exportedTablesLoading, isMatch]
- );
-
- // If the state is export_complete, we might have loaded the page directly
- // and thus we don't have the sets of exportedTablesCompleted, but we know they exist
- const exportFullyCompleted = stepperState === "export_complete";
-
- const completed = matchingCompletedTable ?? (exportFullyCompleted || false);
- const loading = matchingLoadingTable ?? false;
- const unstarted = !completed && !loading;
-
- return {
- completed,
- loading,
- unstarted,
- };
-};
-
-const useStepperDebug = () => useStepper()[0].debug;
-
-import EmbeddedQueryStyles from "./EmbeddedQuery.module.css";
-
-const ExportEmbedPreviewTableOrQuery = <
- ExportInputShape extends ExportQueryInput | ExportTableInput
->({
- importedRepository,
- exportInput,
- makeQuery,
- makeMatchInputToExported,
-}: {
- exportInput: ExportInputShape;
- makeQuery: (
- tableOrQueryInput: ExportInputShape & {
- splitgraphNamespace: string;
- splitgraphRepository: string;
- }
- ) => string;
- makeMatchInputToExported: (
- tableOrQueryInput: ExportInputShape
- ) => (exported: ExportTable) => boolean;
- importedRepository: {
- splitgraphNamespace: string;
- splitgraphRepository: string;
- };
-}) => {
- const debug = useStepperDebug();
-
- const embedProps = {
- importedRepository,
- tableName:
- "destinationTable" in exportInput
- ? exportInput.destinationTable
- : exportInput.table,
- makeQuery: () => makeQuery({ ...exportInput, ...importedRepository }),
- };
-
- const { loading, completed } = useFindMatchingExportTable(
- makeMatchInputToExported(exportInput)
- );
-
- const heading =
- "table" in exportInput
- ? exportInput.table
- : `${exportInput.destinationSchema}.${exportInput.destinationTable}`;
-
- const [selectedTab, setSelectedTab] = useState<"splitgraph" | "seafowl">(
- "splitgraph"
- );
-
- const linkToConsole = useMemo(() => {
- switch (selectedTab) {
- case "splitgraph":
- return {
- anchor: "Open in Console",
- href: makeSplitgraphQueryHref(
- makeQuery({ ...exportInput, ...importedRepository })
- ),
- };
-
- case "seafowl":
- return {
- anchor: "Open in Console",
- href: makeSeafowlQueryHref(
- makeQuery({ ...exportInput, ...importedRepository })
- ),
- };
- }
- }, [selectedTab]);
-
- return (
- <div className={EmbeddedQueryStyles.embeddedQuery}>
- <h4 className={EmbeddedQueryStyles.heading}>
- <code>{heading}</code>
- </h4>
- <div className={EmbeddedQueryStyles.topBar}>
- <div className={EmbeddedQueryStyles.consoleFlavorButtonsAndLoadingBar}>
- <TabButton
- onClick={() => setSelectedTab("splitgraph")}
- active={selectedTab === "splitgraph"}
- style={{ marginRight: "1rem" }}
- title={
- selectedTab === "splitgraph"
- ? ""
- : "Query the imported data in Splitgraph"
- }
- >
- data.splitgraph.com
- </TabButton>
- <TabButton
- onClick={() => setSelectedTab("seafowl")}
- active={selectedTab === "seafowl"}
- disabled={!completed}
- style={{ marginRight: "1rem" }}
- title={
- selectedTab === "seafowl"
- ? ""
- : completed
- ? "Query the exported data in Seafowl"
- : "Once you export the data to Seafowl, you can send the same query to Seafowl"
- }
- >
- demo.seafowl.cloud
- </TabButton>
- {loading && (
- <LoadingBar
- formatTimeElapsed={(seconds) =>
- `Export to Seafowl: Started ${seconds} seconds ago...`
- }
- />
- )}
- </div>
- <div className={EmbeddedQueryStyles.embedControls}>
- <a
- href={linkToConsole.href}
- target="_blank"
- rel="noopener"
- className={EmbeddedQueryStyles.openInConsoleLink}
- >
- <IconOpenInConsole size={14} />
- {linkToConsole.anchor}
- </a>
- </div>
- </div>
-
- {debug && <pre>{JSON.stringify({ completed, loading }, null, 2)}</pre>}
- {
- <div
- style={{
- visibility: selectedTab === "splitgraph" ? "visible" : "hidden",
- display: selectedTab === "seafowl" ? "none" : "block",
- }}
- >
- <SplitgraphEmbeddedQuery {...embedProps} />
- </div>
- }
- {completed && (
- <div
- style={{
- visibility: selectedTab === "seafowl" ? "visible" : "hidden",
- display: selectedTab === "splitgraph" ? "none" : "block",
- }}
- >
- <SeafowlEmbeddedQuery {...embedProps} />
- </div>
- )}
- </div>
- );
-};
-
-import TabButtonStyle from "./TabButton.module.css";
-import { LoadingBar } from "../LoadingBar";
-
-interface TabButtonProps extends ButtonHTMLAttributes<HTMLButtonElement> {
- active: boolean;
- onClick: () => void;
-}
-
-const TabButton = ({
- active,
- onClick,
- disabled: alwaysDisabled,
- children,
- ...rest
-}: React.PropsWithChildren<TabButtonProps>) => {
- const className = [
- TabButtonStyle["tab-button"],
- ...(active
- ? [TabButtonStyle["tab-button-active"]]
- : [TabButtonStyle["tab-button-inactive"]]),
- ...(alwaysDisabled ? [TabButtonStyle["tab-button-disabled"]] : []),
- ].join(" ");
-
- return (
- <button
- onClick={onClick}
- disabled={active || alwaysDisabled}
- className={className}
- {...rest}
- >
- {children}
- </button>
- );
-};
-
-export const IconOpenInConsole = ({ size }: { size: number | string }) => (
- <svg
- width={size}
- height={size}
- viewBox="0 0 20 20"
- fill="none"
- xmlns="http://www.w3.org/2000/svg"
- >
- <path
- d="M10.625 2.5H3.75C2.36929 2.5 1.25 3.61929 1.25 5V15C1.25 16.3807 2.36929 17.5 3.75 17.5H16.25C17.6307 17.5 18.75 16.3807 18.75 15V10.625"
- stroke="currentColor"
- strokeWidth="1.25"
- />
- <path
- d="M18.7501 8.2291L18.7501 2.49946M13.213 2.49961L18.7501 2.49946M18.7501 2.49946L12.5712 8.67797"
- stroke="currentColor"
- strokeWidth="1.25"
- />
- <path
- d="M5 6.875L8.125 10L5 13.125"
- stroke="currentColor"
- strokeWidth="1.25"
- />
- <path d="M10.625 13.125H15" stroke="currentColor" strokeWidth="1.25" />
- </svg>
-);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
index f80ed7f..810414e 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
@@ -1,18 +1,20 @@
-// StepperContext.tsx
-import React, { useContext, ReactNode } from "react";
+import { createContext, useContext, useMemo } from "react";
import {
StepperState,
StepperAction,
useStepperReducer,
} from "./stepper-states";
+import type { ExportTable } from "./stepper-states";
// Define the context
-const StepperContext = React.createContext<
+const StepperContext = createContext<
[StepperState, React.Dispatch<StepperAction>] | undefined
>(undefined);
-export const StepperContextProvider: React.FC<{ children: ReactNode }> = ({
+export const StepperContextProvider = ({
children,
+}: {
+ children: React.ReactNode;
}) => {
const [state, dispatch] = useStepperReducer();
@@ -31,3 +33,48 @@ export const useStepper = () => {
}
return context;
};
+
+export const useStepperDebug = () => useStepper()[0].debug;
+
+/**
+ * Given a function to match a candidate `ExportTable` to (presumably) an `ExportTableInput`,
+ * determine if the table (which could also be a query - it's keyed by `destinationSchema`
+ * and `destinationTable`) is currently exporting (`loading`) or has exported (`completed`).
+ *
+ * Return `{ loading, completed, unstarted }`, where:
+ *
+ * * `loading` is `true` if there is a match in the `exportedTablesLoading` set,
+ * * `completed` is `true` if there is a match in the `exportedTablesCompleted` set
+ * (or if `stepperState` is `export_complete`),
+ * * `unstarted` is `true` if there is no match in either set.
+ *
+ */
+export const useFindMatchingExportTable = (
+ isMatch: (candidateTable: ExportTable) => boolean
+) => {
+ const [{ stepperState, exportedTablesLoading, exportedTablesCompleted }] =
+ useStepper();
+
+ const matchingCompletedTable = useMemo(
+ () => Array.from(exportedTablesCompleted).find(isMatch),
+ [exportedTablesCompleted, isMatch]
+ );
+ const matchingLoadingTable = useMemo(
+ () => Array.from(exportedTablesLoading).find(isMatch),
+ [exportedTablesLoading, isMatch]
+ );
+
+ // If the state is export_complete, we might have loaded the page directly
+ // and thus we don't have the sets of exportedTablesCompleted, but we know they exist
+ const exportFullyCompleted = stepperState === "export_complete";
+
+ const completed = matchingCompletedTable ?? (exportFullyCompleted || false);
+ const loading = matchingLoadingTable ?? false;
+ const unstarted = !completed && !loading;
+
+ return {
+ completed,
+ loading,
+ unstarted,
+ };
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
index 9b9f8cf..5b856d3 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts
@@ -1,40 +1,12 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db";
-export type StartExportToSeafowlRequestShape =
- | {
- tables: ExportTableInput[];
- }
- | { queries: ExportQueryInput[] }
- | { tables: ExportTableInput[]; queries: ExportQueryInput[] };
-
-export type StartExportToSeafowlResponseData =
- | {
- tables: {
- destinationTable: string;
- destinationSchema: string;
- taskId: string;
- }[];
- queries: {
- sourceQuery: string;
- destinationSchema: string;
- destinationTable: string;
- taskId: string;
- }[];
- }
- | { error: string };
-
-export type ExportTableInput = {
- namespace: string;
- repository: string;
- table: string;
-};
-
-export type ExportQueryInput = {
- sourceQuery: string;
- destinationSchema: string;
- destinationTable: string;
-};
+import type {
+ ExportTableInput,
+ ExportQueryInput,
+ StartExportToSeafowlRequestShape,
+ StartExportToSeafowlResponseData,
+} from "../../types";
/**
* To manually send a request, example:
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
index 8183d8d..f377047 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
@@ -9,3 +9,38 @@ export interface TargetSplitgraphRepo {
splitgraphNamespace?: string;
splitgraphRepository: string;
}
+
+export type ExportTableInput = {
+ namespace: string;
+ repository: string;
+ table: string;
+};
+
+export type ExportQueryInput = {
+ sourceQuery: string;
+ destinationSchema: string;
+ destinationTable: string;
+};
+
+export type StartExportToSeafowlRequestShape =
+ | {
+ tables: ExportTableInput[];
+ }
+ | { queries: ExportQueryInput[] }
+ | { tables: ExportTableInput[]; queries: ExportQueryInput[] };
+
+export type StartExportToSeafowlResponseData =
+ | {
+ tables: {
+ destinationTable: string;
+ destinationSchema: string;
+ taskId: string;
+ }[];
+ queries: {
+ sourceQuery: string;
+ destinationSchema: string;
+ destinationTable: string;
+ taskId: string;
+ }[];
+ }
+ | { error: string };
From 418ecfd48ba469ef907b112dd778bec901deb94a Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 21 Jun 2023 22:16:08 +0100
Subject: [PATCH 27/36] Only poll unique export taskIds, and do the polling
from one hook instead of each component
Previously, the API always returned a unique taskId for each table being exported,
but a recent change optimized it to return one taskId for the set of tables being
exported, but still one taskId for each query being exported. Also previously, this
demo code rendered a loading component for each table, and each component had its
own hook for polling the taskId of that table. But now that multiple tables can share
a taskId, it doesn't make sense for each component to poll for its own taskId.
Now, we track the set of taskIds separately from the set of completed tables, and
we only poll for unique taskIds, which we do in a hook instead of in each component. And
each table preview checks the set of completed tables to know whether it's been completed.
---
.../EmbeddedQuery/EmbeddedQuery.tsx | 6 +-
.../ImportExportStepper/DebugPanel.tsx | 6 +-
.../ExportLoadingBars.module.css | 3 -
.../ImportExportStepper/ExportLoadingBars.tsx | 23 ---
.../ImportExportStepper/ExportPanel.tsx | 21 +--
.../ExportTableLoadingBar.module.css | 17 ---
.../ExportTableLoadingBar.tsx | 97 -------------
.../ImportExportStepper/StepperContext.tsx | 46 +-----
.../ImportExportStepper/export-hooks.tsx | 132 ++++++++++++++++++
.../ImportExportStepper/stepper-states.ts | 63 +++++++--
10 files changed, 197 insertions(+), 217 deletions(-)
delete mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css
delete mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
delete mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css
delete mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
index f79d798..ccc9053 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
@@ -9,10 +9,8 @@ import {
SplitgraphEmbeddedQuery,
SeafowlEmbeddedQuery,
} from "../RepositoryAnalytics/ImportedRepoMetadata";
-import {
- useStepperDebug,
- useFindMatchingExportTable,
-} from "../ImportExportStepper/StepperContext";
+import { useStepperDebug } from "../ImportExportStepper/StepperContext";
+import { useFindMatchingExportTable } from "../ImportExportStepper/export-hooks";
import type { ExportTable } from "../ImportExportStepper/stepper-states";
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
index d7306b4..bfb7336 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
@@ -6,7 +6,11 @@ export const DebugPanel = () => {
return (
<div>
<pre style={{ minWidth: "80%", minHeight: "300px" }}>
- {JSON.stringify(state, null, 2)}
+ {JSON.stringify(
+ state,
+ (_key, value) => (value instanceof Set ? Array.from(value) : value),
+ 2
+ )}
</pre>
</div>
);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css
deleted file mode 100644
index 04c119f..0000000
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css
+++ /dev/null
@@ -1,3 +0,0 @@
-.exportLoadingBars {
- background-color: inherit;
-}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
deleted file mode 100644
index 076cded..0000000
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx
+++ /dev/null
@@ -1,23 +0,0 @@
-import { useStepper } from "./StepperContext";
-import { ExportTableLoadingBar } from "./ExportTableLoadingBar";
-import styles from "./ExportLoadingBars.module.css";
-
-export const ExportLoadingBars = () => {
- const [{ exportedTablesLoading }] = useStepper();
-
- return (
- <div className={styles.exportLoadingBars}>
- {Array.from(exportedTablesLoading).map(
- ({ destinationSchema, destinationTable, sourceQuery, taskId }) => (
- <ExportTableLoadingBar
- key={taskId}
- destinationSchema={destinationSchema}
- destinationTable={destinationTable}
- sourceQuery={sourceQuery}
- taskId={taskId}
- />
- )
- )}
- </div>
- );
-};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index d1ae5b4..cdeef09 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -1,6 +1,5 @@
import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
-import { ExportLoadingBars } from "./ExportLoadingBars";
import { splitgraphTablesToExportToSeafowl } from "../../lib/config/github-tables";
import {
@@ -16,29 +15,20 @@ import type {
import { useMemo, useCallback } from "react";
import { StepTitle } from "./StepTitle";
import { StepDescription } from "./StepDescription";
-import {
- SeafowlEmbeddedQuery,
- SplitgraphEmbeddedQuery,
- makeSplitgraphQueryHref,
- makeSeafowlQueryHref,
-} from "../RepositoryAnalytics/ImportedRepoMetadata";
+import { makeSplitgraphQueryHref } from "../RepositoryAnalytics/ImportedRepoMetadata";
import type { ExportTable } from "./stepper-states";
import { ExportEmbedPreviewTableOrQuery } from "../EmbeddedQuery/EmbeddedQuery";
+import { usePollExportTasks } from "./export-hooks";
export const ExportPanel = () => {
const [
- {
- stepperState,
- exportError,
- splitgraphRepository,
- splitgraphNamespace,
- exportedTablesCompleted,
- repository: githubRepositoryFromStepper,
- },
+ { stepperState, exportError, splitgraphRepository, splitgraphNamespace },
dispatch,
] = useStepper();
+ usePollExportTasks();
+
const queriesToExport = useMemo<ExportQueryInput[]>(
() =>
makeQueriesToExport({
@@ -239,7 +229,6 @@ ${genericDemoQuery}
splitgraphNamespace={splitgraphNamespace}
/>
)}
- {stepperState === "awaiting_export" && <ExportLoadingBars />}
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css
deleted file mode 100644
index 9f75de7..0000000
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css
+++ /dev/null
@@ -1,17 +0,0 @@
-/* components/ImportExportStepper/ExportTableLoadingBar.module.css */
-
-.exportTableLoadingBar {
- background-color: inherit;
-}
-
-.loadingBar {
- background-color: inherit;
-}
-
-.completedBar {
- background-color: inherit;
-}
-
-.tableName {
- background-color: inherit;
-}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
deleted file mode 100644
index db8bb8f..0000000
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx
+++ /dev/null
@@ -1,97 +0,0 @@
-import { useEffect } from "react";
-import { useStepper } from "./StepperContext";
-import styles from "./ExportTableLoadingBar.module.css";
-
-interface ExportTableLoadingBarProps {
- destinationTable: string;
- destinationSchema: string;
- sourceQuery?: string;
- taskId: string;
-}
-
-export const ExportTableLoadingBar = ({
- destinationTable,
- destinationSchema,
- sourceQuery,
- taskId,
-}: React.PropsWithoutRef<ExportTableLoadingBarProps>) => {
- const [{ stepperState, exportedTablesLoading }, dispatch] = useStepper();
-
- useEffect(() => {
- if (!taskId || !destinationTable) {
- console.log(
- "Don't check export until we have taskId and destinationTable"
- );
- console.table({
- taskId,
- destinationTable,
- });
- return;
- }
-
- if (stepperState !== "awaiting_export") {
- console.log("Done waiting for export");
- return;
- }
-
- const pollExportTask = async () => {
- try {
- const response = await fetch("/api/await-export-to-seafowl-task", {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- },
- body: JSON.stringify({
- taskId,
- }),
- });
- const data = await response.json();
-
- if (data.completed) {
- dispatch({
- type: "export_table_task_complete",
- completedTable: {
- destinationTable,
- taskId,
- destinationSchema,
- sourceQuery,
- },
- });
- } else if (data.error) {
- if (!data.completed) {
- console.log("WARN: Failed status, not completed:", data.error);
- } else {
- throw new Error(data.error);
- }
- }
- } catch (error) {
- dispatch({
- type: "export_error",
- error: `Error exporting ${destinationTable}: ${error.message}`,
- });
- }
- };
-
- const interval = setInterval(pollExportTask, 3000);
- return () => clearInterval(interval);
- }, [stepperState, destinationTable, taskId, dispatch]);
-
- const isLoading = !!Array.from(exportedTablesLoading).find(
- (t) => t.taskId === taskId
- );
-
- console.log("----- exportedTablesLoading ----");
- console.log(exportedTablesLoading);
- console.log("----- ----");
-
- return (
- <div className={styles.exportTableLoadingBar}>
- <div className={styles.loadingBar}>
- {isLoading
- ? `Loading ${destinationTable}...`
- : `Successfully exported ${destinationTable}`}
- </div>
- <div className={styles.destinationTable}>{destinationTable}</div>
- </div>
- );
-};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
index 810414e..9eb3b0c 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx
@@ -1,10 +1,9 @@
-import { createContext, useContext, useMemo } from "react";
+import { createContext, useContext } from "react";
import {
StepperState,
StepperAction,
useStepperReducer,
} from "./stepper-states";
-import type { ExportTable } from "./stepper-states";
// Define the context
const StepperContext = createContext<
@@ -35,46 +34,3 @@ export const useStepper = () => {
};
export const useStepperDebug = () => useStepper()[0].debug;
-
-/**
- * Given a function to match a candidate `ExportTable` to (presumably) an `ExportTableInput`,
- * determine if the table (which could also be a query - it's keyed by `destinationSchema`
- * and `destinationTable`) is currently exporting (`loading`) or has exported (`completed`).
- *
- * Return `{ loading, completed, unstarted }`, where:
- *
- * * `loading` is `true` if there is a match in the `exportedTablesLoading` set,
- * * `completed` is `true` if there is a match in the `exportedTablesCompleted` set
- * (or if `stepperState` is `export_complete`),
- * * `unstarted` is `true` if there is no match in either set.
- *
- */
-export const useFindMatchingExportTable = (
- isMatch: (candidateTable: ExportTable) => boolean
-) => {
- const [{ stepperState, exportedTablesLoading, exportedTablesCompleted }] =
- useStepper();
-
- const matchingCompletedTable = useMemo(
- () => Array.from(exportedTablesCompleted).find(isMatch),
- [exportedTablesCompleted, isMatch]
- );
- const matchingLoadingTable = useMemo(
- () => Array.from(exportedTablesLoading).find(isMatch),
- [exportedTablesLoading, isMatch]
- );
-
- // If the state is export_complete, we might have loaded the page directly
- // and thus we don't have the sets of exportedTablesCompleted, but we know they exist
- const exportFullyCompleted = stepperState === "export_complete";
-
- const completed = matchingCompletedTable ?? (exportFullyCompleted || false);
- const loading = matchingLoadingTable ?? false;
- const unstarted = !completed && !loading;
-
- return {
- completed,
- loading,
- unstarted,
- };
-};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
new file mode 100644
index 0000000..c0185d3
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
@@ -0,0 +1,132 @@
+import { useEffect, useMemo } from "react";
+import type { ExportTable } from "./stepper-states";
+import { useStepper } from "./StepperContext";
+
+/**
+ * Given a function to match a candidate `ExportTable` to (presumably) an `ExportTableInput`,
+ * determine if the table (which could also be a query - it's keyed by `destinationSchema`
+ * and `destinationTable`) is currently exporting (`loading`) or has exported (`completed`).
+ *
+ * Return `{ loading, completed, unstarted }`, where:
+ *
+ * * `loading` is `true` if there is a match in the `exportedTablesLoading` set,
+ * * `completed` is `true` if there is a match in the `exportedTablesCompleted` set
+ * (or if `stepperState` is `export_complete`),
+ * * `unstarted` is `true` if there is no match in either set.
+ *
+ */
+export const useFindMatchingExportTable = (
+ isMatch: (candidateTable: ExportTable) => boolean
+) => {
+ const [{ stepperState, exportedTablesLoading, exportedTablesCompleted }] =
+ useStepper();
+
+ const matchingCompletedTable = useMemo(
+ () => Array.from(exportedTablesCompleted).find(isMatch),
+ [exportedTablesCompleted, isMatch]
+ );
+ const matchingLoadingTable = useMemo(
+ () => Array.from(exportedTablesLoading).find(isMatch),
+ [exportedTablesLoading, isMatch]
+ );
+
+ // If the state is export_complete, we might have loaded the page directly
+ // and thus we don't have the sets of exportedTablesCompleted, but we know they exist
+ const exportFullyCompleted = stepperState === "export_complete";
+
+ const completed = matchingCompletedTable ?? (exportFullyCompleted || false);
+ const loading = matchingLoadingTable ?? false;
+ const unstarted = !completed && !loading;
+
+ return {
+ completed,
+ loading,
+ unstarted,
+ };
+};
+
+export const usePollExportTasks = () => {
+ const [{ stepperState, loadingExportTasks }, dispatch] = useStepper();
+
+ useEffect(() => {
+ if (stepperState !== "awaiting_export") {
+ return;
+ }
+
+ const taskIds = Array.from(loadingExportTasks).map(({ taskId }) => taskId);
+
+ if (taskIds.length === 0) {
+ return;
+ }
+
+ const abortController = new AbortController();
+
+ const pollEachTaskOnce = () =>
+ Promise.all(
+ taskIds.map((taskId) =>
+ pollExportTaskOnce({
+ taskId,
+ onSuccess: ({ taskId }) =>
+ dispatch({
+ type: "export_task_complete",
+ completedTask: { taskId },
+ }),
+ onError: ({ taskId, error }) =>
+ dispatch({
+ type: "export_error",
+ error: `Error exporting ${taskId}: ${error.message}`,
+ }),
+ abortSignal: abortController.signal,
+ })
+ )
+ );
+
+ const interval = setInterval(pollEachTaskOnce, 3000);
+ return () => {
+ clearInterval(interval);
+ abortController.abort();
+ };
+ }, [loadingExportTasks, stepperState, dispatch]);
+};
+
+const pollExportTaskOnce = async ({
+ taskId,
+ onSuccess,
+ onError,
+ abortSignal,
+}: {
+ taskId: string;
+ onSuccess: ({ taskId }: { taskId: string }) => void;
+ onError: ({ taskId, error }: { taskId: string; error: any }) => void;
+ abortSignal: AbortSignal;
+}) => {
+ try {
+ const response = await fetch("/api/await-export-to-seafowl-task", {
+ signal: abortSignal,
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ taskId,
+ }),
+ });
+ const data = await response.json();
+
+ if (data.completed) {
+ onSuccess({ taskId });
+ } else if (data.error) {
+ if (!data.completed) {
+ console.log("WARN: Failed status, not completed:", data.error);
+ } else {
+ throw new Error(data.error);
+ }
+ }
+ } catch (error) {
+ if (error.name === "AbortError") {
+ return;
+ }
+
+ onError({ taskId, error });
+ }
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index 0030053..837d824 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -10,6 +10,12 @@ export type ExportTable = {
sourceQuery?: string;
};
+// NOTE: Multiple tables can have the same taskId, so we track them separately
+// in order to not need to redundantly poll the API for each table individually
+export type ExportTask = {
+ taskId: string;
+};
+
export type StepperState = {
stepperState:
| "uninitialized"
@@ -27,6 +33,8 @@ export type StepperState = {
exportedTablesCompleted?: Set<ExportTable>;
exportError?: string;
debug?: string | null;
+ loadingExportTasks?: Set<ExportTask>;
+ completedExportTasks?: Set<ExportTask>;
};
export type StepperAction =
@@ -40,6 +48,7 @@ export type StepperAction =
| { type: "import_complete" }
| { type: "start_export"; tables: ExportTable[] }
| { type: "export_table_task_complete"; completedTable: ExportTable }
+ | { type: "export_task_complete"; completedTask: ExportTask }
| { type: "export_complete" }
| { type: "export_error"; error: string }
| { type: "import_error"; error: string }
@@ -54,6 +63,8 @@ const initialState: StepperState = {
importTaskId: null,
exportedTablesLoading: new Set<ExportTable>(),
exportedTablesCompleted: new Set<ExportTable>(),
+ loadingExportTasks: new Set<ExportTask>(),
+ completedExportTasks: new Set<ExportTask>(),
importError: null,
exportError: null,
debug: null,
@@ -216,33 +227,61 @@ const stepperReducer = (
});
}
+ // The API returns a list of tables where multiple can have the same taskId
+ // We want a unique set of taskIds so that we only poll for each taskId once
+ // (but note that we're storing a set of {taskId} objects but need to compare uniqueness on taskId)
+ const loadingExportTasks = new Set<ExportTask>(
+ Array.from(
+ new Set<ExportTask["taskId"]>(
+ Array.from(tables).map(({ taskId }) => taskId)
+ )
+ ).map((taskId) => ({ taskId }))
+ );
+ const completedExportTasks = new Set<ExportTask>();
+
return {
...state,
exportedTablesLoading,
exportedTablesCompleted,
+ loadingExportTasks,
+ completedExportTasks,
stepperState: "awaiting_export",
};
- case "export_table_task_complete":
- const { completedTable } = action;
+ case "export_task_complete":
+ const {
+ completedTask: { taskId: completedTaskId },
+ } = action;
- // We're storing a set of completedTable objects, so we need to find the matching one to remove it
- const loadingTablesAfterRemoval = new Set(state.exportedTablesLoading);
- const loadingTabletoRemove = Array.from(loadingTablesAfterRemoval).find(
- ({ taskId }) => taskId === completedTable.taskId
+ // One taskId could match multiple tables, so find reference to each of them
+ // and then use that reference to delete them from loading set and add them to completed set
+ const completedTables = Array.from(state.exportedTablesLoading).filter(
+ ({ taskId }) => taskId === completedTaskId
);
- loadingTablesAfterRemoval.delete(loadingTabletoRemove);
-
- // Then we can add the matching one to the completed table
+ const loadingTablesAfterRemoval = new Set(state.exportedTablesLoading);
const completedTablesAfterAdded = new Set(state.exportedTablesCompleted);
- completedTablesAfterAdded.add(completedTable);
+ for (const completedTable of completedTables) {
+ loadingTablesAfterRemoval.delete(completedTable);
+ completedTablesAfterAdded.add(completedTable);
+ }
+
+ // There should only be one matching task, so find it and create new updated sets
+ const completedTask = Array.from(state.loadingExportTasks).find(
+ ({ taskId }) => taskId === completedTaskId
+ );
+ const loadingTasksAfterRemoval = new Set(state.loadingExportTasks);
+ const completedTasksAfterAdded = new Set(state.completedExportTasks);
+ loadingTasksAfterRemoval.delete(completedTask);
+ completedTasksAfterAdded.add(completedTask);
return {
...state,
exportedTablesLoading: loadingTablesAfterRemoval,
exportedTablesCompleted: completedTablesAfterAdded,
+ loadingExportTasks: loadingTasksAfterRemoval,
+ completedExportTasks: completedTasksAfterAdded,
stepperState:
- loadingTablesAfterRemoval.size === 0
+ loadingTasksAfterRemoval.size === 0
? "export_complete"
: "awaiting_export",
};
@@ -264,6 +303,8 @@ const stepperReducer = (
case "export_error":
return {
...state,
+ loadingExportTasks: new Set<ExportTask>(),
+ completedExportTasks: new Set<ExportTask>(),
exportedTablesLoading: new Set<ExportTable>(),
exportedTablesCompleted: new Set<ExportTable>(),
stepperState: "import_complete",
From 9835599223c08a60974551d4ef83d253b19c0c51 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Fri, 23 Jun 2023 04:07:28 +0100
Subject: [PATCH 28/36] Add workaround for failed exports of queries: create a
fallback table
For each query to export, optionally provide a fallback `CREATE TABLE`
query which will run if the export job for the query fails. Implement
this by calling an API route `/api/create-fallback-table-after-failed-export`
after an export for a query fails for any reason.
This works around the bug where queries with an empty result fail
to export to Seafowl, see: https://github.com/splitgraph/seafowl/issues/423
---
.../.env.test.local | 10 +
.../ImportExportStepper/DebugPanel.tsx | 10 +-
.../ImportExportStepper/ExportPanel.tsx | 5 +
.../ImportExportStepper/export-hooks.tsx | 130 ++++++++++-
.../ImportExportStepper/stepper-states.ts | 46 +++-
.../env-vars.d.ts | 24 ++
.../lib/backend/seafowl-db.ts | 83 +++++++
.../lib/backend/splitgraph-db.ts | 1 +
.../lib/config/queries-to-export.ts | 205 +++++++++++++-----
...eate-fallback-table-after-failed-export.ts | 85 ++++++++
.../types.ts | 22 ++
11 files changed, 551 insertions(+), 70 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/seafowl-db.ts
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/create-fallback-table-after-failed-export.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
index 2d80ce0..85b47c9 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local
@@ -19,3 +19,13 @@ GITHUB_PAT_SECRET="github_pat_**********************_***************************
# e.g. To intercept requests to Splitgraph API sent from madatdata libraries in API routes
# You can also set this by running: yarn dev-mitm (see package.json)
# MITMPROXY_ADDRESS="http://localhost:7979"
+
+# OPTIONAL: Set Seafowl environment variables to use for creating fallback tables when exports fail
+# NOTE 1: At the moment the instance URL must be https://demo.seafowl.cloud because that's where
+# the Splitgraph export API exports tables to when no instance URL is specified, and we are
+# currently not specifying the instance URL when starting exports, and only use it when creating fallback tables.
+# NOTE 2: The dbname (SEAFOWL_INSTANCE_DATABASE) MUST match NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE
+#
+# SEAFOWL_INSTANCE_URL="https://demo.seafowl.cloud"
+# SEAFOWL_INSTANCE_SECRET="********************************"
+# SEAFOWL_INSTANCE_DATABASE="**********"
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
index bfb7336..25193f1 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx
@@ -8,7 +8,15 @@ export const DebugPanel = () => {
<pre style={{ minWidth: "80%", minHeight: "300px" }}>
{JSON.stringify(
state,
- (_key, value) => (value instanceof Set ? Array.from(value) : value),
+ (_key, value) => {
+ if (value instanceof Set) {
+ return Array.from(value);
+ } else if (value instanceof Map) {
+ return Object.fromEntries(value);
+ } else {
+ return value;
+ }
+ },
2
)}
</pre>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index cdeef09..264609b 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -87,6 +87,11 @@ export const ExportPanel = () => {
destinationTable,
destinationSchema,
sourceQuery,
+ fallbackCreateTableQuery: queriesToExport.find(
+ (q) =>
+ q.destinationSchema === destinationSchema &&
+ q.destinationTable === destinationTable
+ )?.fallbackCreateTableQuery,
})
),
...data["tables"].map(
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
index c0185d3..9e7f7d6 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
@@ -46,7 +46,10 @@ export const useFindMatchingExportTable = (
};
export const usePollExportTasks = () => {
- const [{ stepperState, loadingExportTasks }, dispatch] = useStepper();
+ const [
+ { stepperState, loadingExportTasks, exportedTablesLoading },
+ dispatch,
+ ] = useStepper();
useEffect(() => {
if (stepperState !== "awaiting_export") {
@@ -71,11 +74,50 @@ export const usePollExportTasks = () => {
type: "export_task_complete",
completedTask: { taskId },
}),
- onError: ({ taskId, error }) =>
+ onError: async ({ taskId, error }) => {
+ // If the task failed but we're not going to retry, then check if
+ // there is a fallback query to create the table, and if so,
+ // create it before marking the task as complete.
+ if (!error.retryable) {
+ // NOTE: There is an implicit assumption that `exportedTablesLoading`
+ // and `loadingExportTasks` are updated at the same time, which they
+ // are, by the reducer that handles the `export_task_start` and
+ // `export_task_complete` actions.
+ const maybeExportedQueryWithCreateTableFallback = Array.from(
+ exportedTablesLoading
+ ).find(
+ (t) => t.taskId === taskId && t.fallbackCreateTableQuery
+ );
+
+ if (maybeExportedQueryWithCreateTableFallback) {
+ await createFallbackTableAfterFailedExport({
+ destinationSchema:
+ maybeExportedQueryWithCreateTableFallback.destinationSchema,
+ destinationTable:
+ maybeExportedQueryWithCreateTableFallback.destinationTable,
+ fallbackCreateTableQuery:
+ maybeExportedQueryWithCreateTableFallback.fallbackCreateTableQuery,
+
+ // On error or success, we mutate the error variable which
+ // will be passed by `dispatch` outside of this conditional.
+ onError: (errorCreatingFallbackTable) => {
+ error.message = `${error.message} (and also error creating fallback: ${errorCreatingFallbackTable.message})`;
+ },
+ onSuccess: () => {
+ error = undefined; // No error because we consider the task complete after creating the fallback table.
+ },
+ });
+ }
+ }
+
dispatch({
- type: "export_error",
- error: `Error exporting ${taskId}: ${error.message}`,
- }),
+ type: "export_task_complete",
+ completedTask: {
+ taskId,
+ error,
+ },
+ });
+ },
abortSignal: abortController.signal,
})
)
@@ -86,7 +128,7 @@ export const usePollExportTasks = () => {
clearInterval(interval);
abortController.abort();
};
- }, [loadingExportTasks, stepperState, dispatch]);
+ }, [loadingExportTasks, exportedTablesLoading, stepperState, dispatch]);
};
const pollExportTaskOnce = async ({
@@ -97,7 +139,13 @@ const pollExportTaskOnce = async ({
}: {
taskId: string;
onSuccess: ({ taskId }: { taskId: string }) => void;
- onError: ({ taskId, error }: { taskId: string; error: any }) => void;
+ onError: ({
+ taskId,
+ error,
+ }: {
+ taskId: string;
+ error: { message: string; retryable: boolean };
+ }) => void;
abortSignal: AbortSignal;
}) => {
try {
@@ -118,6 +166,7 @@ const pollExportTaskOnce = async ({
} else if (data.error) {
if (!data.completed) {
console.log("WARN: Failed status, not completed:", data.error);
+ onError({ taskId, error: { message: data.error, retryable: false } });
} else {
throw new Error(data.error);
}
@@ -127,6 +176,71 @@ const pollExportTaskOnce = async ({
return;
}
- onError({ taskId, error });
+ onError({
+ taskId,
+ error: {
+ message: `Error exporting ${taskId}: ${
+ error.message ?? error.name ?? "unknown"
+ }`,
+ retryable: true,
+ },
+ });
+ }
+};
+
+/**
+ * Call the API route to create a fallback table after a failed export.
+ *
+ * Note that both `destinationTable` and `destinationSchema` should already
+ * be included in the `fallbackCreateTableQuery`, but we need them so that
+ * the endpoint can separately `CREATE SCHEMA` and `DROP TABLE` in case the
+ * schema does not yet exist, or the table already exists (we overwrite it to
+ * be consistent with behavior of Splitgraph export API).
+ */
+const createFallbackTableAfterFailedExport = async ({
+ destinationSchema,
+ destinationTable,
+ fallbackCreateTableQuery,
+ onSuccess,
+ onError,
+}: Required<
+ Pick<
+ ExportTable,
+ "destinationSchema" | "destinationTable" | "fallbackCreateTableQuery"
+ >
+> & {
+ onSuccess: () => void;
+ onError: (error: { message: string }) => void;
+}) => {
+ try {
+ const response = await fetch(
+ "/api/create-fallback-table-after-failed-export",
+ {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ destinationSchema,
+ destinationTable,
+ fallbackCreateTableQuery,
+ }),
+ }
+ );
+ const data = await response.json();
+ if (data.error || !data.success) {
+ console.log(
+ `FAIL: error from endpoint creating fallback table: ${data.error}`
+ );
+ onError({ message: data.error ?? "unknown" });
+ } else {
+ console.log("SUCCESS: created fallback table");
+ onSuccess();
+ }
+ } catch (error) {
+ console.log(`FAIL: caught error while creating fallback table: ${error}`);
+ onError({
+ message: `${error.message ?? error.name ?? "unknown"}`,
+ });
}
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index 837d824..6185105 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -8,12 +8,14 @@ export type ExportTable = {
destinationTable: string;
taskId: string;
sourceQuery?: string;
+ fallbackCreateTableQuery?: string;
};
// NOTE: Multiple tables can have the same taskId, so we track them separately
// in order to not need to redundantly poll the API for each table individually
export type ExportTask = {
taskId: string;
+ error?: { message: string; retryable: boolean };
};
export type StepperState = {
@@ -35,6 +37,7 @@ export type StepperState = {
debug?: string | null;
loadingExportTasks?: Set<ExportTask>;
completedExportTasks?: Set<ExportTask>;
+ tasksWithError?: Map<string, string[]>; // taskId -> errors
};
export type StepperAction =
@@ -65,6 +68,7 @@ const initialState: StepperState = {
exportedTablesCompleted: new Set<ExportTable>(),
loadingExportTasks: new Set<ExportTask>(),
completedExportTasks: new Set<ExportTask>(),
+ tasksWithError: new Map<string, string[]>(),
importError: null,
exportError: null,
debug: null,
@@ -217,12 +221,14 @@ const stepperReducer = (
destinationTable,
destinationSchema,
sourceQuery,
+ fallbackCreateTableQuery,
taskId,
} of tables) {
exportedTablesLoading.add({
destinationTable,
destinationSchema,
sourceQuery,
+ fallbackCreateTableQuery,
taskId,
});
}
@@ -248,11 +254,47 @@ const stepperReducer = (
stepperState: "awaiting_export",
};
+ /**
+ * NOTE: A task is "completed" even if it received an error, in which case
+ * we will retry it up to maxRetryCount if `error.retryable` is `true`
+ *
+ * That is, _all tasks_ will eventually "complete," whether successfully or not.
+ */
case "export_task_complete":
const {
- completedTask: { taskId: completedTaskId },
+ completedTask: { taskId: completedTaskId, error: maybeError },
} = action;
+ const maxRetryCount = 3;
+
+ const updatedTasksWithError = new Map(state.tasksWithError);
+ const previousErrors = updatedTasksWithError.get(completedTaskId) ?? [];
+ const hadPreviousError = previousErrors.length > 0;
+
+ if (!maybeError && hadPreviousError) {
+ updatedTasksWithError.delete(completedTaskId);
+ } else if (maybeError) {
+ updatedTasksWithError.set(completedTaskId, [
+ ...previousErrors,
+ maybeError.message,
+ ]);
+ const numAttempts = updatedTasksWithError.get(completedTaskId).length;
+
+ if (maybeError.retryable && numAttempts < maxRetryCount) {
+ console.log("RETRY: ", completedTaskId, `(${numAttempts} so far)`);
+ return {
+ ...state,
+ tasksWithError: updatedTasksWithError,
+ };
+ } else {
+ console.log(
+ "FAIL: ",
+ completedTaskId,
+ `(${numAttempts} reached max ${maxRetryCount})`
+ );
+ }
+ }
+
// One taskId could match multiple tables, so find reference to each of them
// and then use that reference to delete them from loading set and add them to completed set
const completedTables = Array.from(state.exportedTablesLoading).filter(
@@ -394,7 +436,7 @@ const useMarkAsComplete = (
if (!data.status) {
throw new Error(
- "Got unexpected resposne shape when marking import/export complete"
+ "Got unexpected response shape when marking import/export complete"
);
}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
index 650db4e..8781703 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts
@@ -38,6 +38,30 @@ namespace NodeJS {
*/
MITMPROXY_ADDRESS?: string;
+ /**
+ * Optionally provide the SEAFOWL_INSTANCE_URL to use for creating fallback tables
+ * when an export fails.
+ *
+ * Note that at the moment, this must only be set to https://demo.seafowl.cloud
+ * because that's where Splitgraph exports to by default, and we are not currently
+ * passing any instance URL to the Splitgraph export API.
+ */
+ SEAFOWL_INSTANCE_URL?: "https://demo.seafowl.cloud";
+
+ /**
+ * Optionally provide the SEAFOWL_INSTANCE_SECRET to use for creating fallback tables
+ * when an export fails.
+ */
+ SEAFOWL_INSTANCE_SECRET?: string;
+
+ /**
+ * Optionally provide the dbname to use for creating fallback tables
+ * when an export fails.
+ *
+ * Note this MUST match the NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE
+ */
+ SEAFOWL_INSTANCE_DATABASE?: string;
+
/**
* The namespace of the repository in Splitgraph where metadata is stored
* containing the state of imported GitHub repositories, which should contain
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/seafowl-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/seafowl-db.ts
new file mode 100644
index 0000000..8ae7af1
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/seafowl-db.ts
@@ -0,0 +1,83 @@
+import { makeSeafowlHTTPContext } from "@madatdata/core";
+
+export const makeAuthenticatedSeafowlHTTPContext = () => {
+ const { instanceURL, instanceDatabase, instanceSecret } =
+ getRequiredValidAuthenticatedSeafowlInstanceConfig();
+
+ // NOTE: This config object is a mess and will be simplified in a future madatdata update
+ // It's only necessary here because we're passing a secret
+ return makeSeafowlHTTPContext({
+ database: {
+ dbname: instanceDatabase,
+ },
+ authenticatedCredential: {
+ token: instanceSecret,
+ anonymous: false,
+ },
+ host: {
+ baseUrls: {
+ sql: instanceURL,
+ gql: "...",
+ auth: "...",
+ },
+ dataHost: new URL(instanceURL).host,
+ apexDomain: "...",
+ apiHost: "...",
+ postgres: {
+ host: "127.0.0.1",
+ port: 6432,
+ ssl: false,
+ },
+ },
+ });
+};
+
+const getRequiredValidAuthenticatedSeafowlInstanceConfig = () => {
+ const instanceURL = process.env.SEAFOWL_INSTANCE_URL;
+
+ if (!instanceURL) {
+ throw new Error("Missing SEAFOWL_INSTANCE_URL");
+ }
+
+ // This could be temporary if we want to allow configuring the instance URL,
+ // but for now we export to Splitgraph using no instance URL, which means
+ // it exports to demo.seafowl.cloud, and we only use this for creating
+ // fallback tables on failed exports (which is mostly a workaround anyway)
+ if (instanceURL && instanceURL !== "https://demo.seafowl.cloud") {
+ throw new Error(`If SEAFOWL_INSTANCE_URL is set, it should be set to https://demo.seafowl.cloud,
+ because that's where Splitgraph exports to by default, and we are not currently passing
+ any instance URL to the Splitgraph export API (though we could do that).
+ `);
+ }
+
+ const instanceSecret = process.env.SEAFOWL_INSTANCE_SECRET;
+ if (!instanceSecret) {
+ throw new Error("Missing SEAFOWL_INSTANCE_SECRET");
+ }
+
+ // This is at the config level, just like SPLITGRAPH_NAMESPACE, since the two
+ // of them are supposed to match
+ const instanceDatabase = process.env.SEAFOWL_INSTANCE_DATABASE;
+ if (!instanceDatabase) {
+ throw new Error("Missing SEAFOWL_INSTANCE_DATABASE");
+ }
+
+ const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+ if (!META_NAMESPACE) {
+ throw new Error(
+ "Missing NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE"
+ );
+ }
+
+ if (instanceDatabase !== META_NAMESPACE) {
+ throw new Error(`SEAFOWL_INSTANCE_DATABASE (${instanceDatabase}) should match
+ NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE (${META_NAMESPACE})`);
+ }
+
+ return {
+ instanceURL,
+ instanceSecret,
+ instanceDatabase,
+ };
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
index ca1f2f2..5aa95b9 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts
@@ -7,6 +7,7 @@ import { defaultSplitgraphHost } from "@madatdata/core";
const SPLITGRAPH_API_KEY = process.env.SPLITGRAPH_API_KEY;
const SPLITGRAPH_API_SECRET = process.env.SPLITGRAPH_API_SECRET;
+// Throw top level error on missing keys because these are _always_ required app to run
if (!SPLITGRAPH_API_KEY || !SPLITGRAPH_API_SECRET) {
throw new Error(
"Environment variable SPLITGRAPH_API_KEY or SPLITGRAPH_API_SECRET is not set." +
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
index 099d4f2..94ebabe 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
@@ -17,80 +17,111 @@ export const makeQueriesToExport = ({
sourceQuery: string;
destinationSchema: string;
destinationTable: string;
+ /**
+ * Optionally provide a DDL query to create the (empty) destination table in
+ * case the export from Splitgraph fails. This is a workaround of a bug where
+ * exports from Splitgraph to Seafowl fail if the destination table does not
+ * contain any rows. See: https://github.com/splitgraph/seafowl/issues/423
+ *
+ * This way, even if a table fails to load, we can at least reference it in subsequent
+ * analytics queries without challenges like conditionally checking if it exists.
+ */
+ fallbackCreateTableQuery?: string;
}[] => [
+ {
+ destinationSchema: seafowlDestinationSchema,
+ destinationTable: "simple_stargazers_query",
+ sourceQuery: `
+SELECT * FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".stargazers`,
+ },
+
{
destinationSchema: seafowlDestinationSchema,
destinationTable: "monthly_user_stats",
sourceQuery: `
- WITH
+WITH
- commits AS (
- SELECT
- date_trunc('month', created_at) AS created_at_month,
- author->>'login' AS username,
- count(*) as no_commits
- FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".commits
- GROUP BY 1, 2
- ),
+commits AS (
+ SELECT
+ date_trunc('month', created_at) AS created_at_month,
+ author->>'login' AS username,
+ count(*) as no_commits
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".commits
+ GROUP BY 1, 2
+),
- comments AS (
- SELECT
- date_trunc('month', created_at) AS created_at_month,
- "user"->>'login' AS username,
- count(*) filter (where exists(select regexp_matches(issue_url, '.*/pull/.*'))) as no_pull_request_comments,
- count(*) filter (where exists(select regexp_matches(issue_url, '.*/issue/.*'))) as no_issue_comments,
- sum(length(body)) as total_comment_length
- FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".comments
- GROUP BY 1, 2
- ),
+comments AS (
+ SELECT
+ date_trunc('month', created_at) AS created_at_month,
+ "user"->>'login' AS username,
+ count(*) filter (where exists(select regexp_matches(issue_url, '.*/pull/.*'))) as no_pull_request_comments,
+ count(*) filter (where exists(select regexp_matches(issue_url, '.*/issue/.*'))) as no_issue_comments,
+ sum(length(body)) as total_comment_length
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".comments
+ GROUP BY 1, 2
+),
- pull_requests AS (
- WITH pull_request_creator AS (
- SELECT id, "user"->>'login' AS username
- FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_requests
- )
+pull_requests AS (
+ WITH pull_request_creator AS (
+ SELECT id, "user"->>'login' AS username
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_requests
+ )
- SELECT
- date_trunc('month', updated_at) AS created_at_month,
- username,
- count(*) filter (where merged = true) AS merged_pull_requests,
- count(*) AS total_pull_requests,
- sum(additions::integer) filter (where merged = true) AS lines_added,
- sum(deletions::integer) filter (where merged = true) AS lines_deleted
- FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_request_stats
- INNER JOIN pull_request_creator USING (id)
- GROUP BY 1, 2
- ),
+ SELECT
+ date_trunc('month', updated_at) AS created_at_month,
+ username,
+ count(*) filter (where merged = true) AS merged_pull_requests,
+ count(*) AS total_pull_requests,
+ sum(additions::integer) filter (where merged = true) AS lines_added,
+ sum(deletions::integer) filter (where merged = true) AS lines_deleted
+ FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_request_stats
+ INNER JOIN pull_request_creator USING (id)
+ GROUP BY 1, 2
+),
- all_months_users AS (
- SELECT DISTINCT created_at_month, username FROM commits
- UNION SELECT DISTINCT created_at_month, username FROM comments
- UNION SELECT DISTINCT created_at_month, username FROM pull_requests
- ),
+all_months_users AS (
+ SELECT DISTINCT created_at_month, username FROM commits
+ UNION SELECT DISTINCT created_at_month, username FROM comments
+ UNION SELECT DISTINCT created_at_month, username FROM pull_requests
+),
- user_stats AS (
- SELECT
- amu.created_at_month,
- amu.username,
- COALESCE(cmt.no_commits, 0) AS no_commits,
- COALESCE(cmnt.no_pull_request_comments, 0) AS no_pull_request_comments,
- COALESCE(cmnt.no_issue_comments, 0) AS no_issue_comments,
- COALESCE(cmnt.total_comment_length, 0) AS total_comment_length,
- COALESCE(pr.merged_pull_requests, 0) AS merged_pull_requests,
- COALESCE(pr.total_pull_requests, 0) AS total_pull_requests,
- COALESCE(pr.lines_added, 0) AS lines_added,
- COALESCE(pr.lines_deleted, 0) AS lines_deleted
+user_stats AS (
+ SELECT
+ amu.created_at_month,
+ amu.username,
+ COALESCE(cmt.no_commits, 0) AS no_commits,
+ COALESCE(cmnt.no_pull_request_comments, 0) AS no_pull_request_comments,
+ COALESCE(cmnt.no_issue_comments, 0) AS no_issue_comments,
+ COALESCE(cmnt.total_comment_length, 0) AS total_comment_length,
+ COALESCE(pr.merged_pull_requests, 0) AS merged_pull_requests,
+ COALESCE(pr.total_pull_requests, 0) AS total_pull_requests,
+ COALESCE(pr.lines_added, 0) AS lines_added,
+ COALESCE(pr.lines_deleted, 0) AS lines_deleted
- FROM all_months_users amu
- LEFT JOIN commits cmt ON amu.created_at_month = cmt.created_at_month AND amu.username = cmt.username
- LEFT JOIN comments cmnt ON amu.created_at_month = cmnt.created_at_month AND amu.username = cmnt.username
- LEFT JOIN pull_requests pr ON amu.created_at_month = pr.created_at_month AND amu.username = pr.username
+ FROM all_months_users amu
+ LEFT JOIN commits cmt ON amu.created_at_month = cmt.created_at_month AND amu.username = cmt.username
+ LEFT JOIN comments cmnt ON amu.created_at_month = cmnt.created_at_month AND amu.username = cmnt.username
+ LEFT JOIN pull_requests pr ON amu.created_at_month = pr.created_at_month AND amu.username = pr.username
- ORDER BY created_at_month ASC, username ASC
- )
+ ORDER BY created_at_month ASC, username ASC
+)
- SELECT * FROM user_stats;
+SELECT * FROM user_stats;
`,
+ fallbackCreateTableQuery: `
+CREATE TABLE "${seafowlDestinationSchema}".monthly_user_stats (
+ created_at_month TIMESTAMP,
+ username VARCHAR,
+ no_commits BIGINT,
+ no_pull_request_comments BIGINT,
+ no_issue_comments BIGINT,
+ total_comment_length BIGINT,
+ merged_pull_requests BIGINT,
+ total_pull_requests BIGINT,
+ lines_added BIGINT,
+ lines_deleted BIGINT
+);
+ `,
},
{
destinationSchema: seafowlDestinationSchema,
@@ -111,6 +142,62 @@ SELECT
FROM
"${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}"."issue_reactions"
GROUP BY 1, 2 ORDER BY 2, 3 DESC;
+`,
+ fallbackCreateTableQuery: `
+CREATE TABLE "${seafowlDestinationSchema}".monthly_issue_stats (
+ issue_number BIGINT,
+ created_at_month TIMESTAMP,
+ total_reacts BIGINT,
+ no_plus_one BIGINT,
+ no_minus_one BIGINT,
+ no_laugh BIGINT,
+ no_confused BIGINT,
+ no_heart BIGINT,
+ no_hooray BIGINT,
+ no_rocket BIGINT,
+ no_eyes BIGINT
+);
`,
},
];
+
+/** A generic demo query that can be used to show off Splitgraph */
+export const genericDemoQuery = `WITH t (
+ c_int16_smallint,
+ c_int32_int,
+ c_int64_bigint,
+ c_utf8_char,
+ c_utf8_varchar,
+ c_utf8_text,
+ c_float32_float,
+ c_float32_real,
+ c_boolean_boolean,
+ c_date32_date,
+ c_timestamp_microseconds_timestamp
+
+ ) AS (
+ VALUES(
+ /* Int16 / SMALLINT */
+ 42::SMALLINT,
+ /* Int32 / INT */
+ 99::INT,
+ /* Int64 / BIGINT */
+ 420420::BIGINT,
+ /* Utf8 / CHAR */
+ 'x'::CHAR,
+ /* Utf8 / VARCHAR */
+ 'abcdefghijklmnopqrstuvwxyz'::VARCHAR,
+ /* Utf8 / TEXT */
+ 'zyxwvutsrqponmlkjihgfedcba'::TEXT,
+ /* Float32 / FLOAT */
+ 4.4::FLOAT,
+ /* Float32 / REAL */
+ 2.0::REAL,
+ /* Boolean / BOOLEAN */
+ 't'::BOOLEAN,
+ /* Date32 / DATE */
+ '1997-06-17'::DATE,
+ /* Timestamp(us) / TIMESTAMP */
+ '2018-11-11T11:11:11.111111111'::TIMESTAMP
+ )
+ ) SELECT * FROM t;`;
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/create-fallback-table-after-failed-export.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/create-fallback-table-after-failed-export.ts
new file mode 100644
index 0000000..9a2340a
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/create-fallback-table-after-failed-export.ts
@@ -0,0 +1,85 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { makeAuthenticatedSeafowlHTTPContext } from "../../lib/backend/seafowl-db";
+import type {
+ CreateFallbackTableForFailedExportRequestShape,
+ CreateFallbackTableForFailedExportResponseData,
+} from "../../types";
+
+/**
+ curl -i \
+ -H "Content-Type: application/json" http://localhost:3000/api/create-fallback-table-after-failed-export \
+ -d '{ "taskId": "2923fd6f-2197-495a-9df1-2428a9ca8dee" }'
+ */
+export default async function handler(
+ req: NextApiRequest,
+ res: NextApiResponse<CreateFallbackTableForFailedExportResponseData>
+) {
+ const { fallbackCreateTableQuery, destinationSchema, destinationTable } =
+ req.body as CreateFallbackTableForFailedExportRequestShape;
+
+ const errors = [];
+
+ if (!fallbackCreateTableQuery) {
+ errors.push("missing fallbackCreateTableQuery in request body");
+ }
+
+ if (!destinationSchema) {
+ errors.push("missing destinationSchema in request body");
+ }
+
+ if (!destinationTable) {
+ errors.push("missing destinationTable in request body");
+ }
+
+ if (typeof fallbackCreateTableQuery !== "string") {
+ errors.push("invalid fallbackCreateTableQuery in request body");
+ }
+
+ if (typeof destinationSchema !== "string") {
+ errors.push("invalid destinationSchema in request body");
+ }
+
+ if (!fallbackCreateTableQuery.includes(destinationSchema)) {
+ errors.push("fallbackCreateTableQuery must include destinationSchema");
+ }
+
+ if (typeof destinationTable !== "string") {
+ errors.push("invalid destinationTable in request body");
+ }
+
+ if (!fallbackCreateTableQuery.includes(destinationTable)) {
+ errors.push("fallbackCreateTableQuery must include destinationTable");
+ }
+
+ if (errors.length > 0) {
+ res.status(400).json({ error: errors.join(", "), success: false });
+ return;
+ }
+
+ try {
+ await createFallbackTable(
+ req.body as CreateFallbackTableForFailedExportRequestShape
+ );
+ res.status(200).json({ success: true });
+ return;
+ } catch (err) {
+ console.trace(err);
+ res.status(400).json({ error: err.message, success: false });
+ }
+}
+
+const createFallbackTable = async ({
+ fallbackCreateTableQuery,
+ destinationTable,
+ destinationSchema,
+}: CreateFallbackTableForFailedExportRequestShape) => {
+ const { client } = makeAuthenticatedSeafowlHTTPContext();
+
+ // NOTE: client.execute should never throw (on error it returns an object including .error)
+ // i.e. Even if the table doesn't exist, or if the schema already existed, we don't need to try/catch
+ await client.execute(
+ `DROP TABLE IF EXISTS "${destinationSchema}"."${destinationTable}";`
+ );
+ await client.execute(`CREATE SCHEMA "${destinationSchema}";`);
+ await client.execute(fallbackCreateTableQuery);
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
index f377047..6ce9024 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
@@ -20,6 +20,7 @@ export type ExportQueryInput = {
sourceQuery: string;
destinationSchema: string;
destinationTable: string;
+ fallbackCreateTableQuery?: string;
};
export type StartExportToSeafowlRequestShape =
@@ -44,3 +45,24 @@ export type StartExportToSeafowlResponseData =
}[];
}
| { error: string };
+
+export type CreateFallbackTableForFailedExportRequestShape = {
+ /**
+ * The query to execute to create the fallback table. Note that it should
+ * already include `destinationSchema` and `destinationTable` in the query,
+ * but those still need to be passed separately to the endpoint so that it
+ * can `CREATE SCHEMA` and `DROP TABLE` prior to executing the `CREATE TABLE` query.
+ */
+ fallbackCreateTableQuery: string;
+ destinationSchema: string;
+ destinationTable: string;
+};
+
+export type CreateFallbackTableForFailedExportResponseData =
+ | {
+ error: string;
+ success: false;
+ }
+ | {
+ success: true;
+ };
From c260e109bcbccb7843913a87249b6c7b76e3245f Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Fri, 23 Jun 2023 04:17:19 +0100
Subject: [PATCH 29/36] When embedding Seafowl query of exported queries,
select from the destinationTable
The point of exporting a query from Splitgraph to Seafowl is that once
the result is in Seafowl, we can just select from the destinationTable
and forget about the original query (which might not even be compatible
with Seafowl). So make sure that when we're embedding an exported query,
we only render the query in the embedded Splitgraph query editor, and
for the embedded Seafowl Console, we render a query that simply selects
from the destinationTable.
---
.../EmbeddedQuery/EmbeddedQuery.tsx | 22 +++++++++++++++++--
.../ImportExportStepper/ExportPanel.tsx | 6 +++++
2 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
index ccc9053..1d47792 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
@@ -24,6 +24,7 @@ export const ExportEmbedPreviewTableOrQuery = <
importedRepository,
exportInput,
makeQuery,
+ makeSeafowlQuery,
makeMatchInputToExported,
}: {
exportInput: ExportInputShape;
@@ -33,6 +34,12 @@ export const ExportEmbedPreviewTableOrQuery = <
splitgraphRepository: string;
}
) => string;
+ makeSeafowlQuery?: (
+ tableOrQueryInput: ExportInputShape & {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+ }
+ ) => string;
makeMatchInputToExported: (
tableOrQueryInput: ExportInputShape
) => (exported: ExportTable) => boolean;
@@ -79,7 +86,10 @@ export const ExportEmbedPreviewTableOrQuery = <
return {
anchor: "Open in Console",
href: makeSeafowlQueryHref(
- makeQuery({ ...exportInput, ...importedRepository })
+ (makeSeafowlQuery ?? makeQuery)({
+ ...exportInput,
+ ...importedRepository,
+ })
),
};
}
@@ -158,7 +168,15 @@ export const ExportEmbedPreviewTableOrQuery = <
display: selectedTab === "splitgraph" ? "none" : "block",
}}
>
- <SeafowlEmbeddedQuery {...embedProps} />
+ <SeafowlEmbeddedQuery
+ {...embedProps}
+ makeQuery={
+ makeSeafowlQuery
+ ? () =>
+ makeSeafowlQuery({ ...exportInput, ...importedRepository })
+ : embedProps.makeQuery
+ }
+ />
</div>
)}
</div>
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index 264609b..716cbc2 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -323,7 +323,13 @@ const ExportPreview = ({
key={`export-query-preview-${exportQuery.destinationTable}-${exportQuery.destinationSchema}`}
exportInput={exportQuery}
importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+ // This is the query we run on Splitgraph that we exported to Seafowl
makeQuery={({ sourceQuery }) => sourceQuery}
+ // But once it's exported, we can just select from its table in Seafowl (and
+ // besides, the sourceQuery might not be compatible with Seafowl anyway)
+ makeSeafowlQuery={({ destinationSchema, destinationTable }) =>
+ `SELECT * FROM "${destinationSchema}"."${destinationTable}";`
+ }
makeMatchInputToExported={(exportQueryInput) =>
(exportTable: ExportTable) => {
return (
From 4cc3e4142dba7b48f54446145f4658d675a763bf Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 28 Jun 2023 17:18:55 +0100
Subject: [PATCH 30/36] Move embedded preview components to be shared with
export panel and repo page
---
.../EmbeddedQuery/EmbeddedPreviews.module.css | 13 ++
.../EmbeddedQuery/EmbeddedPreviews.tsx | 153 +++++++++++++++
.../EmbeddedQuery/EmbeddedQuery.tsx | 21 +-
.../ExportPanel.module.css | 25 ---
.../ImportExportStepper/ExportPanel.tsx | 183 ++++--------------
.../ImportExportStepper/export-hooks.tsx | 6 +-
.../ImportExportStepper/stepper-states.ts | 38 +---
.../lib/config/github-tables.ts | 33 +++-
.../lib/config/queries-to-export.ts | 21 ++
.../lib/util.ts | 36 ++++
.../types.ts | 8 +
11 files changed, 318 insertions(+), 219 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.module.css
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.tsx
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib/util.ts
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.module.css
new file mode 100644
index 0000000..47c5be5
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.module.css
@@ -0,0 +1,13 @@
+.embeddedPreviewHeading {
+ margin-bottom: 0;
+}
+
+.embeddedPreviewDescription {
+ margin-bottom: 1rem;
+}
+
+.note {
+ font-size: small;
+ /* color: red !important; */
+ display: block;
+}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.tsx
new file mode 100644
index 0000000..81600a1
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.tsx
@@ -0,0 +1,153 @@
+import styles from "./EmbeddedPreviews.module.css";
+import type {
+ ExportTable,
+ ExportQueryInput,
+ ExportTableInput,
+} from "../../types";
+
+import { EmbedPreviewTableOrQuery } from "../EmbeddedQuery/EmbeddedQuery";
+import { ComponentProps } from "react";
+
+export const EmbeddedTablePreviewHeadingAndDescription = ({
+ exportComplete,
+}: {
+ exportComplete: boolean;
+}) => {
+ return (
+ <>
+ {!exportComplete ? (
+ <>
+ <h2 className={styles.embeddedPreviewHeading}>Tables to Export</h2>
+ <p className={styles.embeddedPreviewDescription}>
+ These are the tables that we'll export from Splitgraph to Seafowl.
+ You can query them in Splitgraph now, and then when the export is
+ complete, you'll be able to query them in Seafowl too.
+ </p>
+ </>
+ ) : (
+ <>
+ <h2 className={styles.embeddedPreviewHeading}>Exported Tables</h2>
+ <p className={styles.embeddedPreviewDescription}>
+ We successfully exported the tables to Seafowl, so now you can query
+ them in Seafowl too.
+ </p>
+ </>
+ )}
+ </>
+ );
+};
+
+export const EmbeddedTablePreviews = ({
+ tablesToExport,
+ splitgraphRepository,
+ splitgraphNamespace,
+ useLoadingOrCompleted,
+}: {
+ tablesToExport: ExportTableInput[];
+ splitgraphRepository: string;
+ splitgraphNamespace: string;
+ useLoadingOrCompleted: ComponentProps<
+ typeof EmbedPreviewTableOrQuery
+ >["useLoadingOrCompleted"];
+}) => {
+ return (
+ <>
+ {tablesToExport.map((exportTable) => (
+ <EmbedPreviewTableOrQuery
+ key={`export-table-preview-${exportTable.table}`}
+ exportInput={exportTable}
+ importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+ makeQuery={({ splitgraphNamespace, splitgraphRepository, table }) =>
+ `SELECT * FROM "${splitgraphNamespace}/${splitgraphRepository}"."${table}";`
+ }
+ useLoadingOrCompleted={useLoadingOrCompleted}
+ makeMatchInputToExported={(exportTableInput) => (exportTable) => {
+ return (
+ exportTable.destinationSchema === exportTableInput.repository &&
+ exportTable.destinationTable === exportTableInput.table
+ );
+ }}
+ />
+ ))}
+ </>
+ );
+};
+
+export const EmbeddedQueryPreviewHeadingAndDescription = ({
+ exportComplete,
+}: {
+ exportComplete: boolean;
+}) => {
+ return (
+ <>
+ {" "}
+ {!exportComplete ? (
+ <>
+ <h2 className={styles.embeddedPreviewHeading}>Queries to Export</h2>
+ <p className={styles.embeddedPreviewDescription}>
+ We've prepared a few queries to export from Splitgraph to Seafowl,
+ so that we can use them to render the charts that we want.
+ Splitgraph will execute the query and insert its result into
+ Seafowl. You can query them in Splitgraph now, and then when the
+ export is complete, you'll be able to query them in Seafowl too.
+ </p>
+ </>
+ ) : (
+ <>
+ <h2 className={styles.embeddedPreviewHeading}>Exported Queries</h2>
+ <p className={styles.embeddedPreviewDescription}>
+ We successfully exported these queries from Splitgraph to Seafowl,
+ so now you can query them in Seafowl too.{" "}
+ <em className={styles.note}>
+ Note: If some queries failed to export, it's probably because they
+ had empty result sets (e.g. the table of issue reactions)
+ </em>
+ </p>
+ </>
+ )}
+ </>
+ );
+};
+
+export const EmbeddedQueryPreviews = ({
+ queriesToExport,
+ splitgraphRepository,
+ splitgraphNamespace,
+ useLoadingOrCompleted,
+}: {
+ queriesToExport: ExportQueryInput[];
+ splitgraphRepository: string;
+ splitgraphNamespace: string;
+ useLoadingOrCompleted: ComponentProps<
+ typeof EmbedPreviewTableOrQuery
+ >["useLoadingOrCompleted"];
+}) => {
+ return (
+ <>
+ {queriesToExport.map((exportQuery) => (
+ <EmbedPreviewTableOrQuery
+ key={`export-query-preview-${exportQuery.destinationTable}-${exportQuery.destinationSchema}`}
+ exportInput={exportQuery}
+ importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+ // This is the query we run on Splitgraph that we exported to Seafowl
+ makeQuery={({ sourceQuery }) => sourceQuery}
+ // But once it's exported, we can just select from its table in Seafowl (and
+ // besides, the sourceQuery might not be compatible with Seafowl anyway)
+ makeSeafowlQuery={({ destinationSchema, destinationTable }) =>
+ `SELECT * FROM "${destinationSchema}"."${destinationTable}";`
+ }
+ useLoadingOrCompleted={useLoadingOrCompleted}
+ makeMatchInputToExported={(exportQueryInput) =>
+ (exportTable: ExportTable) => {
+ return (
+ exportTable.destinationSchema ===
+ exportQueryInput.destinationSchema &&
+ exportTable.destinationTable ===
+ exportQueryInput.destinationTable
+ );
+ }}
+ />
+ ))}
+ </>
+ );
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
index 1d47792..68790a7 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.tsx
@@ -1,5 +1,9 @@
import EmbeddedQueryStyles from "./EmbeddedQuery.module.css";
-import type { ExportQueryInput, ExportTableInput } from "../../types";
+import type {
+ ExportTable,
+ ExportQueryInput,
+ ExportTableInput,
+} from "../../types";
import { useState, useMemo } from "react";
import {
makeSplitgraphQueryHref,
@@ -9,16 +13,13 @@ import {
SplitgraphEmbeddedQuery,
SeafowlEmbeddedQuery,
} from "../RepositoryAnalytics/ImportedRepoMetadata";
-import { useStepperDebug } from "../ImportExportStepper/StepperContext";
-import { useFindMatchingExportTable } from "../ImportExportStepper/export-hooks";
-
-import type { ExportTable } from "../ImportExportStepper/stepper-states";
import { LoadingBar } from "../LoadingBar";
import { TabButton } from "./TabButton";
+import { useDebug } from "../../lib/util";
-export const ExportEmbedPreviewTableOrQuery = <
+export const EmbedPreviewTableOrQuery = <
ExportInputShape extends ExportQueryInput | ExportTableInput
>({
importedRepository,
@@ -26,7 +27,11 @@ export const ExportEmbedPreviewTableOrQuery = <
makeQuery,
makeSeafowlQuery,
makeMatchInputToExported,
+ useLoadingOrCompleted,
}: {
+ useLoadingOrCompleted?: (
+ isMatch?: (candidateTable: ExportTable) => boolean
+ ) => { loading: boolean; completed: boolean };
exportInput: ExportInputShape;
makeQuery: (
tableOrQueryInput: ExportInputShape & {
@@ -48,7 +53,7 @@ export const ExportEmbedPreviewTableOrQuery = <
splitgraphRepository: string;
};
}) => {
- const debug = useStepperDebug();
+ const debug = useDebug();
const embedProps = {
importedRepository,
@@ -59,7 +64,7 @@ export const ExportEmbedPreviewTableOrQuery = <
makeQuery: () => makeQuery({ ...exportInput, ...importedRepository }),
};
- const { loading, completed } = useFindMatchingExportTable(
+ const { loading, completed } = useLoadingOrCompleted(
makeMatchInputToExported(exportInput)
);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
index d99dfe7..7ac88a4 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css
@@ -1,19 +1,8 @@
.exportPanel {
- /* Styles for the export panel container */
background: inherit;
margin-top: 2rem;
}
-.querySeafowlButton {
- /* Styles for the query Seafowl button */
- background: inherit;
-}
-
-.viewReportButton {
- /* Styles for the view report button */
- background: inherit;
-}
-
.startExportButton {
color: var(--background);
background-color: var(--secondary);
@@ -43,17 +32,3 @@
.exportInfo p {
margin-bottom: 1rem;
}
-
-.exportPreviewHeading {
- margin-bottom: 0;
-}
-
-.exportPreviewDescription {
- margin-bottom: 1rem;
-}
-
-.exportNote {
- font-size: small;
- /* color: red !important; */
- display: block;
-}
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
index 716cbc2..ab8f124 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx
@@ -1,26 +1,30 @@
import { useStepper } from "./StepperContext";
import styles from "./ExportPanel.module.css";
-import { splitgraphTablesToExportToSeafowl } from "../../lib/config/github-tables";
+import { useTablesToExport } from "../../lib/config/github-tables";
import {
genericDemoQuery,
- makeQueriesToExport,
+ useQueriesToExport,
} from "../../lib/config/queries-to-export";
import type {
- ExportQueryInput,
- ExportTableInput,
StartExportToSeafowlRequestShape,
StartExportToSeafowlResponseData,
} from "../../types";
-import { useMemo, useCallback } from "react";
+import { useCallback } from "react";
import { StepTitle } from "./StepTitle";
import { StepDescription } from "./StepDescription";
import { makeSplitgraphQueryHref } from "../RepositoryAnalytics/ImportedRepoMetadata";
-import type { ExportTable } from "./stepper-states";
-import { ExportEmbedPreviewTableOrQuery } from "../EmbeddedQuery/EmbeddedQuery";
import { usePollExportTasks } from "./export-hooks";
+import {
+ EmbeddedTablePreviews,
+ EmbeddedQueryPreviews,
+ EmbeddedTablePreviewHeadingAndDescription,
+ EmbeddedQueryPreviewHeadingAndDescription,
+} from "../EmbeddedQuery/EmbeddedPreviews";
+import { useFindMatchingExportTable } from "./export-hooks";
+
export const ExportPanel = () => {
const [
{ stepperState, exportError, splitgraphRepository, splitgraphNamespace },
@@ -29,29 +33,15 @@ export const ExportPanel = () => {
usePollExportTasks();
- const queriesToExport = useMemo<ExportQueryInput[]>(
- () =>
- makeQueriesToExport({
- splitgraphSourceRepository: splitgraphRepository,
- splitgraphSourceNamespace: splitgraphNamespace,
- seafowlDestinationSchema: `${splitgraphNamespace}/${splitgraphRepository}`,
- }),
- [splitgraphRepository, splitgraphNamespace]
- );
+ const queriesToExport = useQueriesToExport({
+ splitgraphNamespace,
+ splitgraphRepository,
+ });
- const tablesToExport = useMemo<ExportTableInput[]>(
- () =>
- splitgraphTablesToExportToSeafowl.map((tableName) => ({
- namespace: splitgraphNamespace,
- repository: splitgraphRepository,
- table: tableName,
- })),
- [
- splitgraphNamespace,
- splitgraphRepository,
- splitgraphTablesToExportToSeafowl,
- ]
- );
+ const tablesToExport = useTablesToExport({
+ splitgraphNamespace,
+ splitgraphRepository,
+ });
const handleStartExport = useCallback(async () => {
const abortController = new AbortController();
@@ -221,126 +211,27 @@ ${genericDemoQuery}
{["import_complete", "awaiting_export", "export_complete"].includes(
stepperState
) && (
- <ExportPreview
- stepperState={
- stepperState as
- | "import_complete"
- | "awaiting_export"
- | "export_complete"
- }
- tablesToExport={tablesToExport}
- queriesToExport={queriesToExport}
- splitgraphRepository={splitgraphRepository}
- splitgraphNamespace={splitgraphNamespace}
- />
- )}
- </div>
- );
-};
-
-const ExportPreview = ({
- stepperState,
- tablesToExport,
- queriesToExport,
- splitgraphRepository,
- splitgraphNamespace,
-}: {
- stepperState: "import_complete" | "awaiting_export" | "export_complete";
- tablesToExport: ExportTableInput[];
- queriesToExport: ExportQueryInput[];
- splitgraphRepository: string;
- splitgraphNamespace: string;
-}) => {
- return (
- <>
- {stepperState !== "export_complete" ? (
- <>
- <h2 className={styles.exportPreviewHeading}>Tables to Export</h2>
- <p className={styles.exportPreviewDescription}>
- These are the tables that we'll export from Splitgraph to Seafowl.
- You can query them in Splitgraph now, and then when the export is
- complete, you'll be able to query them in Seafowl too.
- </p>
- </>
- ) : (
<>
- <h2 className={styles.exportPreviewHeading}>Exported Tables</h2>
- <p className={styles.exportPreviewDescription}>
- We successfully exported the tables to Seafowl, so now you can query
- them in Seafowl too.
- </p>
- </>
- )}
-
- {tablesToExport
- .filter((_) => true)
- .map((exportTable) => (
- <ExportEmbedPreviewTableOrQuery
- key={`export-table-preview-${exportTable.table}`}
- exportInput={exportTable}
- importedRepository={{ splitgraphNamespace, splitgraphRepository }}
- makeQuery={({ splitgraphNamespace, splitgraphRepository, table }) =>
- `SELECT * FROM "${splitgraphNamespace}/${splitgraphRepository}"."${table}";`
- }
- makeMatchInputToExported={(exportTableInput) => (exportTable) => {
- return (
- exportTable.destinationSchema === exportTableInput.repository &&
- exportTable.destinationTable === exportTableInput.table
- );
- }}
+ <EmbeddedTablePreviewHeadingAndDescription
+ exportComplete={stepperState === "export_complete"}
+ />
+ <EmbeddedTablePreviews
+ useLoadingOrCompleted={useFindMatchingExportTable}
+ tablesToExport={tablesToExport}
+ splitgraphRepository={splitgraphRepository}
+ splitgraphNamespace={splitgraphNamespace}
+ />
+ <EmbeddedQueryPreviewHeadingAndDescription
+ exportComplete={stepperState === "export_complete"}
+ />
+ <EmbeddedQueryPreviews
+ useLoadingOrCompleted={useFindMatchingExportTable}
+ queriesToExport={queriesToExport}
+ splitgraphRepository={splitgraphRepository}
+ splitgraphNamespace={splitgraphNamespace}
/>
- ))}
-
- {stepperState !== "export_complete" ? (
- <>
- <h2 className={styles.exportPreviewHeading}>Queries to Export</h2>
- <p className={styles.exportPreviewDescription}>
- We've prepared a few queries to export from Splitgraph to Seafowl,
- so that we can use them to render the charts that we want.
- Splitgraph will execute the query and insert its result into
- Seafowl. You can query them in Splitgraph now, and then when the
- export is complete, you'll be able to query them in Seafowl too.
- </p>
- </>
- ) : (
- <>
- <h2 className={styles.exportPreviewHeading}>Exported Queries</h2>
- <p className={styles.exportPreviewDescription}>
- We successfully exported these queries from Splitgraph to Seafowl,
- so now you can query them in Seafowl too.{" "}
- <em className={styles.exportNote}>
- Note: If some queries failed to export, it's probably because they
- had empty result sets (e.g. the table of issue reactions)
- </em>
- </p>
</>
)}
-
- {queriesToExport
- .filter((_) => true)
- .map((exportQuery) => (
- <ExportEmbedPreviewTableOrQuery
- key={`export-query-preview-${exportQuery.destinationTable}-${exportQuery.destinationSchema}`}
- exportInput={exportQuery}
- importedRepository={{ splitgraphNamespace, splitgraphRepository }}
- // This is the query we run on Splitgraph that we exported to Seafowl
- makeQuery={({ sourceQuery }) => sourceQuery}
- // But once it's exported, we can just select from its table in Seafowl (and
- // besides, the sourceQuery might not be compatible with Seafowl anyway)
- makeSeafowlQuery={({ destinationSchema, destinationTable }) =>
- `SELECT * FROM "${destinationSchema}"."${destinationTable}";`
- }
- makeMatchInputToExported={(exportQueryInput) =>
- (exportTable: ExportTable) => {
- return (
- exportTable.destinationSchema ===
- exportQueryInput.destinationSchema &&
- exportTable.destinationTable ===
- exportQueryInput.destinationTable
- );
- }}
- />
- ))}
- </>
+ </div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
index 9e7f7d6..310f9db 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/export-hooks.tsx
@@ -1,5 +1,5 @@
import { useEffect, useMemo } from "react";
-import type { ExportTable } from "./stepper-states";
+import type { ExportTable } from "../../types";
import { useStepper } from "./StepperContext";
/**
@@ -34,8 +34,8 @@ export const useFindMatchingExportTable = (
// and thus we don't have the sets of exportedTablesCompleted, but we know they exist
const exportFullyCompleted = stepperState === "export_complete";
- const completed = matchingCompletedTable ?? (exportFullyCompleted || false);
- const loading = matchingLoadingTable ?? false;
+ const completed = !!matchingCompletedTable ?? (exportFullyCompleted || false);
+ const loading = !!matchingLoadingTable ?? false;
const unstarted = !completed && !loading;
return {
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
index 6185105..fc2ac40 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts
@@ -1,15 +1,9 @@
import { useRouter, type NextRouter } from "next/router";
import { ParsedUrlQuery } from "querystring";
import { useEffect, useReducer } from "react";
+import type { ExportTable } from "../../types";
export type GitHubRepository = { namespace: string; repository: string };
-
-export type ExportTable = {
- destinationSchema: string;
- destinationTable: string;
- taskId: string;
- sourceQuery?: string;
- fallbackCreateTableQuery?: string;
-};
+import { getQueryParamAsString, requireKeys } from "../../lib/util";
// NOTE: Multiple tables can have the same taskId, so we track them separately
// in order to not need to redundantly poll the API for each table individually
@@ -74,21 +68,6 @@ const initialState: StepperState = {
debug: null,
};
-const getQueryParamAsString = <T extends string = string>(
- query: ParsedUrlQuery,
- key: string
-): T | null => {
- if (Array.isArray(query[key]) && query[key].length > 0) {
- throw new Error(`expected only one query param but got multiple: ${key}`);
- }
-
- if (!(key in query)) {
- return null;
- }
-
- return query[key] as T;
-};
-
const queryParamParsers: {
[K in keyof StepperState]: (query: ParsedUrlQuery) => StepperState[K];
} = {
@@ -111,19 +90,6 @@ const queryParamParsers: {
debug: (query) => getQueryParamAsString(query, "debug"),
};
-const requireKeys = <T extends Record<string, unknown>>(
- obj: T,
- requiredKeys: (keyof T)[]
-) => {
- const missingKeys = requiredKeys.filter(
- (requiredKey) => !(requiredKey in obj)
- );
-
- if (missingKeys.length > 0) {
- throw new Error("missing required keys: " + missingKeys.join(", "));
- }
-};
-
const stepperStateValidators: {
[K in StepperState["stepperState"]]: (stateFromQuery: StepperState) => void;
} = {
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
index 92e3903..76813cb 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts
@@ -1,3 +1,6 @@
+import { useMemo } from "react";
+import type { ExportTableInput } from "../../types";
+
/**
* List of GitHub table names that we want to import with the Airbyte connector
* into Splitgraph. By default, there are 163 tables available. But we only want
@@ -7,7 +10,8 @@
* Note that Airbyte will still import tables that depend on these tables due
* to foreign keys, and will also import airbyte metaata tables.
*/
-export const relevantGitHubTableNamesForImport = `commits
+export const relevantGitHubTableNamesForImport = `stargazers
+commits
comments
pull_requests
pull_request_stats
@@ -15,6 +19,33 @@ issue_reactions`
.split("\n")
.filter((t) => !!t);
+export const splitgraphTablesToExportToSeafowl = [
+ "stargazers",
+ "stargazers_user",
+];
+
+export const useTablesToExport = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+}: {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+}) => {
+ return useMemo<ExportTableInput[]>(
+ () =>
+ splitgraphTablesToExportToSeafowl.map((tableName) => ({
+ namespace: splitgraphNamespace,
+ repository: splitgraphRepository,
+ table: tableName,
+ })),
+ [
+ splitgraphNamespace,
+ splitgraphRepository,
+ splitgraphTablesToExportToSeafowl,
+ ]
+ );
+};
+
/**
* List of "downstream" GitHub table names that will be imported by default by
* the `airbyte-github` connector, given the list of `relevantGitHubTableNamesForImport`,
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
index 94ebabe..3b26679 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts
@@ -1,3 +1,6 @@
+import { useMemo } from "react";
+import type { ExportQueryInput } from "../../types";
+
/**
* Return a a list of queries to export from Splitgraph to Seafowl, given the
* source repository (where the GitHub data was imported into), and the destination
@@ -161,6 +164,24 @@ CREATE TABLE "${seafowlDestinationSchema}".monthly_issue_stats (
},
];
+export const useQueriesToExport = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+}: {
+ splitgraphNamespace: string;
+ splitgraphRepository: string;
+}) => {
+ return useMemo<ExportQueryInput[]>(
+ () =>
+ makeQueriesToExport({
+ splitgraphSourceRepository: splitgraphRepository,
+ splitgraphSourceNamespace: splitgraphNamespace,
+ seafowlDestinationSchema: `${splitgraphNamespace}/${splitgraphRepository}`,
+ }),
+ [splitgraphRepository, splitgraphNamespace]
+ );
+};
+
/** A generic demo query that can be used to show off Splitgraph */
export const genericDemoQuery = `WITH t (
c_int16_smallint,
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/util.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/util.ts
new file mode 100644
index 0000000..e908061
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/util.ts
@@ -0,0 +1,36 @@
+import { useRouter } from "next/router";
+import type { ParsedUrlQuery } from "querystring";
+
+export const useDebug = () => {
+ const { query } = useRouter();
+
+ return query.debug;
+};
+
+export const getQueryParamAsString = <T extends string = string>(
+ query: ParsedUrlQuery,
+ key: string
+): T | null => {
+ if (Array.isArray(query[key]) && query[key].length > 0) {
+ throw new Error(`expected only one query param but got multiple: ${key}`);
+ }
+
+ if (!(key in query)) {
+ return null;
+ }
+
+ return query[key] as T;
+};
+
+export const requireKeys = <T extends Record<string, unknown>>(
+ obj: T,
+ requiredKeys: (keyof T)[]
+) => {
+ const missingKeys = requiredKeys.filter(
+ (requiredKey) => !(requiredKey in obj)
+ );
+
+ if (missingKeys.length > 0) {
+ throw new Error("missing required keys: " + missingKeys.join(", "));
+ }
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
index 6ce9024..8b962b6 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/types.ts
@@ -10,6 +10,14 @@ export interface TargetSplitgraphRepo {
splitgraphRepository: string;
}
+export type ExportTable = {
+ destinationSchema: string;
+ destinationTable: string;
+ taskId: string;
+ sourceQuery?: string;
+ fallbackCreateTableQuery?: string;
+};
+
export type ExportTableInput = {
namespace: string;
repository: string;
From ab57659109bf3bf57ce48045503f83410db9ece0 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 28 Jun 2023 17:19:32 +0100
Subject: [PATCH 31/36] Add `reduceRows` method to `useSqlPlot` for case where
mapping isn't enough
---
.../RepositoryAnalytics/useSqlPlot.tsx | 34 ++++++++++++++++---
1 file changed, 29 insertions(+), 5 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
index 3a2fc5d..50c7043 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
@@ -15,10 +15,12 @@ import { useMemo } from "react";
export const useSqlPlot = <
RowShape extends UnknownObjectShape,
SqlParams extends object,
- MappedRow extends UnknownObjectShape
+ MappedRow extends UnknownObjectShape,
+ ReducedRow extends UnknownObjectShape
>({
sqlParams,
mapRows,
+ reduceRows,
buildQuery,
makePlotOptions,
isRenderable,
@@ -34,6 +36,11 @@ export const useSqlPlot = <
* to a `Date` object.
*/
mapRows?: (row: RowShape) => MappedRow;
+
+ /**
+ * An optional function to transform the mapped rows into a different aggregation
+ */
+ reduceRows?: (rows: MappedRow[]) => ReducedRow[];
/**
* A builder function that returns a SQL query given a set of parameters, which
* will be the parameters passed as the `sqlParams` parameter.
@@ -43,7 +50,7 @@ export const useSqlPlot = <
* A function to call after receiving the result of the SQL query (and mapping
* its rows if applicable), to create the options given to Observable {@link Plot.plot}
*/
- makePlotOptions: (rows: MappedRow[]) => Plot.PlotOptions;
+ makePlotOptions: (rows: ReducedRow[]) => Plot.PlotOptions;
/**
* A function to call to determine if the chart is renderable. This is helpful
* during server side rendering, when Observable Plot doesn't typically work well,
@@ -73,10 +80,27 @@ export const useSqlPlot = <
);
}, [response, error]);
- const plotOptions = useMemo(() => makePlotOptions(mappedRows), [mappedRows]);
+ const reducedRows = useMemo(() => {
+ if (mappedRows.length === 0) {
+ return [];
+ }
+
+ if (!reduceRows) {
+ return mappedRows as unknown as ReducedRow[];
+ }
+
+ return reduceRows(mappedRows);
+ }, [mappedRows]);
+
+ console.log(JSON.stringify(reducedRows));
+
+ const plotOptions = useMemo(
+ () => makePlotOptions(reducedRows),
+ [reducedRows]
+ );
useEffect(() => {
- if (mappedRows === undefined) {
+ if (reducedRows === undefined) {
return;
}
@@ -90,7 +114,7 @@ export const useSqlPlot = <
}
return () => plot.remove();
- }, [mappedRows]);
+ }, [reducedRows]);
const renderPlot = useCallback(
() => <div ref={containerRef} />,
From 15d14e2b311226cfdd4865a65949f0ee20b4c6aa Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 28 Jun 2023 17:22:05 +0100
Subject: [PATCH 32/36] Add stacked bar chart of issue reactions by month with
bars broken down by reaction type
---
.../components/RepositoryAnalytics/Charts.tsx | 3 +
.../charts/MonthlyIssueStats.tsx | 144 ++++++++++++++++++
2 files changed, 147 insertions(+)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/MonthlyIssueStats.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
index 6eea242..93c8674 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
@@ -6,6 +6,7 @@ import { SqlProvider, makeSeafowlHTTPContext } from "@madatdata/react";
import { useMemo } from "react";
import { StargazersChart } from "./charts/StargazersChart";
+import { MonthlyIssueStatsTable } from "./charts/MonthlyIssueStats";
export interface ChartsProps {
importedRepository: ImportedRepository;
@@ -29,7 +30,9 @@ export const Charts = ({ importedRepository }: ChartsProps) => {
<SqlProvider dataContext={seafowlDataContext}>
<h3>Stargazers</h3>
<StargazersChart {...importedRepository} />
+ <MonthlyIssueStatsTable {...importedRepository} />
</SqlProvider>
+ MonthlyIssueStatsTable
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/MonthlyIssueStats.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/MonthlyIssueStats.tsx
new file mode 100644
index 0000000..f6ce62e
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/MonthlyIssueStats.tsx
@@ -0,0 +1,144 @@
+import * as Plot from "@observablehq/plot";
+import { useSqlPlot } from "../useSqlPlot";
+import type { ImportedRepository, TargetSplitgraphRepo } from "../../../types";
+
+// Assume meta namespace contains both the meta tables, and all imported repositories and tables
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+
+type Reaction =
+ // | "all"
+ | "plus_one"
+ | "minus_one"
+ | "laugh"
+ | "confused"
+ | "heart"
+ | "hooray"
+ | "rocket"
+ | "eyes";
+
+type MappedMonthlyIssueStatsRow = MonthlyIssueStatsRow & {
+ created_at_month: Date;
+};
+
+/**
+ * A stacked bar chart of the number of reactions each month, grouped by reaction type
+ */
+export const MonthlyIssueStats = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+}: ImportedRepository) => {
+ const renderPlot = useSqlPlot({
+ sqlParams: { splitgraphNamespace, splitgraphRepository },
+ buildQuery: monthlyIssueStatsTableQuery,
+ mapRows: (r: MonthlyIssueStatsRow) => ({
+ ...r,
+ created_at_month: new Date(r.created_at_month),
+ }),
+ reduceRows: (rows: MappedMonthlyIssueStatsRow[]) => {
+ const reactions = new Map<
+ Reaction,
+ { created_at_month: Date; count: number }[]
+ >();
+
+ for (const row of rows) {
+ for (const reaction of [
+ "plus_one",
+ "minus_one",
+ "laugh",
+ "confused",
+ "heart",
+ "hooray",
+ "rocket",
+ "eyes",
+ ] as Reaction[]) {
+ if (!reactions.has(reaction)) {
+ reactions.set(reaction, []);
+ }
+
+ reactions.get(reaction)!.push({
+ created_at_month: row.created_at_month,
+ count: (() => {
+ switch (reaction) {
+ case "plus_one":
+ return row.total_plus_ones;
+ case "minus_one":
+ return row.total_minus_ones;
+ case "laugh":
+ return row.total_laughs;
+ case "confused":
+ return row.total_confused;
+ case "heart":
+ return row.total_hearts;
+ case "hooray":
+ return row.total_hoorays;
+ case "rocket":
+ return row.total_rockets;
+ case "eyes":
+ return row.total_eyes;
+ }
+ })(),
+ });
+ }
+ }
+
+ return Array.from(reactions.entries()).flatMap(([reaction, series]) =>
+ series.map((d) => ({ reaction, ...d }))
+ );
+ },
+ isRenderable: (p) => !!p.splitgraphRepository,
+
+ makePlotOptions: (issueStats) => ({
+ y: { grid: true },
+ color: { legend: true },
+ marks: [
+ Plot.rectY(issueStats, {
+ x: "created_at_month",
+ y: "count",
+ interval: "month",
+ fill: "reaction",
+ }),
+ Plot.ruleY([0]),
+ ],
+ }),
+ });
+
+ return renderPlot();
+};
+
+/** Shape of row returned by {@link monthlyIssueStatsTableQuery} */
+export type MonthlyIssueStatsRow = {
+ created_at_month: string;
+ num_issues: number;
+ total_reacts: number;
+ total_plus_ones: number;
+ total_minus_ones: number;
+ total_laughs: number;
+ total_confused: number;
+ total_hearts: number;
+ total_hoorays: number;
+ total_rockets: number;
+ total_eyes: number;
+};
+
+/** Time series of GitHub stargazers for the given repository */
+export const monthlyIssueStatsTableQuery = ({
+ splitgraphNamespace = META_NAMESPACE,
+ splitgraphRepository,
+}: TargetSplitgraphRepo) => {
+ return `SELECT
+ created_at_month,
+ count(issue_number) as num_issues,
+ sum(total_reacts) as total_reacts,
+ sum(no_plus_one) as total_plus_ones,
+ sum(no_minus_one) as total_minus_ones,
+ sum(no_laugh) as total_laughs,
+ sum(no_confused) as total_confused,
+ sum(no_heart) as total_hearts,
+ sum(no_hooray) as total_hoorays,
+ sum(no_rocket) as total_rockets,
+ sum(no_eyes) as total_eyes
+FROM "${splitgraphNamespace}/${splitgraphRepository}"."monthly_issue_stats"
+GROUP BY created_at_month
+ORDER BY created_at_month ASC;`;
+};
From 06c57b9e150a8402a8386a71b618f6ebe91ab8e0 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Wed, 28 Jun 2023 17:22:46 +0100
Subject: [PATCH 33/36] Make repo page have three tabs: tables, queries and
charts
---
.../components/EmbeddedQuery/TabButton.tsx | 10 +-
.../ImportedRepoMetadata.tsx | 5 -
.../[github_repository].tsx | 114 +++++++++++++++++-
3 files changed, 120 insertions(+), 9 deletions(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx
index 7f4c9e9..ae6368f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/TabButton.tsx
@@ -1,10 +1,11 @@
-import type { ButtonHTMLAttributes } from "react";
+import type { ButtonHTMLAttributes, CSSProperties } from "react";
import TabButtonStyle from "./TabButton.module.css";
interface TabButtonProps extends ButtonHTMLAttributes<HTMLButtonElement> {
active: boolean;
onClick: () => void;
+ size?: CSSProperties["fontSize"];
}
export const TabButton = ({
@@ -12,6 +13,7 @@ export const TabButton = ({
onClick,
disabled: alwaysDisabled,
children,
+ size,
...rest
}: React.PropsWithChildren<TabButtonProps>) => {
const className = [
@@ -29,7 +31,11 @@ export const TabButton = ({
className={className}
{...rest}
>
- {children}
+ {typeof size !== "undefined" ? (
+ <span style={{ fontSize: size }}>{children}</span>
+ ) : (
+ children
+ )}
</button>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
index bc05767..81c708d 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/ImportedRepoMetadata.tsx
@@ -46,11 +46,6 @@ export const ImportedRepoMetadata = ({
/>
</li>
</ul>
- <SeafowlEmbeddedQuery
- importedRepository={importedRepository}
- tableName={"stargazers"}
- makeQuery={makeStargazersTableQuery}
- />
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
index 06cec7d..6d8bfa4 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx
@@ -7,7 +7,50 @@ import { useRouter } from "next/router";
import type { ImportedRepository } from "../../types";
import { ImportedRepoMetadata } from "../../components/RepositoryAnalytics/ImportedRepoMetadata";
-import { useMemo } from "react";
+import { useCallback, useMemo } from "react";
+
+import {
+ EmbeddedQueryPreviews,
+ EmbeddedTablePreviews,
+ EmbeddedQueryPreviewHeadingAndDescription,
+ EmbeddedTablePreviewHeadingAndDescription,
+} from "../../components/EmbeddedQuery/EmbeddedPreviews";
+
+import { useQueriesToExport } from "../../lib/config/queries-to-export";
+import { useTablesToExport } from "../../lib/config/github-tables";
+import { getQueryParamAsString } from "../../lib/util";
+import { TabButton } from "../../components/EmbeddedQuery/TabButton";
+
+type ActiveTab = "charts" | "tables" | "queries";
+
+const useActiveTab = (defaultTab: ActiveTab) => {
+ const router = useRouter();
+
+ const activeTab =
+ getQueryParamAsString<ActiveTab>(router.query, "activeTab") ?? defaultTab;
+
+ const switchTab = useCallback(
+ (nextTab: ActiveTab) => {
+ if (nextTab === activeTab) {
+ return;
+ }
+
+ return router.push({
+ pathname: router.pathname,
+ query: {
+ ...router.query,
+ activeTab: nextTab,
+ },
+ });
+ },
+ [router.query]
+ );
+
+ return {
+ activeTab,
+ switchTab,
+ };
+};
const useImportedRepoFromURL = () => {
const { query } = useRouter();
@@ -43,12 +86,79 @@ const useImportedRepoFromURL = () => {
const RepositoryAnalyticsPage = () => {
const importedRepository = useImportedRepoFromURL();
+ const tablesToExport = useTablesToExport(importedRepository);
+ const queriesToExport = useQueriesToExport(importedRepository);
+
+ const { activeTab, switchTab } = useActiveTab("charts");
+
return (
<BaseLayout sidebar={<Sidebar />}>
<ImportedRepoMetadata importedRepository={importedRepository} />
- <Charts importedRepository={importedRepository} />
+ <PageTabs activeTab={activeTab} switchTab={switchTab} />
+
+ <TabPane active={activeTab === "charts"}>
+ <Charts importedRepository={importedRepository} />
+ </TabPane>
+
+ <TabPane active={activeTab === "tables"}>
+ <EmbeddedTablePreviewHeadingAndDescription exportComplete={true} />
+ <EmbeddedTablePreviews
+ useLoadingOrCompleted={() => ({ loading: false, completed: true })}
+ tablesToExport={tablesToExport}
+ splitgraphRepository={importedRepository.splitgraphRepository}
+ splitgraphNamespace={importedRepository.splitgraphNamespace}
+ />
+ </TabPane>
+
+ <TabPane active={activeTab === "queries"}>
+ <EmbeddedQueryPreviewHeadingAndDescription exportComplete={true} />
+ <EmbeddedQueryPreviews
+ useLoadingOrCompleted={() => ({ loading: false, completed: true })}
+ queriesToExport={queriesToExport}
+ splitgraphRepository={importedRepository.splitgraphRepository}
+ splitgraphNamespace={importedRepository.splitgraphNamespace}
+ />
+ </TabPane>
</BaseLayout>
);
};
+const TabPane = ({ active, children }: { active: boolean; children: any }) => {
+ return <div style={{ display: active ? "block" : "none" }}>{children}</div>;
+};
+
+const PageTabs = ({
+ activeTab,
+ switchTab,
+}: ReturnType<typeof useActiveTab>) => {
+ return (
+ <div>
+ <TabButton
+ active={activeTab === "charts"}
+ onClick={() => switchTab("charts")}
+ style={{ marginRight: "1rem" }}
+ size="1.5rem"
+ >
+ Charts
+ </TabButton>
+ <TabButton
+ active={activeTab === "tables"}
+ onClick={() => switchTab("tables")}
+ style={{ marginRight: "1rem" }}
+ size="1.5rem"
+ >
+ Raw Tables
+ </TabButton>
+ <TabButton
+ active={activeTab === "queries"}
+ onClick={() => switchTab("queries")}
+ style={{ marginRight: "1rem" }}
+ size="1.5rem"
+ >
+ Raw Queries
+ </TabButton>
+ </div>
+ );
+};
+
export default RepositoryAnalyticsPage;
From 93022b16ef5afc3fc80cac62b2e4a727bc80fe3d Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Thu, 29 Jun 2023 02:10:59 +0100
Subject: [PATCH 34/36] Rename chart to IssueReactsByMonth
---
.../components/RepositoryAnalytics/Charts.tsx | 6 +--
...yIssueStats.tsx => IssueReactsByMonth.tsx} | 41 +++++++++++++++----
2 files changed, 37 insertions(+), 10 deletions(-)
rename examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/{MonthlyIssueStats.tsx => IssueReactsByMonth.tsx} (78%)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
index 93c8674..98b2a27 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
@@ -6,7 +6,7 @@ import { SqlProvider, makeSeafowlHTTPContext } from "@madatdata/react";
import { useMemo } from "react";
import { StargazersChart } from "./charts/StargazersChart";
-import { MonthlyIssueStatsTable } from "./charts/MonthlyIssueStats";
+import { IssueReactsByMonth } from "./charts/IssueReactsByMonth";
export interface ChartsProps {
importedRepository: ImportedRepository;
@@ -30,9 +30,9 @@ export const Charts = ({ importedRepository }: ChartsProps) => {
<SqlProvider dataContext={seafowlDataContext}>
<h3>Stargazers</h3>
<StargazersChart {...importedRepository} />
- <MonthlyIssueStatsTable {...importedRepository} />
+ <h3>Issue Reacts by Month</h3>
+ <IssueReactsByMonth {...importedRepository} />
</SqlProvider>
- MonthlyIssueStatsTable
</div>
);
};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/MonthlyIssueStats.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/IssueReactsByMonth.tsx
similarity index 78%
rename from examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/MonthlyIssueStats.tsx
rename to examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/IssueReactsByMonth.tsx
index f6ce62e..408457a 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/MonthlyIssueStats.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/IssueReactsByMonth.tsx
@@ -17,25 +17,25 @@ type Reaction =
| "rocket"
| "eyes";
-type MappedMonthlyIssueStatsRow = MonthlyIssueStatsRow & {
+type MappedIssueReactsByMonthRow = IssueReactsByMonthRow & {
created_at_month: Date;
};
/**
* A stacked bar chart of the number of reactions each month, grouped by reaction type
*/
-export const MonthlyIssueStats = ({
+export const IssueReactsByMonth = ({
splitgraphNamespace,
splitgraphRepository,
}: ImportedRepository) => {
const renderPlot = useSqlPlot({
sqlParams: { splitgraphNamespace, splitgraphRepository },
buildQuery: monthlyIssueStatsTableQuery,
- mapRows: (r: MonthlyIssueStatsRow) => ({
+ mapRows: (r: IssueReactsByMonthRow) => ({
...r,
created_at_month: new Date(r.created_at_month),
}),
- reduceRows: (rows: MappedMonthlyIssueStatsRow[]) => {
+ reduceRows: (rows: MappedIssueReactsByMonthRow[]) => {
const reactions = new Map<
Reaction,
{ created_at_month: Date; count: number }[]
@@ -89,14 +89,41 @@ export const MonthlyIssueStats = ({
isRenderable: (p) => !!p.splitgraphRepository,
makePlotOptions: (issueStats) => ({
- y: { grid: true },
- color: { legend: true },
+ y: { grid: true, label: "Number of Reactions" },
+ x: {
+ label: "Month",
+ },
+ color: {
+ legend: true,
+ label: "Reaction",
+ tickFormat: (reaction) => {
+ switch (reaction) {
+ case "plus_one":
+ return "👍 plus_one";
+ case "minus_one":
+ return "👎 minus_one";
+ case "laugh":
+ return "😄 laugh";
+ case "confused":
+ return "😕 confused";
+ case "heart":
+ return "❤️ heart";
+ case "hooray":
+ return "🎉 hooray";
+ case "rocket":
+ return "🚀 rocket";
+ case "eyes":
+ return "👀 eyes";
+ }
+ },
+ },
marks: [
Plot.rectY(issueStats, {
x: "created_at_month",
y: "count",
interval: "month",
fill: "reaction",
+ tip: true,
}),
Plot.ruleY([0]),
],
@@ -107,7 +134,7 @@ export const MonthlyIssueStats = ({
};
/** Shape of row returned by {@link monthlyIssueStatsTableQuery} */
-export type MonthlyIssueStatsRow = {
+export type IssueReactsByMonthRow = {
created_at_month: string;
num_issues: number;
total_reacts: number;
From afa4c070a91f2d852aed9b38a9a13b6480a1df71 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Thu, 29 Jun 2023 02:11:30 +0100
Subject: [PATCH 35/36] Bump GitHub import page size to 100, start date to
2023-01-01
---
.../pages/api/start-import-from-github.ts | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
index 5f7c079..47bbea3 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts
@@ -94,7 +94,8 @@ const startImport = async ({
},
params: {
repository: githubSourceRepository,
- start_date: githubStartDate ?? "2020-01-01T00:00:00Z",
+ start_date: githubStartDate ?? "2023-01-01T00:00:00Z",
+ page_size_for_large_streams: 100,
},
},
{
From 06d94f71abf460b7c2002072303c026fe4773e90 Mon Sep 17 00:00:00 2001
From: Miles Richardson <miles@splitgraph.com>
Date: Thu, 29 Jun 2023 03:45:13 +0100
Subject: [PATCH 36/36] Add scatter plot of user comment length vs. lines of
code
---
.../components/RepositoryAnalytics/Charts.tsx | 3 +
.../charts/IssueReactsByMonth.tsx | 1 -
.../charts/UserCodeVsComment.tsx | 102 ++++++++++++++++++
.../RepositoryAnalytics/useSqlPlot.tsx | 2 -
4 files changed, 105 insertions(+), 3 deletions(-)
create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/UserCodeVsComment.tsx
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
index 98b2a27..afc76c9 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx
@@ -7,6 +7,7 @@ import { useMemo } from "react";
import { StargazersChart } from "./charts/StargazersChart";
import { IssueReactsByMonth } from "./charts/IssueReactsByMonth";
+import { UserCodeVsComment } from "./charts/UserCodeVsComment";
export interface ChartsProps {
importedRepository: ImportedRepository;
@@ -32,6 +33,8 @@ export const Charts = ({ importedRepository }: ChartsProps) => {
<StargazersChart {...importedRepository} />
<h3>Issue Reacts by Month</h3>
<IssueReactsByMonth {...importedRepository} />
+ <h3>Code vs. Comment Length</h3>
+ <UserCodeVsComment {...importedRepository} />
</SqlProvider>
</div>
);
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/IssueReactsByMonth.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/IssueReactsByMonth.tsx
index 408457a..a99d27f 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/IssueReactsByMonth.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/IssueReactsByMonth.tsx
@@ -7,7 +7,6 @@ const META_NAMESPACE =
process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
type Reaction =
- // | "all"
| "plus_one"
| "minus_one"
| "laugh"
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/UserCodeVsComment.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/UserCodeVsComment.tsx
new file mode 100644
index 0000000..f6b88a1
--- /dev/null
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/charts/UserCodeVsComment.tsx
@@ -0,0 +1,102 @@
+import * as Plot from "@observablehq/plot";
+import { useSqlPlot } from "../useSqlPlot";
+import type { ImportedRepository, TargetSplitgraphRepo } from "../../../types";
+
+// Assume meta namespace contains both the meta tables, and all imported repositories and tables
+const META_NAMESPACE =
+ process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE;
+
+type CommentLengthRow = {
+ username: string;
+ comment_length: number;
+ net_lines_added: number;
+ total_lines_added: number;
+ total_lines_deleted: number;
+};
+
+const sum = (arr: number[]) => arr.reduce((a, b) => a + b, 0);
+const mean = (arr: number[]) => sum(arr) / arr.length;
+
+/**
+ * A scatter plot of user comment length vs. lines of code
+ */
+export const UserCodeVsComment = ({
+ splitgraphNamespace,
+ splitgraphRepository,
+}: ImportedRepository) => {
+ const renderPlot = useSqlPlot({
+ sqlParams: { splitgraphNamespace, splitgraphRepository },
+ buildQuery: userStatsQuery,
+ mapRows: (r: UserStatsRow) =>
+ ({
+ username: r.username,
+ comment_length: r.total_comment_length,
+ total_lines_added: r.total_lines_added,
+ total_lines_deleted: r.total_lines_deleted,
+ net_lines_added: r.total_lines_added - r.total_lines_deleted,
+ } as CommentLengthRow),
+ isRenderable: (p) => !!p.splitgraphRepository,
+ reduceRows: (rows: CommentLengthRow[]) =>
+ rows.filter((r) => r.username && !r.username.endsWith("[bot]")),
+
+ makePlotOptions: (userStats: CommentLengthRow[]) => ({
+ y: {
+ label: "Length of Comments",
+ type: "symlog",
+ constant: mean(userStats.map((u) => u.comment_length)),
+ },
+ x: {
+ label: "Lines of Code",
+ type: "symlog",
+ constant: mean(userStats.map((u) => u.total_lines_added)),
+ },
+ color: {
+ scheme: "Turbo",
+ },
+ marks: [
+ Plot.dot(userStats, {
+ x: "comment_length",
+ y: "total_lines_added",
+ stroke: "username",
+ fill: "username",
+ tip: true,
+ }),
+ Plot.ruleY([0]),
+ ],
+ }),
+ });
+
+ return renderPlot();
+};
+
+/** Shape of row returned by {@link userStatsQuery} */
+export type UserStatsRow = {
+ username: string;
+ total_commits: number;
+ total_pull_request_comments: number;
+ total_issue_comments: number;
+ total_comment_length: number;
+ total_merged_pull_requests: number;
+ total_pull_requests: number;
+ total_lines_added: number;
+ total_lines_deleted: number;
+};
+
+/** Time series of GitHub stargazers for the given repository */
+export const userStatsQuery = ({
+ splitgraphNamespace = META_NAMESPACE,
+ splitgraphRepository,
+}: TargetSplitgraphRepo) => {
+ return `SELECT
+ username,
+ sum(no_commits) as total_commits,
+ sum(no_pull_request_comments) as total_pull_request_comments,
+ sum(no_issue_comments) as total_issue_comments,
+ sum(total_comment_length) as total_comment_length,
+ sum(merged_pull_requests) as total_merged_pull_requests,
+ sum(total_pull_requests) as total_pull_requests,
+ sum(lines_added) as total_lines_added,
+ sum(lines_deleted) as total_lines_deleted
+FROM "${splitgraphNamespace}/${splitgraphRepository}"."monthly_user_stats"
+GROUP BY username;`;
+};
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
index 50c7043..3e93b3d 100644
--- a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
+++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/useSqlPlot.tsx
@@ -92,8 +92,6 @@ export const useSqlPlot = <
return reduceRows(mappedRows);
}, [mappedRows]);
- console.log(JSON.stringify(reducedRows));
-
const plotOptions = useMemo(
() => makePlotOptions(reducedRows),
[reducedRows]