Merge pull request #142 from supabase-community/feat/pg_dump-in-the-browser

feat: use pg_dump to download the database schema and data

Author: jgoux

apps/web/components/sidebar/database-menu-item.tsx

@@ -41,7 +41,7 @@ import { useDatabaseUpdateMutation } from '~/data/databases/database-update-muta
 import { useIntegrationQuery } from '~/data/integrations/integration-query'
 import { MergedDatabase } from '~/data/merged-databases/merged-databases'
 import { useQueryEvent } from '~/lib/hooks'
-import { downloadFile, getDeployUrl, getOauthUrl, titleToKebabCase } from '~/lib/util'
+import { downloadFileFromUrl, getDeployUrl, getOauthUrl, titleToKebabCase } from '~/lib/util'
 import { cn } from '~/lib/utils'
 
 export type DatabaseMenuItemProps = {
@@ -319,13 +319,23 @@ export function DatabaseMenuItem({ database, isActive, onClick }: DatabaseMenuIt
                       throw new Error('dbManager is not available')
                     }
 
-                    const db = await dbManager.getDbInstance(database.id)
-                    const dumpBlob = await db.dumpDataDir()
+                    // Ensure the db worker is ready
+                    await dbManager.getDbInstance(database.id)
 
-                    const fileName = `${titleToKebabCase(database.name ?? 'My Database')}-${Date.now()}`
-                    const file = new File([dumpBlob], fileName, { type: dumpBlob.type })
+                    const bc = new BroadcastChannel(`${database.id}:pg-dump`)
 
-                    downloadFile(file)
+                    bc.addEventListener('message', (event) => {
+                      if (event.data.action === 'dump-result') {
+                        downloadFileFromUrl(event.data.url, event.data.filename)
+                        bc.close()
+                        setIsPopoverOpen(false)
+                      }
+                    })
+
+                    bc.postMessage({
+                      action: 'execute-dump',
+                      filename: `${titleToKebabCase(database.name ?? 'My Database')}-${Date.now()}.sql`,
+                    })
                   }}
                 >
                   <Download

apps/web/lib/db/index.ts

@@ -4,6 +4,7 @@ import { PGliteWorker } from '@electric-sql/pglite/worker'
 import { Message as AiMessage, ToolInvocation } from 'ai'
 import { codeBlock } from 'common-tags'
 import { nanoid } from 'nanoid'
+import { downloadFileFromUrl } from '../util'
 
 export type Database = {
   id: string
@@ -47,7 +48,7 @@ export class DbManager {
   /**
    * Creates a PGlite instance that runs in a web worker
    */
-  static async createPGlite(options?: PGliteOptions): Promise<PGliteInterface> {
+  static async createPGlite(options?: PGliteOptions & { id?: string }) {
     if (typeof window === 'undefined') {
       throw new Error('PGlite worker instances are only available in the browser')
     }
@@ -59,7 +60,7 @@ export class DbManager {
       new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' }),
       {
         // Opt out of PGlite worker leader election / shared DBs
-        id: nanoid(),
+        id: options?.id ?? nanoid(),
         ...options,
       }
     )
@@ -274,7 +275,7 @@ export class DbManager {
     return metaDb.sql`insert into databases (id, name, created_at, is_hidden) values ${join(values, ',')} on conflict (id) do nothing`
   }
 
-  async getDbInstance(id: string, loadDataDir?: Blob | File) {
+  async getDbInstance(id: string, loadDataDir?: Blob | File): Promise<PGliteInterface> {
     const openDatabasePromise = this.databaseConnections.get(id)
 
     if (openDatabasePromise) {
@@ -292,7 +293,7 @@ export class DbManager {
 
       await this.handleUnsupportedPGVersion(dbPath)
 
-      const db = await DbManager.createPGlite({ dataDir: `idb://${dbPath}`, loadDataDir })
+      const db = await DbManager.createPGlite({ dataDir: `idb://${dbPath}`, loadDataDir, id })
       await runMigrations(db, migrations)
 
       return db

apps/web/lib/db/worker.ts

@@ -1,10 +1,12 @@
 import { PGlite } from '@electric-sql/pglite'
 import { vector } from '@electric-sql/pglite/vector'
 import { PGliteWorkerOptions, worker } from '@electric-sql/pglite/worker'
+import { pgDump } from '@electric-sql/pglite-tools/pg_dump'
+import { codeBlock } from 'common-tags'
 
 worker({
   async init(options: PGliteWorkerOptions) {
-    return new PGlite({
+    const db = new PGlite({
       ...options,
       extensions: {
         ...options.extensions,
@@ -13,5 +15,42 @@ worker({
         vector,
       },
     })
+
+    const bc = new BroadcastChannel(`${options.id}:pg-dump`)
+
+    bc.addEventListener('message', async (event) => {
+      if (event.data.action === 'execute-dump') {
+        let dump = await pgDump({ pg: db })
+        // clear prepared statements
+        await db.query('deallocate all')
+        let dumpContent = await dump.text()
+        // patch for old PGlite versions where the vector extension was not included in the dump
+        if (!dumpContent.includes('CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public;')) {
+          const insertPoint = 'ALTER SCHEMA meta OWNER TO postgres;'
+          const insertPointIndex = dumpContent.indexOf(insertPoint) + insertPoint.length
+          dumpContent = codeBlock`
+            ${dumpContent.slice(0, insertPointIndex)}
+
+            --
+            -- Name: vector; Type: EXTENSION; Schema: -; Owner: -
+            --
+            
+            CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public;
+
+            ${dumpContent.slice(insertPointIndex)}`
+
+          // Create new blob with modified content
+          dump = new File([dumpContent], event.data.filename)
+        }
+        const url = URL.createObjectURL(dump)
+        bc.postMessage({
+          action: 'dump-result',
+          filename: event.data.filename,
+          url,
+        })
+      }
+    })
+
+    return db
   },
 })

apps/web/lib/util.ts

@@ -7,18 +7,25 @@ export const currentDomainUrl = process.env.NEXT_PUBLIC_CURRENT_DOMAIN!
 export const currentDomainHostname = new URL(currentDomainUrl).hostname
 
 /**
- * Programmatically download a `File`.
+ * Programmatically download a `File` from a given URL.
  */
-export function downloadFile(file: File) {
-  const url = URL.createObjectURL(file)
+export function downloadFileFromUrl(url: string, filename: string) {
   const a = document.createElement('a')
   a.href = url
-  a.download = file.name
+  a.download = filename
   document.body.appendChild(a)
   a.click()
   a.remove()
 }
 
+/**
+ * Programmatically download a `File`.
+ */
+export function downloadFile(file: File) {
+  const url = URL.createObjectURL(file)
+  downloadFileFromUrl(url, file.name)
+}
+
 export async function requestFileUpload() {
   return new Promise<File>((resolve, reject) => {
     // Create a temporary file input element

apps/web/package.json

@@ -15,7 +15,8 @@
     "@ai-sdk/openai": "^1.0.4",
     "@dagrejs/dagre": "^1.1.2",
     "@database.build/deploy": "*",
-    "@electric-sql/pglite": "^0.2.9",
+    "@electric-sql/pglite": "^0.2.14",
+    "@electric-sql/pglite-tools": "^0.2.2",
     "@gregnr/postgres-meta": "^0.82.0-dev.2",
     "@hookform/resolvers": "^3.9.1",
     "@monaco-editor/react": "^4.6.0",