feat: Add S3 support for collections

.env.example

@@ -32,6 +32,10 @@ ASSETS=local
 ASSETS_BASE_URL=http://localhost:3000/assets
 ASSETS_S3_URI=
 
+# How collections are stored and loaded (local or s3)
+COLLECTIONS=local
+COLLECTIONS_BASE_URL=
+
 # By default world data is stored in a local sqlite database in the world folder
 # Optionally set this to a postgres uri to store remotely, eg `postgres://username:password@host:port/database`
 DB_URI=local

src/server/CollectionsLocal.js

@@ -0,0 +1,59 @@
+import fs from 'fs-extra'
+import path from 'path'
+import { importApp } from '../core/extras/appTools'
+import { assets } from './assets'
+
+export class CollectionsLocal {
+  constructor() {
+    this.list = []
+    this.blueprints = new Set()
+  }
+
+  async init({ rootDir, worldDir }) {
+    console.log('[collections] initializing from local filesystem')
+    this.dir = path.join(worldDir, '/collections')
+    // ensure collections directory exists
+    await fs.ensureDir(this.dir)
+    // copy over built-in collections
+    await fs.copy(path.join(rootDir, 'src/world/collections'), this.dir)
+    // ensure all collections apps are installed
+    let folderNames = fs.readdirSync(this.dir)
+    folderNames.sort((a, b) => {
+      // keep "default" first then sort alphabetically
+      if (a === 'default') return -1
+      if (b === 'default') return 1
+      return a.localeCompare(b)
+    })
+    for (const folderName of folderNames) {
+      const folderPath = path.join(this.dir, folderName)
+      const stats = fs.statSync(folderPath)
+      if (!stats.isDirectory()) continue
+      const manifestPath = path.join(folderPath, 'manifest.json')
+      if (!fs.existsSync(manifestPath)) continue
+      const manifest = fs.readJsonSync(manifestPath)
+      const blueprints = []
+      for (const appFilename of manifest.apps) {
+        const appPath = path.join(folderPath, appFilename)
+        const appBuffer = fs.readFileSync(appPath)
+        const appFile = new File([appBuffer], appFilename, {
+          type: 'application/octet-stream',
+        })
+        const app = await importApp(appFile)
+        for (const asset of app.assets) {
+          // const file = asset.file
+          // const assetFilename = asset.url.slice(8) // remove 'asset://' prefix
+          await assets.upload(asset.file)
+        }
+        blueprints.push(app.blueprint)
+      }
+      this.list.push({
+        id: folderName,
+        name: manifest.name,
+        blueprints,
+      })
+      for (const blueprint of blueprints) {
+        this.blueprints.add(blueprint)
+      }
+    }
+  }
+}

src/server/CollectionsS3.js

@@ -0,0 +1,110 @@
+import { importApp } from '../core/extras/appTools'
+import { assets } from './assets'
+
+export class CollectionsS3 {
+  constructor() {
+    this.baseUrl = process.env.COLLECTIONS_BASE_URL
+    if (!this.baseUrl) {
+      throw new Error('COLLECTIONS_BASE_URL environment variable is required')
+    }
+
+    // Ensure baseUrl ends with /
+    if (!this.baseUrl.endsWith('/')) {
+      this.baseUrl += '/'
+    }
+  }
+
+  async init({ rootDir, worldDir }) {
+    console.log('[collections] initializing from CloudFront')
+    this.list = []
+    this.blueprints = new Set()
+
+    // List all collection folders from CloudFront
+    const collectionFolders = await this.listCollectionFolders()
+
+    for (const folderName of collectionFolders) {
+      try {
+        const collection = await this.loadCollection(folderName)
+        if (collection) {
+          this.list.push(collection)
+          for (const blueprint of collection.blueprints) {
+            this.blueprints.add(blueprint)
+          }
+        }
+      } catch (error) {
+        console.error(`[collections] Failed to load collection ${folderName}:`, error)
+      }
+    }
+  }
+
+  async listCollectionFolders() {
+    // For now, we'll use a predefined list or try to discover collections
+    // This could be enhanced to read from a collections index file
+    const commonCollections = ['default']
+
+    // Try to discover collections by checking for manifest.json
+    const discoveredCollections = []
+    for (const collectionName of commonCollections) {
+      try {
+        const manifestUrl = `${this.baseUrl}${collectionName}/manifest.json`
+        const response = await fetch(manifestUrl)
+        if (response.ok) {
+          discoveredCollections.push(collectionName)
+        }
+      } catch (error) {
+        // Collection doesn't exist, skip
+      }
+    }
+
+    return discoveredCollections
+  }
+
+  async loadCollection(folderName) {
+    try {
+      // Load manifest.json from CloudFront
+      const manifestUrl = `${this.baseUrl}${folderName}/manifest.json`
+      const manifestResponse = await fetch(manifestUrl)
+
+      if (!manifestResponse.ok) {
+        throw new Error(`Failed to fetch manifest: ${manifestResponse.status}`)
+      }
+
+      const manifest = await manifestResponse.json()
+
+      const blueprints = []
+
+      // Load each app file from CloudFront
+      for (const appFilename of manifest.apps) {
+        const appUrl = `${this.baseUrl}${folderName}/${appFilename}`
+        const appResponse = await fetch(appUrl)
+
+        if (!appResponse.ok) {
+          throw new Error(`Failed to fetch app ${appFilename}: ${appResponse.status}`)
+        }
+
+        const appBuffer = await appResponse.arrayBuffer()
+        const appFile = new File([appBuffer], appFilename, {
+          type: 'application/octet-stream',
+        })
+
+        const app = await importApp(appFile)
+
+        // Upload assets to the main assets system
+        for (const asset of app.assets) {
+          await assets.upload(asset.file)
+        }
+
+        blueprints.push(app.blueprint)
+      }
+
+      return {
+        id: folderName,
+        name: manifest.name,
+        blueprints,
+      }
+    } catch (error) {
+      console.error(`Failed to load collection ${folderName}:`, error)
+      return null
+    }
+  }
+}

src/server/collections.js

@@ -1,61 +1,4 @@
-import fs from 'fs-extra'
-import path from 'path'
-import { importApp } from '../core/extras/appTools'
-import { assets } from './assets'
+import { CollectionsS3 } from './CollectionsS3'
+import { CollectionsLocal } from './CollectionsLocal'
 
-class Collections {
-  constructor() {
-    this.list = []
-    this.blueprints = new Set()
-  }
-
-  async init({ rootDir, worldDir }) {
-    console.log('[collections] initializing')
-    this.dir = path.join(worldDir, '/collections')
-    // ensure collections directory exists
-    await fs.ensureDir(this.dir)
-    // copy over built-in collections
-    await fs.copy(path.join(rootDir, 'src/world/collections'), this.dir)
-    // ensure all collections apps are installed
-    let folderNames = fs.readdirSync(this.dir)
-    folderNames.sort((a, b) => {
-      // keep "default" first then sort alphabetically
-      if (a === 'default') return -1
-      if (b === 'default') return 1
-      return a.localeCompare(b)
-    })
-    for (const folderName of folderNames) {
-      const folderPath = path.join(this.dir, folderName)
-      const stats = fs.statSync(folderPath)
-      if (!stats.isDirectory()) continue
-      const manifestPath = path.join(folderPath, 'manifest.json')
-      if (!fs.existsSync(manifestPath)) continue
-      const manifest = fs.readJsonSync(manifestPath)
-      const blueprints = []
-      for (const appFilename of manifest.apps) {
-        const appPath = path.join(folderPath, appFilename)
-        const appBuffer = fs.readFileSync(appPath)
-        const appFile = new File([appBuffer], appFilename, {
-          type: 'application/octet-stream',
-        })
-        const app = await importApp(appFile)
-        for (const asset of app.assets) {
-          // const file = asset.file
-          // const assetFilename = asset.url.slice(8) // remove 'asset://' prefix
-          await assets.upload(asset.file)
-        }
-        blueprints.push(app.blueprint)
-      }
-      this.list.push({
-        id: folderName,
-        name: manifest.name,
-        blueprints,
-      })
-      for (const blueprint of blueprints) {
-        this.blueprints.add(blueprint)
-      }
-    }
-  }
-}
-
-export const collections = new Collections()
+export const collections = process.env.COLLECTIONS === 's3' ? new CollectionsS3() : new CollectionsLocal()

src/server/index.js

@@ -20,15 +20,12 @@ import { cleaner } from './cleaner'
 
 const rootDir = path.join(__dirname, '../')
 const worldDir = path.join(rootDir, process.env.WORLD)
-const port = process.env.PORT
+const port = process.env.PORT || 3000
 
 // check envs
 if (!process.env.WORLD) {
   throw new Error('[envs] WORLD not set')
 }
-if (!process.env.PORT) {
-  throw new Error('[envs] PORT not set')
-}
 if (!process.env.JWT_SECRET) {
   throw new Error('[envs] JWT_SECRET not set')
 }
@@ -59,6 +56,9 @@ if (!process.env.ASSETS_BASE_URL) {
 if (process.env.ASSETS === 's3' && !process.env.ASSETS_S3_URI) {
   throw new Error(`[envs] ASSETS_S3_URI must be set when using ASSETS=s3`)
 }
+if (process.env.COLLECTIONS === 's3' && !process.env.COLLECTIONS_BASE_URL) {
+  throw new Error(`[envs] COLLECTIONS_BASE_URL must be set when using COLLECTIONS=s3`)
+}
 
 const fastify = Fastify({ logger: { level: 'error' } })