From 1ecee60655c408195d84aa5a59e69a80c92bebde Mon Sep 17 00:00:00 2001 From: DevStarlight Date: Thu, 18 Sep 2025 01:39:52 +0200 Subject: [PATCH 1/5] feat: Add S3 support for collections - Create CollectionsS3 class to load collections from S3 - Create CollectionsLocal class for local filesystem compatibility - Add factory pattern to select between S3 and local collections - Add environment variables COLLECTIONS and COLLECTIONS_S3_URI - Update .env.example with new collection configuration options - Maintain backward compatibility with existing local collections system This allows collections to be stored and served from S3 similar to how assets already work, providing better scalability and distribution. --- .env.example | 4 + src/server/CollectionsLocal.js | 59 +++++++++ src/server/CollectionsS3.js | 211 +++++++++++++++++++++++++++++++++ src/server/collections.js | 63 +--------- src/server/index.js | 6 + 5 files changed, 283 insertions(+), 60 deletions(-) create mode 100644 src/server/CollectionsLocal.js create mode 100644 src/server/CollectionsS3.js diff --git a/.env.example b/.env.example index ac6ae2c1..0ac7bc40 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,10 @@ ASSETS=local ASSETS_BASE_URL=http://localhost:3000/assets ASSETS_S3_URI= +# How collections are stored and loaded (local or s3) +COLLECTIONS=local +COLLECTIONS_S3_URI= + # By default world data is stored in a local sqlite database in the world folder # Optionally set this to a postgres uri to store remotely, eg `postgres://username:password@host:port/database` DB_URI=local diff --git a/src/server/CollectionsLocal.js b/src/server/CollectionsLocal.js new file mode 100644 index 00000000..0a5a72b4 --- /dev/null +++ b/src/server/CollectionsLocal.js @@ -0,0 +1,59 @@ +import fs from 'fs-extra' +import path from 'path' +import { importApp } from '../core/extras/appTools' +import { assets } from './assets' + +export class CollectionsLocal { + constructor() { + this.list = [] + this.blueprints = new Set() + } + + async init({ rootDir, worldDir }) { + console.log('[collections] initializing from local filesystem') + this.dir = path.join(worldDir, '/collections') + // ensure collections directory exists + await fs.ensureDir(this.dir) + // copy over built-in collections + await fs.copy(path.join(rootDir, 'src/world/collections'), this.dir) + // ensure all collections apps are installed + let folderNames = fs.readdirSync(this.dir) + folderNames.sort((a, b) => { + // keep "default" first then sort alphabetically + if (a === 'default') return -1 + if (b === 'default') return 1 + return a.localeCompare(b) + }) + for (const folderName of folderNames) { + const folderPath = path.join(this.dir, folderName) + const stats = fs.statSync(folderPath) + if (!stats.isDirectory()) continue + const manifestPath = path.join(folderPath, 'manifest.json') + if (!fs.existsSync(manifestPath)) continue + const manifest = fs.readJsonSync(manifestPath) + const blueprints = [] + for (const appFilename of manifest.apps) { + const appPath = path.join(folderPath, appFilename) + const appBuffer = fs.readFileSync(appPath) + const appFile = new File([appBuffer], appFilename, { + type: 'application/octet-stream', + }) + const app = await importApp(appFile) + for (const asset of app.assets) { + // const file = asset.file + // const assetFilename = asset.url.slice(8) // remove 'asset://' prefix + await assets.upload(asset.file) + } + blueprints.push(app.blueprint) + } + this.list.push({ + id: folderName, + name: manifest.name, + blueprints, + }) + for (const blueprint of blueprints) { + this.blueprints.add(blueprint) + } + } + } +} diff --git a/src/server/CollectionsS3.js b/src/server/CollectionsS3.js new file mode 100644 index 00000000..d34f7603 --- /dev/null +++ b/src/server/CollectionsS3.js @@ -0,0 +1,211 @@ +import { + GetObjectCommand, + ListObjectsV2Command, + S3Client, +} from '@aws-sdk/client-s3' +import { importApp } from '../core/extras/appTools' +import { assets } from './assets' + +export class CollectionsS3 { + constructor() { + // Parse S3 URI: s3://access_key:secret_key@endpoint/bucket/prefix + // or for AWS: s3://access_key:secret_key@bucket.s3.region.amazonaws.com/prefix + // or simple AWS: s3://access_key:secret_key@bucket/prefix (defaults to us-east-1) + const uri = process.env.COLLECTIONS_S3_URI + if (!uri) { + throw new Error('COLLECTIONS_S3_URI environment variable is required') + } + + const config = this.parseURI(uri) + this.bucketName = config.bucket + this.prefix = config.prefix || 'collections/' + + // Initialize S3 client + this.client = new S3Client({ + region: config.region, + endpoint: config.endpoint, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + forcePathStyle: config.forcePathStyle, + }) + } + + parseURI(uri) { + try { + // Remove s3:// prefix + if (!uri.startsWith('s3://')) { + throw new Error('URI must start with s3://') + } + uri = uri.slice(5) + + // Split by @ to separate credentials from endpoint/bucket + const [credentials, endpointBucket] = uri.split('@') + if (!credentials || !endpointBucket) { + throw new Error('Invalid S3 URI format') + } + + const [accessKeyId, secretAccessKey] = credentials.split(':') + if (!accessKeyId || !secretAccessKey) { + throw new Error('Missing access key or secret key') + } + + // Parse endpoint/bucket/prefix + const parts = endpointBucket.split('/') + const endpointBucketPart = parts[0] + const prefix = parts.slice(1).join('/') + + // Determine if this is AWS or custom endpoint + let region = 'us-east-1' + let endpoint = undefined + let forcePathStyle = false + let bucket = endpointBucketPart + + if (endpointBucketPart.includes('.s3.') && endpointBucketPart.includes('.amazonaws.com')) { + // AWS format: bucket.s3.region.amazonaws.com + const match = endpointBucketPart.match(/^(.+)\.s3\.(.+)\.amazonaws\.com$/) + if (match) { + bucket = match[1] + region = match[2] + } + } else if (endpointBucketPart.includes('://')) { + // Custom endpoint format: https://endpoint/bucket + const url = new URL(endpointBucketPart) + endpoint = `${url.protocol}//${url.host}` + bucket = url.pathname.slice(1) // Remove leading slash + forcePathStyle = true + } else { + // Simple AWS format: bucket (defaults to us-east-1) + bucket = endpointBucketPart + } + + return { + accessKeyId, + secretAccessKey, + bucket, + region, + endpoint, + forcePathStyle, + prefix, + } + } catch (error) { + throw new Error(`Failed to parse S3 URI: ${error.message}`) + } + } + + async init({ rootDir, worldDir }) { + console.log('[collections] initializing from S3') + this.list = [] + this.blueprints = new Set() + + // List all collection folders in S3 + const collectionFolders = await this.listCollectionFolders() + + for (const folderName of collectionFolders) { + try { + const collection = await this.loadCollection(folderName) + if (collection) { + this.list.push(collection) + for (const blueprint of collection.blueprints) { + this.blueprints.add(blueprint) + } + } + } catch (error) { + console.error(`[collections] Failed to load collection ${folderName}:`, error) + } + } + } + + async listCollectionFolders() { + const folders = new Set() + let continuationToken = undefined + + do { + try { + const response = await this.client.send( + new ListObjectsV2Command({ + Bucket: this.bucketName, + Prefix: this.prefix, + Delimiter: '/', + ContinuationToken: continuationToken, + }) + ) + + if (response.CommonPrefixes) { + for (const prefix of response.CommonPrefixes) { + // Extract folder name from prefix + const folderPath = prefix.Prefix.replace(this.prefix, '') + const folderName = folderPath.replace('/', '') + if (folderName) { + folders.add(folderName) + } + } + } + + continuationToken = response.NextContinuationToken + } catch (error) { + throw new Error(`Failed to list S3 collection folders: ${error.message}`) + } + } while (continuationToken) + + // Sort folders, keeping "default" first + return Array.from(folders).sort((a, b) => { + if (a === 'default') return -1 + if (b === 'default') return 1 + return a.localeCompare(b) + }) + } + + async loadCollection(folderName) { + try { + // Load manifest.json + const manifestKey = `${this.prefix}${folderName}/manifest.json` + const manifestResponse = await this.client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: manifestKey, + }) + ) + + const manifestBuffer = await manifestResponse.Body.transformToByteArray() + const manifest = JSON.parse(Buffer.from(manifestBuffer).toString()) + + const blueprints = [] + + // Load each app file + for (const appFilename of manifest.apps) { + const appKey = `${this.prefix}${folderName}/${appFilename}` + const appResponse = await this.client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: appKey, + }) + ) + + const appBuffer = await appResponse.Body.transformToByteArray() + const appFile = new File([appBuffer], appFilename, { + type: 'application/octet-stream', + }) + + const app = await importApp(appFile) + + // Upload assets to the main assets system + for (const asset of app.assets) { + await assets.upload(asset.file) + } + + blueprints.push(app.blueprint) + } + + return { + id: folderName, + name: manifest.name, + blueprints, + } + } catch (error) { + console.error(`Failed to load collection ${folderName}:`, error) + return null + } + } +} diff --git a/src/server/collections.js b/src/server/collections.js index 6d927313..300942fc 100644 --- a/src/server/collections.js +++ b/src/server/collections.js @@ -1,61 +1,4 @@ -import fs from 'fs-extra' -import path from 'path' -import { importApp } from '../core/extras/appTools' -import { assets } from './assets' +import { CollectionsS3 } from './CollectionsS3' +import { CollectionsLocal } from './CollectionsLocal' -class Collections { - constructor() { - this.list = [] - this.blueprints = new Set() - } - - async init({ rootDir, worldDir }) { - console.log('[collections] initializing') - this.dir = path.join(worldDir, '/collections') - // ensure collections directory exists - await fs.ensureDir(this.dir) - // copy over built-in collections - await fs.copy(path.join(rootDir, 'src/world/collections'), this.dir) - // ensure all collections apps are installed - let folderNames = fs.readdirSync(this.dir) - folderNames.sort((a, b) => { - // keep "default" first then sort alphabetically - if (a === 'default') return -1 - if (b === 'default') return 1 - return a.localeCompare(b) - }) - for (const folderName of folderNames) { - const folderPath = path.join(this.dir, folderName) - const stats = fs.statSync(folderPath) - if (!stats.isDirectory()) continue - const manifestPath = path.join(folderPath, 'manifest.json') - if (!fs.existsSync(manifestPath)) continue - const manifest = fs.readJsonSync(manifestPath) - const blueprints = [] - for (const appFilename of manifest.apps) { - const appPath = path.join(folderPath, appFilename) - const appBuffer = fs.readFileSync(appPath) - const appFile = new File([appBuffer], appFilename, { - type: 'application/octet-stream', - }) - const app = await importApp(appFile) - for (const asset of app.assets) { - // const file = asset.file - // const assetFilename = asset.url.slice(8) // remove 'asset://' prefix - await assets.upload(asset.file) - } - blueprints.push(app.blueprint) - } - this.list.push({ - id: folderName, - name: manifest.name, - blueprints, - }) - for (const blueprint of blueprints) { - this.blueprints.add(blueprint) - } - } - } -} - -export const collections = new Collections() +export const collections = process.env.COLLECTIONS === 's3' ? new CollectionsS3() : new CollectionsLocal() \ No newline at end of file diff --git a/src/server/index.js b/src/server/index.js index 67d7345b..33972b4f 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -59,6 +59,12 @@ if (!process.env.ASSETS_BASE_URL) { if (process.env.ASSETS === 's3' && !process.env.ASSETS_S3_URI) { throw new Error(`[envs] ASSETS_S3_URI must be set when using ASSETS=s3`) } +if (!process.env.COLLECTIONS) { + throw new Error(`[envs] COLLECTIONS must be set to 'local' or 's3'`) +} +if (process.env.COLLECTIONS === 's3' && !process.env.COLLECTIONS_S3_URI) { + throw new Error(`[envs] COLLECTIONS_S3_URI must be set when using COLLECTIONS=s3`) +} const fastify = Fastify({ logger: { level: 'error' } }) From e98b1aa32c7e3c9f4c417868021d6e5e10a7bf15 Mon Sep 17 00:00:00 2001 From: DevStarlight Date: Tue, 23 Sep 2025 13:22:51 +0200 Subject: [PATCH 2/5] feat: add default port fallback to 3000 - Add default port value (3000) when PORT env var is not set - Remove PORT validation that was throwing error when not defined - Improves server startup resilience following legacy code principles --- src/server/index.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/server/index.js b/src/server/index.js index 33972b4f..a47ec8e6 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -20,15 +20,12 @@ import { cleaner } from './cleaner' const rootDir = path.join(__dirname, '../') const worldDir = path.join(rootDir, process.env.WORLD) -const port = process.env.PORT +const port = process.env.PORT || 3000 // check envs if (!process.env.WORLD) { throw new Error('[envs] WORLD not set') } -if (!process.env.PORT) { - throw new Error('[envs] PORT not set') -} if (!process.env.JWT_SECRET) { throw new Error('[envs] JWT_SECRET not set') } From 3b98b2ca41c8d8dffe29fd6dd6510a192708aa2e Mon Sep 17 00:00:00 2001 From: DevStarlight Date: Tue, 23 Sep 2025 13:26:02 +0200 Subject: [PATCH 3/5] feat: Improve the code for only reading from source (it doesn't require any write) --- src/server/CollectionsS3.js | 181 ++++++++---------------------------- src/server/index.js | 4 +- 2 files changed, 42 insertions(+), 143 deletions(-) diff --git a/src/server/CollectionsS3.js b/src/server/CollectionsS3.js index d34f7603..56c15c31 100644 --- a/src/server/CollectionsS3.js +++ b/src/server/CollectionsS3.js @@ -1,105 +1,25 @@ -import { - GetObjectCommand, - ListObjectsV2Command, - S3Client, -} from '@aws-sdk/client-s3' import { importApp } from '../core/extras/appTools' import { assets } from './assets' export class CollectionsS3 { constructor() { - // Parse S3 URI: s3://access_key:secret_key@endpoint/bucket/prefix - // or for AWS: s3://access_key:secret_key@bucket.s3.region.amazonaws.com/prefix - // or simple AWS: s3://access_key:secret_key@bucket/prefix (defaults to us-east-1) - const uri = process.env.COLLECTIONS_S3_URI - if (!uri) { - throw new Error('COLLECTIONS_S3_URI environment variable is required') + this.baseUrl = process.env.COLLECTIONS_BASE_URL + if (!this.baseUrl) { + throw new Error('COLLECTIONS_BASE_URL environment variable is required') } - - const config = this.parseURI(uri) - this.bucketName = config.bucket - this.prefix = config.prefix || 'collections/' - - // Initialize S3 client - this.client = new S3Client({ - region: config.region, - endpoint: config.endpoint, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - forcePathStyle: config.forcePathStyle, - }) - } - - parseURI(uri) { - try { - // Remove s3:// prefix - if (!uri.startsWith('s3://')) { - throw new Error('URI must start with s3://') - } - uri = uri.slice(5) - - // Split by @ to separate credentials from endpoint/bucket - const [credentials, endpointBucket] = uri.split('@') - if (!credentials || !endpointBucket) { - throw new Error('Invalid S3 URI format') - } - - const [accessKeyId, secretAccessKey] = credentials.split(':') - if (!accessKeyId || !secretAccessKey) { - throw new Error('Missing access key or secret key') - } - - // Parse endpoint/bucket/prefix - const parts = endpointBucket.split('/') - const endpointBucketPart = parts[0] - const prefix = parts.slice(1).join('/') - - // Determine if this is AWS or custom endpoint - let region = 'us-east-1' - let endpoint = undefined - let forcePathStyle = false - let bucket = endpointBucketPart - - if (endpointBucketPart.includes('.s3.') && endpointBucketPart.includes('.amazonaws.com')) { - // AWS format: bucket.s3.region.amazonaws.com - const match = endpointBucketPart.match(/^(.+)\.s3\.(.+)\.amazonaws\.com$/) - if (match) { - bucket = match[1] - region = match[2] - } - } else if (endpointBucketPart.includes('://')) { - // Custom endpoint format: https://endpoint/bucket - const url = new URL(endpointBucketPart) - endpoint = `${url.protocol}//${url.host}` - bucket = url.pathname.slice(1) // Remove leading slash - forcePathStyle = true - } else { - // Simple AWS format: bucket (defaults to us-east-1) - bucket = endpointBucketPart - } - - return { - accessKeyId, - secretAccessKey, - bucket, - region, - endpoint, - forcePathStyle, - prefix, - } - } catch (error) { - throw new Error(`Failed to parse S3 URI: ${error.message}`) + + // Ensure baseUrl ends with / + if (!this.baseUrl.endsWith('/')) { + this.baseUrl += '/' } } async init({ rootDir, worldDir }) { - console.log('[collections] initializing from S3') + console.log('[collections] initializing from CloudFront') this.list = [] this.blueprints = new Set() - // List all collection folders in S3 + // List all collection folders from CloudFront const collectionFolders = await this.listCollectionFolders() for (const folderName of collectionFolders) { @@ -118,72 +38,51 @@ export class CollectionsS3 { } async listCollectionFolders() { - const folders = new Set() - let continuationToken = undefined - - do { + // For now, we'll use a predefined list or try to discover collections + // This could be enhanced to read from a collections index file + const commonCollections = ['default'] + + // Try to discover collections by checking for manifest.json + const discoveredCollections = [] + for (const collectionName of commonCollections) { try { - const response = await this.client.send( - new ListObjectsV2Command({ - Bucket: this.bucketName, - Prefix: this.prefix, - Delimiter: '/', - ContinuationToken: continuationToken, - }) - ) - - if (response.CommonPrefixes) { - for (const prefix of response.CommonPrefixes) { - // Extract folder name from prefix - const folderPath = prefix.Prefix.replace(this.prefix, '') - const folderName = folderPath.replace('/', '') - if (folderName) { - folders.add(folderName) - } - } + const manifestUrl = `${this.baseUrl}${collectionName}/manifest.json` + const response = await fetch(manifestUrl) + if (response.ok) { + discoveredCollections.push(collectionName) } - - continuationToken = response.NextContinuationToken } catch (error) { - throw new Error(`Failed to list S3 collection folders: ${error.message}`) + // Collection doesn't exist, skip } - } while (continuationToken) - - // Sort folders, keeping "default" first - return Array.from(folders).sort((a, b) => { - if (a === 'default') return -1 - if (b === 'default') return 1 - return a.localeCompare(b) - }) + } + + return discoveredCollections } async loadCollection(folderName) { try { - // Load manifest.json - const manifestKey = `${this.prefix}${folderName}/manifest.json` - const manifestResponse = await this.client.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: manifestKey, - }) - ) + // Load manifest.json from CloudFront + const manifestUrl = `${this.baseUrl}${folderName}/manifest.json` + const manifestResponse = await fetch(manifestUrl) - const manifestBuffer = await manifestResponse.Body.transformToByteArray() - const manifest = JSON.parse(Buffer.from(manifestBuffer).toString()) + if (!manifestResponse.ok) { + throw new Error(`Failed to fetch manifest: ${manifestResponse.status}`) + } + + const manifest = await manifestResponse.json() const blueprints = [] - // Load each app file + // Load each app file from CloudFront for (const appFilename of manifest.apps) { - const appKey = `${this.prefix}${folderName}/${appFilename}` - const appResponse = await this.client.send( - new GetObjectCommand({ - Bucket: this.bucketName, - Key: appKey, - }) - ) + const appUrl = `${this.baseUrl}${folderName}/${appFilename}` + const appResponse = await fetch(appUrl) + + if (!appResponse.ok) { + throw new Error(`Failed to fetch app ${appFilename}: ${appResponse.status}`) + } - const appBuffer = await appResponse.Body.transformToByteArray() + const appBuffer = await appResponse.arrayBuffer() const appFile = new File([appBuffer], appFilename, { type: 'application/octet-stream', }) diff --git a/src/server/index.js b/src/server/index.js index a47ec8e6..4be61659 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -59,8 +59,8 @@ if (process.env.ASSETS === 's3' && !process.env.ASSETS_S3_URI) { if (!process.env.COLLECTIONS) { throw new Error(`[envs] COLLECTIONS must be set to 'local' or 's3'`) } -if (process.env.COLLECTIONS === 's3' && !process.env.COLLECTIONS_S3_URI) { - throw new Error(`[envs] COLLECTIONS_S3_URI must be set when using COLLECTIONS=s3`) +if (process.env.COLLECTIONS === 's3' && !process.env.COLLECTIONS_BASE_URL) { + throw new Error(`[envs] COLLECTIONS_BASE_URL must be set when using COLLECTIONS=s3`) } const fastify = Fastify({ logger: { level: 'error' } }) From 59871ab3c98f8d1dcfeabdead851a7591ba17f17 Mon Sep 17 00:00:00 2001 From: DevStarlight Date: Tue, 23 Sep 2025 13:27:16 +0200 Subject: [PATCH 4/5] feat: Update .env.example with s3 collections property --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 0ac7bc40..6c65498f 100644 --- a/.env.example +++ b/.env.example @@ -34,7 +34,7 @@ ASSETS_S3_URI= # How collections are stored and loaded (local or s3) COLLECTIONS=local -COLLECTIONS_S3_URI= +COLLECTIONS_BASE_URL= # By default world data is stored in a local sqlite database in the world folder # Optionally set this to a postgres uri to store remotely, eg `postgres://username:password@host:port/database` From afbf9a4383ffcebc97e48481d2466f3bb8ad4d43 Mon Sep 17 00:00:00 2001 From: DevStarlight Date: Wed, 1 Oct 2025 21:54:10 +0200 Subject: [PATCH 5/5] fix: Make COLLECTIONS prop optional in the .env variables --- src/server/index.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/server/index.js b/src/server/index.js index 4be61659..69f201af 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -56,9 +56,6 @@ if (!process.env.ASSETS_BASE_URL) { if (process.env.ASSETS === 's3' && !process.env.ASSETS_S3_URI) { throw new Error(`[envs] ASSETS_S3_URI must be set when using ASSETS=s3`) } -if (!process.env.COLLECTIONS) { - throw new Error(`[envs] COLLECTIONS must be set to 'local' or 's3'`) -} if (process.env.COLLECTIONS === 's3' && !process.env.COLLECTIONS_BASE_URL) { throw new Error(`[envs] COLLECTIONS_BASE_URL must be set when using COLLECTIONS=s3`) }