feat: Migrate to SQLite

companion/lib/Cloud/Controller.ts

@@ -6,12 +6,23 @@ import { xyToOldBankIndex } from '@companion-app/shared/ControlId.js'
 import { delay } from '../Resources/Util.js'
 import type { ControlLocation } from '@companion-app/shared/Model/Common.js'
 import type { Registry } from '../Registry.js'
-import type { CloudDatabase } from '../Data/CloudDatabase.js'
 import type { DataCache } from '../Data/Cache.js'
 import type { ClientSocket } from '../UI/Handler.js'
 import type { ImageResult } from '../Graphics/ImageResult.js'
+import nodeMachineId from 'node-machine-id'
 
 const CLOUD_URL = 'https://api.bitfocus.io/v1'
+const CLOUD_TABLE: string = 'cloud'
+
+function generateMachineId() {
+	try {
+		return nodeMachineId.machineIdSync(true)
+	} catch (e) {
+		// The nodeMachineId call can fail if the machine has stricter security that blocks regedit
+		// If that happens, fallback to a uuid, which while not stable, is better than nothing
+		return v4()
+	}
+}
 
 /**
  * The class that manages the Bitfocus Cloud functionality
@@ -80,30 +91,28 @@ export class CloudController extends CoreBase {
 		canActivate: false,
 	}
 
-	readonly clouddb: CloudDatabase
 	readonly cache: DataCache
 
-	constructor(registry: Registry, clouddb: CloudDatabase, cache: DataCache) {
+	constructor(registry: Registry, cache: DataCache) {
 		super(registry, 'Cloud/Controller')
 
-		this.clouddb = clouddb
 		this.cache = cache
 
-		this.data = this.clouddb.getKey('auth', {
+		this.data = this.db.getTableKey(CLOUD_TABLE, 'auth', {
 			token: '',
 			user: '',
 			connections: {},
 			cloudActive: false,
 		})
 
 		this.companionId = registry.appInfo.machineId
-		const uuid = this.clouddb.getKey('uuid', undefined)
+		const uuid = this.db.getTableKey(CLOUD_TABLE, 'uuid', generateMachineId())
 		this.#setState({ uuid })
 
-		const regions = this.cache.getKey('cloud_servers', undefined)
+		const regions = this.cache.getKey('cloud_servers', {})
 
 		if (regions !== undefined) {
-			for (const region of regions) {
+			for (const region of Object.values<any>(regions)) {
 				if (region.id && region.label && region.hostname) {
 					CloudController.availableRegions[region.id] = { host: region.hostname, name: region.label }
 				}
@@ -313,7 +322,7 @@ export class CloudController extends CoreBase {
 	async #handleCloudRegenerateUUID(_client: ClientSocket): Promise<void> {
 		const newUuid = v4()
 		this.#setState({ uuid: newUuid })
-		this.clouddb.setKey('uuid', newUuid)
+		this.db.setTableKey(CLOUD_TABLE, 'uuid', newUuid)
 
 		this.#setState({ cloudActive: false })
 		await delay(1000)
@@ -376,7 +385,7 @@ export class CloudController extends CoreBase {
 			if (responseObject.token !== undefined) {
 				this.data.token = responseObject.token
 				this.data.user = email
-				this.clouddb.setKey('auth', this.data)
+				this.db.setTableKey(CLOUD_TABLE, 'auth', this.data)
 				this.#setState({ authenticated: true, authenticating: false, authenticatedAs: email, error: null })
 				this.#readConnections(this.data.connections)
 			} else {
@@ -399,7 +408,7 @@ export class CloudController extends CoreBase {
 		this.data.token = ''
 		this.data.connections = {}
 		this.data.cloudActive = false
-		this.clouddb.setKey('auth', this.data)
+		this.db.setTableKey(CLOUD_TABLE, 'auth', this.data)
 
 		this.#setState({
 			authenticated: false,
@@ -438,7 +447,7 @@ export class CloudController extends CoreBase {
 
 			if (result.token) {
 				this.data.token = result.token
-				this.clouddb.setKey('auth', this.data)
+				this.db.setTableKey(CLOUD_TABLE, 'auth', this.data)
 				this.#setState({
 					authenticated: true,
 					authenticatedAs: result.customer?.email,
@@ -550,7 +559,7 @@ export class CloudController extends CoreBase {
 
 		this.data.connections[region] = enabled
 
-		this.clouddb.setKey('auth', this.data)
+		this.db.setTableKey(CLOUD_TABLE, 'auth', this.data)
 	}
 
 	/**
@@ -576,7 +585,7 @@ export class CloudController extends CoreBase {
 
 		if (oldState.cloudActive !== newState.cloudActive) {
 			this.data.cloudActive = newState.cloudActive
-			this.clouddb.setKey('auth', this.data)
+			this.db.setTableKey(CLOUD_TABLE, 'auth', this.data)
 
 			if (newState.authenticated) {
 				for (let region in this.#regionInstances) {

companion/lib/Controls/ControlBase.ts

@@ -73,7 +73,7 @@ export abstract class ControlBase<TJson> extends CoreBase {
 		const newJson = this.toJSON(true)
 
 		// Save to db
-		this.db.setKey(['controls', this.controlId], newJson as any)
+		this.db.setTableKey('controls', this.controlId, newJson as any)
 
 		// Now broadcast to any interested clients
 		const roomName = ControlConfigRoom(this.controlId)

companion/lib/Controls/Controller.ts

@@ -738,7 +738,7 @@ export class ControlsController extends CoreBase {
 
 				this.#controls.delete(controlId)
 
-				this.db.setKey(['controls', controlId], undefined)
+				this.db.deleteTableKey('controls', controlId)
 
 				return true
 			}
@@ -1049,7 +1049,7 @@ export class ControlsController extends CoreBase {
 	 */
 	init(): void {
 		// Init all the control classes
-		const config: Record<string, SomeControlModel> = this.db.getKey('controls', {})
+		const config: Record<string, SomeControlModel> = this.db.getTable('controls')
 		for (const [controlId, controlObj] of Object.entries(config)) {
 			if (controlObj && controlObj.type) {
 				const inst = this.#createClassForControl(controlId, 'all', controlObj, false)
@@ -1130,7 +1130,7 @@ export class ControlsController extends CoreBase {
 			control.destroy()
 			this.#controls.delete(controlId)
 
-			this.db.setKey(['controls', controlId], undefined)
+			this.db.deleteTableKey('controls', controlId)
 		}
 
 		const location = this.page.getLocationOfControlId(controlId)

companion/lib/Data/Cache.ts

@@ -1,4 +1,5 @@
-import { DataStoreBase } from './StoreBase.js'
+import { DatabaseDefault, DataStoreBase } from './StoreBase.js'
+import { DataLegacyCache } from './Legacy/Cache.js'
 
 /**
  * The class that manages the applications's disk cache
@@ -24,18 +25,62 @@ export class DataCache extends DataStoreBase {
 	/**
 	 * The stored defaults for a new cache
 	 */
-	private static Defaults: object = {}
-	/**
-	 * The default minimum interval in ms to save to disk (30000 ms)
-	 */
-	private static SaveInterval: number = 30000
+	static Defaults: DatabaseDefault = {
+		main: {
+			cloud_servers: {},
+		},
+	}
 
 	/**
 	 * @param configDir - the root config directory
 	 */
 	constructor(configDir: string) {
-		super(configDir, 'datacache', DataCache.SaveInterval, DataCache.Defaults, 'Data/Cache')
+		super(configDir, 'cache', 'main', 'Data/Cache')
+
+		this.startSQLite()
+	}
+
+	/**
+	 * Create the database tables
+	 */
+	protected create(): void {
+		if (this.store) {
+			const create = this.store.prepare(
+				`CREATE TABLE IF NOT EXISTS ${this.defaultTable} (id STRING UNIQUE, value STRING);`
+			)
+			try {
+				create.run()
+			} catch (e) {
+				this.logger.warn(`Error creating table ${this.defaultTable}`)
+			}
+		}
+	}
+
+	/**
+	 * Save the defaults since a file could not be found/loaded/parsed
+	 */
+	protected loadDefaults(): void {
+		for (const [key, value] of Object.entries(DataCache.Defaults)) {
+			for (const [key2, value2] of Object.entries(value)) {
+				this.setTableKey(key, key2, value2)
+			}
+		}
+
+		this.isFirstRun = true
+	}
+
+	/**
+	 * Skip loading migrating the old DB to SQLite
+	 */
+	protected migrateFileToSqlite(): void {
+		this.create()
+
+		const legacyDB = new DataLegacyCache(this.cfgDir)
+
+		const data = legacyDB.getAll()
 
-		this.loadSync()
+		for (const [key, value] of Object.entries(data)) {
+			this.setKey(key, value)
+		}
 	}
 }

companion/lib/Data/Database.ts

@@ -1,5 +1,8 @@
-import { DataStoreBase } from './StoreBase.js'
+import { DatabaseDefault, DataStoreBase } from './StoreBase.js'
+import { DataLegacyDatabase } from './Legacy/Database.js'
 import { upgradeStartup } from './Upgrade.js'
+import { createTables as createTablesV1 } from './Schema/v1.js'
+import { createTables as createTablesV5 } from './Schema/v5.js'
 
 /**
  * The class that manages the applications's main database
@@ -25,22 +28,55 @@ export class DataDatabase extends DataStoreBase {
 	/**
 	 * The stored defaults for a new db
 	 */
-	static Defaults: object = {
-		page_config_version: 3,
+	static Defaults: DatabaseDefault = {
+		main: {
+			page_config_version: 5,
+		},
 	}
-	/**
-	 * The default minimum interval in ms to save to disk (4000 ms)
-	 */
-	private static SaveInterval: number = 4000
 
 	/**
 	 * @param configDir - the root config directory
 	 */
 	constructor(configDir: string) {
-		super(configDir, 'db', DataDatabase.SaveInterval, DataDatabase.Defaults, 'Data/Database')
+		super(configDir, 'db', 'main', 'Data/Database')
 
-		this.loadSync()
+		this.startSQLite()
 
 		upgradeStartup(this)
 	}
+
+	/**
+	 * Create the database tables
+	 */
+	protected create(): void {
+		createTablesV5(this.store, this.defaultTable, this.logger)
+	}
+
+	/**
+	 * Save the defaults since a file could not be found/loaded/parsed
+	 */
+	protected loadDefaults(): void {
+		for (const [key, value] of Object.entries(DataDatabase.Defaults)) {
+			for (const [key2, value2] of Object.entries(value)) {
+				this.setTableKey(key, key2, value2)
+			}
+		}
+
+		this.isFirstRun = true
+	}
+
+	/**
+	 * Load the old file driver and migrate to SQLite
+	 */
+	protected migrateFileToSqlite(): void {
+		createTablesV1(this.store, this.defaultTable, this.logger)
+
+		const legacyDB = new DataLegacyDatabase(this.cfgDir)
+
+		const data = legacyDB.getAll()
+
+		for (const [key, value] of Object.entries(data)) {
+			this.setKey(key, value)
+		}
+	}
 }

companion/lib/Data/ImportExport.ts

@@ -434,14 +434,6 @@ export class DataImportExport extends CoreBase {
 				archive.append(out, { name: 'log.csv' })
 			}
 
-			try {
-				const _db = this.db.getAll()
-				const out = JSON.stringify(_db)
-				archive.append(out, { name: 'db.ram' })
-			} catch (e) {
-				this.logger.debug(`Support bundle append db: ${e}`)
-			}
-
 			try {
 				const payload = this.registry.ui.update.compilePayload()
 				let out = JSON.stringify(payload)
@@ -520,6 +512,7 @@ export class DataImportExport extends CoreBase {
 			if (object.instances) {
 				for (const inst of Object.values(object.instances)) {
 					if (inst) {
+						/** @ts-ignore */
 						inst.lastUpgradeIndex = inst.lastUpgradeIndex ?? -1
 					}
 				}
@@ -583,6 +576,7 @@ export class DataImportExport extends CoreBase {
 
 					for (const [id, trigger] of Object.entries(object.triggers)) {
 						clientObject.triggers[id] = {
+							/** @ts-ignore */
 							name: trigger.options.name,
 						}
 					}

companion/lib/Data/Legacy/Cache.ts

@@ -0,0 +1,32 @@
+import { DataLegacyStoreBase } from './StoreBase.js'
+
+/**
+ * The class that manages the applications's disk cache
+ *
+ * @author Håkon Nessjøen <haakon@bitfocus.io>
+ * @author Keith Rocheck <keith.rocheck@gmail.com>
+ * @author William Viker <william@bitfocus.io>
+ * @author Julian Waller <me@julusian.co.uk>
+ * @since 2.3.0
+ * @copyright 2022 Bitfocus AS
+ * @license
+ * This program is free software.
+ * You should have received a copy of the MIT licence as well as the Bitfocus
+ * Individual Contributor License Agreement for Companion along with
+ * this program.
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial activities involving the Companion software without
+ * disclosing the source code of your own applications.
+ */
+export class DataLegacyCache extends DataLegacyStoreBase {
+	/**
+	 * @param configDir - the root config directory
+	 */
+	constructor(configDir: string) {
+		super(configDir, 'datacache', 'Data/Legacy/Cache')
+
+		this.loadSync()
+	}
+}