Merge pull request #3481 from FlowFuse/3358-device-auto-snapshot

Implement device auto snapshot

docs/device-agent/deploy.md

@@ -91,6 +91,13 @@ in the Developer Mode options panel.
 You will be prompted to give the snapshot a name and description. See [Snapshots](../user/snapshots.md) for more information
 about working with snapshots.
 
+**Auto Device Snapshots**
+
+For devices that are assigned to an application, the platform will automatically create a snapshot of the device
+when it detects flows modified. This snapshot will be created with the name "Auto Snapshot - yyyy-mm-dd hh:mm-ss".
+Only the last 10 auto snapshots are kept, others are deleted on a first in first out basis.
+
+
 ### Important Notes
 
 * Remote access to the editor requires Device Agent v0.8.0 or later.
@@ -109,3 +116,6 @@ about working with snapshots.
 * The device will not receive any updates from the platform while in Developer Mode.
 * The device must be online and connected to the platform to enable "Editor Access".
 * To minimise server and device resources, it is recommended to disable "Editor Access" when not actively developing flows on a device.
+* Auto snapshots were introduced in FlowFuse V2.1.
+* Auto snapshots are only supported for devices assigned to an application.
+* If an auto snapshot is set as the target snapshot for a device or assigned to a pipeline stage, it will not be auto cleaned up meaning it is possible to have more than 10 auto snapshots.

docs/user/snapshots.md

@@ -60,7 +60,7 @@ To set the **Device Target** of an application owned device:
 1. Go to the devices's page and select the **Snapshots** tab.
 2. In the list of snapshots available, a "Deploy Snapshot" button will be displayed
    for each snapshot as you hover over it.
-3. You will be asked to confirm - click **Set Target** to continue.
+3. You will be asked to confirm - click the **Confirm** button to set it as the target snapshot.
 
 This will cause the snapshot to be pushed out to the device the
 next time it checks in.

forge/auditLog/device.js

@@ -45,6 +45,11 @@ module.exports = {
                 async disabled (actionedBy, error, device) {
                     await log('device.remote-access.disabled', actionedBy, device?.id, generateBody({ error, device }))
                 }
+            },
+            settings: {
+                async updated (actionedBy, error, device, updates) {
+                    await log('device.settings.updated', actionedBy, device?.id, generateBody({ error, device, updates }))
+                }
             }
         }
 

forge/db/controllers/ProjectSnapshot.js

@@ -1,3 +1,181 @@
+const { Op } = require('sequelize')
+const DEVICE_AUTO_SNAPSHOT_LIMIT = 10
+const DEVICE_AUTO_SNAPSHOT_PREFIX = 'Auto Snapshot' // Any changes to the format should be reflected in frontend/src/pages/device/Snapshots/index.vue
+
+const deviceAutoSnapshotUtils = {
+    deployTypeEnum: {
+        full: 'Full',
+        flows: 'Modified Flows',
+        nodes: 'Modified Nodes'
+    },
+    nameRegex: new RegExp(`^${DEVICE_AUTO_SNAPSHOT_PREFIX} - \\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$`), // e.g "Auto Snapshot - 2023-02-01 12:34:56"
+    prefix: DEVICE_AUTO_SNAPSHOT_PREFIX,
+    autoBackupLimit: DEVICE_AUTO_SNAPSHOT_LIMIT,
+    generateName: () => `${DEVICE_AUTO_SNAPSHOT_PREFIX} - ${new Date().toLocaleString('sv-SE')}`, // "base - YYYY-MM-DD HH:MM:SS"
+    generateDescription: (deploymentType = '') => {
+        const deployInfo = deviceAutoSnapshotUtils.deployTypeEnum[deploymentType]
+            ? `${deviceAutoSnapshotUtils.deployTypeEnum[deploymentType]} deployment`
+            : 'deployment'
+        return `Device ${deviceAutoSnapshotUtils.prefix} taken following a ${deployInfo}`
+    },
+    isAutoSnapshot: function (snapshot) {
+        return deviceAutoSnapshotUtils.nameRegex.test(snapshot.name)
+    },
+    /**
+     * Get all auto snapshots for a device
+     *
+     * NOTE: If a `limit` of 10 is provided and some of the snapshots are in use, the actual number of snapshots returned may be less than 10
+     * @param {Object} app - the forge application object
+     * @param {Object} device - a device (model) instance
+     * @param {boolean} [excludeInUse=true] - whether to exclude snapshots that are currently in use by a device, device group or pipeline stage device group
+     * @param {number} [limit=0] - the maximum number of snapshots to query in the database (0 means no limit)
+     */
+    getAutoSnapshots: async function (app, device, excludeInUse = true, limit = 0) {
+        // TODO: the snapshots table should really have a an indexed `type` column to distinguish between auto and manual snapshots
+        // for now, as per MVP, we'll use the name pattern to identify auto snapshots
+
+        // Get snapshots
+        const possibleAutoSnapshots = await app.db.models.ProjectSnapshot.findAll({
+            where: {
+                DeviceId: device.id,
+                // name: { [Op.regexp]: deviceAutoSnapshotUtils.nameRegex } // regex is not supported by sqlite!
+                name: { [Op.like]: `${DEVICE_AUTO_SNAPSHOT_PREFIX} - %-%-% %:%:%` }
+            },
+            order: [['id', 'ASC']]
+        })
+
+        // Filter out any snapshots that don't match the regex
+        const autoSnapshots = possibleAutoSnapshots.filter(deviceAutoSnapshotUtils.isAutoSnapshot)
+
+        // if caller _wants_ all, including those "in use", we can just return here
+        if (!excludeInUse) {
+            return autoSnapshots
+        }
+        // utility function to remove items from an array
+        const removeFromArray = (baseList, removeList) => baseList.filter((item) => !removeList.includes(item))
+
+        // candidates for are those that are not in use
+        let candidateIds = autoSnapshots.map((snapshot) => snapshot.id)
+
+        // since we're excluding "in use" snapshots, we need to check the following tables:
+        // * device
+        // * device groups
+        // * pipeline stage device group
+        // If any of these snapshots are set as active/target, remove them from the candidates list
+
+        // Check `Devices` table
+        const query = {
+            where: {
+                [Op.or]: [
+                    { targetSnapshotId: { [Op.in]: candidateIds } },
+                    { activeSnapshotId: { [Op.in]: candidateIds } }
+                ]
+            }
+        }
+        if (typeof limit === 'number' && limit > 0) {
+            query.limit = limit
+        }
+        const snapshotsInUseInDevices = await app.db.models.Device.findAll(query)
+        const inUseAsTarget = snapshotsInUseInDevices.map((device) => device.targetSnapshotId)
+        const inUseAsActive = snapshotsInUseInDevices.map((device) => device.activeSnapshotId)
+        candidateIds = removeFromArray(candidateIds, inUseAsTarget)
+        candidateIds = removeFromArray(candidateIds, inUseAsActive)
+
+        // Check `DeviceGroups` table
+        if (app.db.models.DeviceGroup) {
+            const snapshotsInUseInDeviceGroups = await app.db.models.DeviceGroup.findAll({
+                where: {
+                    targetSnapshotId: { [Op.in]: candidateIds }
+                }
+            })
+            const inGroupAsTarget = snapshotsInUseInDeviceGroups.map((group) => group.targetSnapshotId)
+            candidateIds = removeFromArray(candidateIds, inGroupAsTarget)
+        }
+
+        // Check `PipelineStageDeviceGroups` table
+        const isLicensed = app.license.active()
+        if (isLicensed && app.db.models.PipelineStageDeviceGroup) {
+            const snapshotsInUseInPipelineStage = await app.db.models.PipelineStageDeviceGroup.findAll({
+                where: {
+                    targetSnapshotId: { [Op.in]: candidateIds }
+                }
+            })
+            const inPipelineStageAsTarget = snapshotsInUseInPipelineStage.map((stage) => stage.targetSnapshotId)
+            candidateIds = removeFromArray(candidateIds, inPipelineStageAsTarget)
+        }
+
+        return autoSnapshots.filter((snapshot) => candidateIds.includes(snapshot.id))
+    },
+    cleanupAutoSnapshots: async function (app, device, limit = DEVICE_AUTO_SNAPSHOT_LIMIT) {
+        // get all auto snapshots for the device (where not in use)
+        const snapshots = await app.db.controllers.ProjectSnapshot.getDeviceAutoSnapshots(device, true, 0)
+        if (snapshots.length > limit) {
+            const toDelete = snapshots.slice(0, snapshots.length - limit).map((snapshot) => snapshot.id)
+            await app.db.models.ProjectSnapshot.destroy({ where: { id: { [Op.in]: toDelete } } })
+        }
+    },
+    doAutoSnapshot: async function (app, device, deploymentType, { clean = true, setAsTarget = false } = {}, meta) {
+        // eslint-disable-next-line no-useless-catch
+        try {
+            // if not permitted, throw an error
+            if (!device) {
+                throw new Error('Device is required')
+            }
+            if (!app.config.features.enabled('deviceAutoSnapshot')) {
+                throw new Error('Device auto snapshot feature is not available')
+            }
+            if (!(await device.getSetting('autoSnapshot'))) {
+                throw new Error('Device auto snapshot is not enabled')
+            }
+            const teamType = await device.Team.getTeamType()
+            const deviceAutoSnapshotEnabledForTeam = teamType.getFeatureProperty('deviceAutoSnapshot', false)
+            if (!deviceAutoSnapshotEnabledForTeam) {
+                throw new Error('Device auto snapshot is not enabled for the team')
+            }
+
+            const saneSnapshotOptions = {
+                name: deviceAutoSnapshotUtils.generateName(),
+                description: deviceAutoSnapshotUtils.generateDescription(deploymentType),
+                setAsTarget
+            }
+
+            // things to do & consider:
+            // 1. create a snapshot from the device
+            // 2. log the snapshot creation in audit log
+            // 3. delete older auto snapshots if the limit is reached (10)
+            //    do NOT delete any snapshots that are currently in use by an target (instance/device/device group)
+            const user = meta?.user || { id: null } // if no user is available, use `null` (system user)
+
+            // 1. create a snapshot from the device
+            const snapShot = await app.db.controllers.ProjectSnapshot.createDeviceSnapshot(
+                device.Application,
+                device,
+                user,
+                saneSnapshotOptions
+            )
+            snapShot.User = user
+
+            // 2. log the snapshot creation in audit log
+            // TODO: device snapshot: implement audit log
+            // await deviceAuditLogger.device.snapshot.created(request.session.User, null, request.device, snapShot)
+
+            // 3. clean up older auto snapshots
+            if (clean === true) {
+                await app.db.controllers.ProjectSnapshot.cleanupDeviceAutoSnapshots(device)
+            }
+
+            return snapShot
+        } catch (error) {
+            // TODO: device snapshot:  implement audit log
+            // await deviceAuditLogger.device.snapshot.created(request.session.User, error, request.device, null)
+            throw error
+        }
+    }
+}
+
+// freeze the object to prevent accidental changes
+Object.freeze(deviceAutoSnapshotUtils)
+
 module.exports = {
     /**
      * Creates a snapshot of the current state of a project.
@@ -168,5 +346,12 @@ module.exports = {
         result.flows.credentials = app.db.controllers.Project.exportCredentials(credentials || {}, keyToDecrypt, options.credentialSecret)
 
         return result
-    }
+    },
+
+    getDeviceAutoSnapshotDescription: deviceAutoSnapshotUtils.generateDescription,
+    getDeviceAutoSnapshotName: deviceAutoSnapshotUtils.generateName,
+    getDeviceAutoSnapshots: deviceAutoSnapshotUtils.getAutoSnapshots,
+    isDeviceAutoSnapshot: deviceAutoSnapshotUtils.isAutoSnapshot,
+    cleanupDeviceAutoSnapshots: deviceAutoSnapshotUtils.cleanupAutoSnapshots,
+    doDeviceAutoSnapshot: deviceAutoSnapshotUtils.doAutoSnapshot
 }

forge/db/models/Device.js

@@ -10,7 +10,12 @@ const Controllers = require('../controllers')
 const { buildPaginationSearchClause } = require('../utils')
 
 const ALLOWED_SETTINGS = {
-    env: 1
+    env: 1,
+    autoSnapshot: 1
+}
+
+const DEFAULT_SETTINGS = {
+    autoSnapshot: true
 }
 
 module.exports = {
@@ -74,7 +79,7 @@ module.exports = {
                     await app.auditLog.Platform.platform.license.overage('system', null, devices)
                 }
             },
-            beforeSave: async (device, options) => {
+            afterSave: async (device, options) => {
                 // since `id`, `name` and `type` are added as FF_DEVICE_xx env vars, we
                 // should update the settings checksum if they are modified
                 if (device.changed('name') || device.changed('type') || device.changed('id')) {
@@ -131,6 +136,7 @@ module.exports = {
                 },
                 async updateSettingsHash (settings) {
                     const _settings = settings || await this.getAllSettings()
+                    delete _settings.autoSnapshot // autoSnapshot is not part of the settings hash
                     this.settingsHash = hashSettings(_settings)
                 },
                 async getAllSettings () {
@@ -140,6 +146,9 @@ module.exports = {
                         result[setting.key] = setting.value
                     })
                     result.env = Controllers.Device.insertPlatformSpecificEnvVars(this, result.env) // add platform specific device env vars
+                    if (!Object.prototype.hasOwnProperty.call(result, 'autoSnapshot')) {
+                        result.autoSnapshot = DEFAULT_SETTINGS.autoSnapshot
+                    }
                     return result
                 },
                 async updateSettings (obj) {
@@ -161,7 +170,7 @@ module.exports = {
                         if (key === 'env' && value && Array.isArray(value)) {
                             value = Controllers.Device.removePlatformSpecificEnvVars(value) // remove platform specific values
                         }
-                        const result = await M.ProjectSettings.upsert({ DeviceId: this.id, key, value })
+                        const result = await M.DeviceSettings.upsert({ DeviceId: this.id, key, value })
                         await this.updateSettingsHash()
                         await this.save()
                         return result
@@ -177,7 +186,7 @@ module.exports = {
                         }
                         return result.value
                     }
-                    return undefined
+                    return DEFAULT_SETTINGS[key]
                 },
                 async getLatestSnapshot () {
                     const snapshots = await this.getProjectSnapshots({

forge/ee/lib/index.js

@@ -25,4 +25,7 @@ module.exports = fp(async function (app, opts) {
 
     // Set the Custom Catalogs Flag
     app.config.features.register('customCatalogs', true, true)
+
+    // Set the Device Auto Snapshot Feature Flag
+    app.config.features.register('deviceAutoSnapshot', true, true)
 }, { name: 'app.ee.lib' })