Merge branch 'emqx-backend-update' into team-broker-client-ui

.eslintrc

@@ -253,8 +253,6 @@
                 "frontend/src/components/tables/cells/TeamTypeCell.vue",
                 "frontend/src/components/tables/cells/UserCell.vue",
                 "frontend/src/components/tables/cells/UserRoleCell.vue",
-                "frontend/src/pages/application/components/ExportProjectComponents.vue",
-                "frontend/src/pages/application/components/ImportProjectComponents.vue",
                 "frontend/src/pages/application/Debug.vue",
                 "frontend/src/pages/device/components/DeviceLastSeenBadge.vue",
                 "frontend/src/pages/device/Overview.vue",

.github/workflows/publish.yml

@@ -26,7 +26,7 @@ jobs:
     if: |
       ( github.event.workflow_run.conclusion == 'success' && github.ref == 'refs/heads/main' ) ||
       ( github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' )
-    uses: 'flowfuse/github-actions-workflows/.github/workflows/publish_node_package.yml@v0.34.0'
+    uses: 'flowfuse/github-actions-workflows/.github/workflows/publish_node_package.yml@v0.36.0'
     with:
       package_name: flowfuse
       build_package: true

config/cypress-shared.config.js

@@ -28,14 +28,17 @@ module.exports = {
                             // fileName looks like a regex
                             debug = debug + JSON.stringify(files)
                             const filteredList = files.filter(file => !!file.match(re))
-                            return filteredList.length === 1
+                            if (filteredList.length === 1) {
+                                return filteredList[0]
+                            }
+                            return false
                         }
                         return false
                     }).then(result => {
                         if (!result) {
                             throw new Error(`${fileName} not found in ${baseDir} ${debug}`)
                         }
-                        return true
+                        return result
                     })
                 },
                 // Clear all files in the downloads folder

docs/migration/introduction.md

@@ -45,5 +45,5 @@ will now be started with the modules installed.
 ### Static Files
 
 Check your `settings.js` file to see if `httpStatic` has been set, if so then
-check for any files in this path. FlowFuse does not currently support serving
-static files so you will need to find alternative hosting for these such as AWS S3.
+check for any files in this path. The files in this path need to be manually
+migrated to [FlowFuse's Static Assets](/docs/user/static-asset-service/) service.

docs/user/snapshots.md

@@ -44,23 +44,37 @@ A snapshot can be downloaded to your local machine for backup or sharing.
 
 To download a snapshot:
 
-1. Go to the instance's page and select the **Snapshots** tab.
+1. Go to the desired application, instance or device overview page and select the **Snapshots** tab.
 2. Open the dropdown menu to the right of the snapshot you want to download and
-   select the **Download Snapshot** option.
+   click the **Download Snapshot** option to open the download dialog.
+3. Select the required components to download.
+   - **Flows**: Include the snapshot flows
+   - **Credentials**: Include the snapshot flows credentials
+   - **Environment Variables**: Include environment variables in the snapshot
+      - **Keys and Values**: Include the keys and values of the environment variables
+      - **Keys Only**: Include only the keys of the environment variables
+4. Enter a secret to encrypt any credentials in the snapshot (optional, depends on components selected).
+5. Click **Download**
 
 ## Upload a snapshot
 
 A snapshot can be uploaded to a Node-RED instance in FlowFuse.
 
 To upload a snapshot:
 
-1. Go to the instance's page and select the **Snapshots** tab.
+1.  Go to the desired instance or device overview page and select the **Snapshots** tab.
 2. Click the **Upload Snapshot** button.
 3. Select the snapshot file from your local machine.
-4. If the snapshot contains credentials, you will be asked to enter the credentials secret.
-   This is the secret that was used to encrypt the credentials in the snapshot.
-5. Update the name and description if required.
-6. Click **Upload**
+4. Update the name and description if required.
+5. Select the components to upload:
+   - **Flows**: Include the snapshots flows
+   - **Credentials**: Include the snapshots flows credentials (visible only if the snapshot contains credentials)
+   - **Environment Variables**: Include environment variables in the snapshot
+      - **Keys and Values**: Include the keys and values of the environment variables
+      - **Keys Only**: Include only the keys of the environment variables
+6. If the snapshot contains credentials and the `Credentials` component is checked,
+   you will be asked to enter a Secret. This will be used to later decrypt any credentials in the snapshots flows.
+7. Click **Upload**
 
 ## Delete a snapshot
 

forge/comms/v2AuthRoutes.js

@@ -42,7 +42,7 @@ module.exports = async function (app) {
                     result: 'allow',
                     is_superuser: false,
                     client_attrs: {
-                        team: 'ff/v1/internal/c/'
+                        team: ''
                     }
                 })
             } else {

forge/db/controllers/Snapshot.js

@@ -20,9 +20,22 @@ module.exports = {
      * @param {String} [options.credentialSecret] secret to encrypt credentials with.
      * @param {Object} [options.credentials] (Optional) credentials to export. If omitted, credentials of the current owner will be re-encrypted with the provided `credentialSecret`.
      * @param {Object} [options.owner] (Optional) The owner project or device. If omitted, the snapshot's owner will be obtained from the database.
+     * @param {Object} [options.components] (Optional) Components to include in the snapshot. Defaults to all components.
+     * @param {Boolean} [options.components.flows] (Optional) Include flows in the snapshot. Defaults to true.
+     * @param {Boolean} [options.components.credentials] (Optional) Include credentials in the snapshot. Defaults to true if `options.components.flows` is true, otherwise false.
+     * @param {'all' | 'keys' | false} [options.components.envVars='all'] (Optional) Env Var options: `'All'` will keep keys and values, `'keys'` will keep only keys, `false` will remove them.
      */
     exportSnapshot: async function (app, snapshot, options) {
-        if (!options.credentialSecret) {
+        const components = {
+            flows: options.components?.flows ?? true,
+            credentials: options.components?.credentials ?? true,
+            envVars: ['all', 'keys', false].includes(options.components?.envVars) ? options.components.envVars : 'all'
+        }
+        if (components.flows === false) {
+            components.credentials = false // no flows, no credentials!
+        }
+
+        if (components.credentials !== false && !options.credentialSecret) {
             return null
         }
 
@@ -54,22 +67,48 @@ module.exports = {
             ...snapshot.toJSON()
         }
 
+        // include/exclude components of the snapshot
+        if (components.flows === false) {
+            result.flows = {
+                flows: [],
+                credentials: {} // no flows, no credentials!
+            }
+        } else if (components.credentials === false) {
+            result.flows = {
+                flows: snapshot.flows.flows || [],
+                credentials: {} // no credentials!
+            }
+        }
+        if (components.envVars === false) {
+            result.settings.env = {}
+        } else if (components.envVars === 'keys' && snapshot.settings?.env && Object.keys(snapshot.settings.env).length > 0) {
+            const keysOnly = Object.keys(snapshot.settings.env).reduce((acc, key) => {
+                acc[key] = ''
+                return acc
+            }, {})
+            result.settings = Object.assign({}, snapshot.settings, { env: keysOnly })
+        }
+
         // loop keys of result.settings.env and remove any that match FF_*
         Object.keys(result.settings.env).forEach((key) => {
             if (key.startsWith('FF_')) {
                 delete result.settings.env[key]
             }
         })
 
-        // use the secret stored in the snapshot, if available...
-        const currentSecret = result.credentialSecret || owner.credentialSecret || (owner.getCredentialSecret && await owner.getCredentialSecret())
-        const credentials = options.credentials ? options.credentials : result.flows.credentials
+        if (components.flows === false || components.credentials === false) {
+            result.flows.credentials = {}
+        } else {
+            // use the secret stored in the snapshot, if available...
+            const currentSecret = result.credentialSecret || owner.credentialSecret || (owner.getCredentialSecret && await owner.getCredentialSecret())
+            const credentials = options.credentials ? options.credentials : result.flows.credentials
 
-        // if provided credentials already encrypted: "exportCredentials" will just return the same credentials
-        // if provided credentials are raw: "exportCredentials" will encrypt them with the secret provided
-        // if credentials are not provided: project's flows credentials will be used, they will be encrypted with the provided secret
-        const keyToDecrypt = (options.credentials && options.credentials.$) ? options.credentialSecret : currentSecret
-        result.flows.credentials = app.db.controllers.Project.exportCredentials(credentials || {}, keyToDecrypt, options.credentialSecret)
+            // if provided credentials already encrypted: "exportCredentials" will just return the same credentials
+            // if provided credentials are raw: "exportCredentials" will encrypt them with the secret provided
+            // if credentials are not provided: project's flows credentials will be used, they will be encrypted with the provided secret
+            const keyToDecrypt = (options.credentials && options.credentials.$) ? options.credentialSecret : currentSecret
+            result.flows.credentials = app.db.controllers.Project.exportCredentials(credentials || {}, keyToDecrypt, options.credentialSecret)
+        }
 
         return result
     },
@@ -143,8 +182,13 @@ module.exports = {
      * @param {*} snapshot - snapshot data
      * @param {String} credentialSecret - secret to encrypt credentials with. Can be null if the snapshot does not contain credentials.
      * @param {*} user - user who uploaded the snapshot
+     * @param {Object} [options] - additional options
+     * @param {Object} [options.components] - Include flows in the snapshot. Defaults to true.
+     * @param {Boolean} [options.components.flows] - Include flows in the snapshot. Defaults to true.
+     * @param {Boolean} [options.components.credentials] - Include credentials in the snapshot. Defaults to true if `options.components.flows` is true, otherwise false.
+     * @param {'all' | 'keys' | false} [options.components.envVars='all'] - Env Var options: `'All'` will keep keys and values, `'keys'` will keep only keys, `false` will remove them.
      */
-    async uploadSnapshot (app, owner, snapshot, credentialSecret, user) {
+    async uploadSnapshot (app, owner, snapshot, credentialSecret, user, options) {
         // Validate the owner
         let ownerType
         if (owner.constructor.name === 'Project') {
@@ -155,33 +199,70 @@ module.exports = {
             throw new Error('Invalid owner type')
         }
 
+        const components = {
+            flows: options?.components?.flows ?? true,
+            credentials: options?.components?.credentials ?? true,
+            envVars: ['all', 'keys', false].includes(options?.components?.envVars) ? options?.components.envVars : 'all'
+        }
+        if (components.flows === false) {
+            components.credentials = false // no flows, no credentials!
+        }
+
+        // shallow clone the snapshot before modifying it
+        const importSnapshot = Object.assign({}, snapshot)
+
+        // include/exclude components of the snapshot
+        if (components.flows === false) {
+            importSnapshot.flows = {
+                flows: [],
+                credentials: {} // no flows, no credentials!
+            }
+        } else if (components.credentials === false) {
+            importSnapshot.flows = {
+                flows: snapshot.flows.flows || [],
+                credentials: {} // no credentials!
+            }
+        }
+        if (components.envVars === false) {
+            importSnapshot.settings = {
+                ...snapshot.settings,
+                env: {} // no env vars!
+            }
+        } else if (components.envVars === 'keys' && snapshot.settings?.env && Object.keys(snapshot.settings.env).length > 0) {
+            const keysOnly = Object.keys(snapshot.settings.env).reduce((acc, key) => {
+                acc[key] = ''
+                return acc
+            }, {})
+            importSnapshot.settings = Object.assign({}, snapshot.settings, { env: keysOnly })
+        }
+
         const targetCredentialSecret = owner.credentialSecret || (owner.getCredentialSecret && await owner.getCredentialSecret()) || credentialSecret
         // 1. If the snapshot includes credentials but no credentialSecret, we should reject it
         // 2. if the snapshot includes credentials and a credentialSecret, we should reencrypt for the owner
-        if (snapshot.flows.credentials?.$) {
+        if (importSnapshot.flows.credentials?.$) {
             if (!credentialSecret) {
                 throw new Error('Missing credentialSecret')
             }
             // Need to re-encrypt the credentials for the target
-            snapshot.flows.credentials = app.db.controllers.Project.exportCredentials(snapshot.flows.credentials, credentialSecret, targetCredentialSecret)
+            importSnapshot.flows.credentials = app.db.controllers.Project.exportCredentials(importSnapshot.flows.credentials, credentialSecret, targetCredentialSecret)
         }
 
         const ProjectId = ownerType === 'instance' ? owner.id : null
         const DeviceId = ownerType === 'device' ? owner.id : null
 
         const snapshotOptions = {
-            name: snapshot.name,
-            description: snapshot.description || '',
+            name: importSnapshot.name,
+            description: importSnapshot.description || '',
             credentialSecret: targetCredentialSecret,
             settings: {
-                settings: snapshot.settings?.settings || {},
-                env: snapshot.settings?.env || {},
-                modules: snapshot.settings?.modules || {}
+                settings: importSnapshot.settings?.settings || {},
+                env: importSnapshot.settings?.env || {},
+                modules: importSnapshot.settings?.modules || {}
 
             },
             flows: {
-                flows: snapshot.flows.flows || [],
-                credentials: snapshot.flows.credentials || { }
+                flows: importSnapshot.flows.flows || [],
+                credentials: importSnapshot.flows.credentials || { }
             },
             ProjectId,
             DeviceId,

forge/db/controllers/Team.js

@@ -12,6 +12,14 @@ module.exports = {
         // Reinflate the object now the user has been added
         const team = await app.db.models.Team.bySlug(newTeam.slug)
 
+        // Record in our Product tracking
+        app.product.capture(user.username, '$ff-team-created', {
+            'team-name': team.name,
+            'created-at': team.createdAt
+        }, {
+            team: team.id
+        })
+
         return team
     },
 

forge/db/models/Notification.js

@@ -89,6 +89,25 @@ module.exports = {
                         count,
                         notifications: rows
                     }
+                },
+                updateNotificationsState: async ({ read = true, ids = [] }, user) => {
+                    if (ids.length === 0) {
+                        return
+                    }
+
+                    ids = ids.map(id => typeof id === 'string'
+                        ? M.Notification.decodeHashid(id)?.[0]
+                        : id)
+
+                    await M.Notification.update(
+                        { read: read ? 1 : 0 },
+                        {
+                            where: {
+                                id: ids.filter(e => e),
+                                UserId: user.id
+                            }
+                        }
+                    )
                 }
             }
         }

forge/ee/routes/teamBroker/index.js

@@ -39,6 +39,7 @@ module.exports = async function (app) {
      * @memberof forge.routes.api.team.broker
      */
     app.get('/clients', {
+        preHandler: app.needsPermission('broker:clients:list'),
         schema: {
             summary: 'List MQTT clients for the team',
             tags: ['MQTT Broker'],
@@ -80,7 +81,7 @@ module.exports = async function (app) {
      * @memberof forge.routes.api.team.broker
      */
     app.post('/client', {
-        preHandler: app.needsPermission('project:create'),
+        preHandler: app.needsPermission('broker:clients:create'),
         schema: {
             summary: 'Create new MQTT client for the team',
             tags: ['MQTT Broker'],
@@ -139,6 +140,7 @@ module.exports = async function (app) {
      * @memberof forge.routes.api.team.broker
      */
     app.get('/client/:username', {
+        preHandler: app.needsPermission('broker:clients:list'),
         schema: {
             summary: 'Get details about a specific MQTT client',
             tags: ['MQTT Broker'],
@@ -181,8 +183,8 @@ module.exports = async function (app) {
      * @static
      * @memberof forge.routes.api.team.broker
      */
-    app.patch('/client/:username', {
-        preHandler: app.needsPermission('project:create'),
+    app.put('/client/:username', {
+        preHandler: app.needsPermission('broker:clients:edit'),
         schema: {
             summary: 'Modify a MQTT Client',
             tags: ['MQTT Broker'],
@@ -264,7 +266,7 @@ module.exports = async function (app) {
      * @memberof forge.routes.api.team.broker
      */
     app.delete('/client/:username', {
-        preHandler: app.needsPermission('project:create'),
+        preHandler: app.needsPermission('broker:clients:delete'),
         schema: {
             summary: 'Delete a MQTT client',
             tags: ['MQTT Broker'],

forge/lib/permissions.js

@@ -172,7 +172,13 @@ const Permissions = {
     'project:files:list': { description: 'List files under a project', role: Roles.Member },
     'project:files:create': { description: 'Upload files to a project', role: Roles.Member },
     'project:files:edit': { description: 'Modify files in a project', role: Roles.Member },
-    'project:files:delete': { description: 'Delete files in a project', role: Roles.Member }
+    'project:files:delete': { description: 'Delete files in a project', role: Roles.Member },
+
+    // Team Broker
+    'broker:clients:list': { description: 'List Team Broker clients', role: Roles.Member },
+    'broker:clients:create': { description: 'Create Team Broker clients', role: Roles.Admin },
+    'broker:clients:edit': { description: 'Edit Team Broker clients', role: Roles.Admin },
+    'broker:clients:delete': { description: 'Delete Team Broker clients', role: Roles.Admin }
 }
 
 module.exports = {