diff --git a/doc/licenses/d3-path-3.1.0/LICENSE b/doc/licenses/d3-path-3.1.0/LICENSE
new file mode 100644
index 0000000000..ed25746bbf
--- /dev/null
+++ b/doc/licenses/d3-path-3.1.0/LICENSE
@@ -0,0 +1,13 @@
+Copyright 2015-2022 Mike Bostock
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
diff --git a/doc/licenses/d3-path-3.1.0/README b/doc/licenses/d3-path-3.1.0/README
new file mode 100644
index 0000000000..5bbf02a561
--- /dev/null
+++ b/doc/licenses/d3-path-3.1.0/README
@@ -0,0 +1,8 @@
+d3-path (https://github.com/d3/d3-path)
+----------------------------------------------------------
+
+ Version: 3.1.0
+ From: 'Mike Bostock'
+ License(s):
+ BSD 0-clause (bundled/d3-path-3.1.0/LICENSE)
+
diff --git a/doc/licenses/d3-path-3.1.0/dep-coordinates.txt b/doc/licenses/d3-path-3.1.0/dep-coordinates.txt
new file mode 100644
index 0000000000..5159b4e033
--- /dev/null
+++ b/doc/licenses/d3-path-3.1.0/dep-coordinates.txt
@@ -0,0 +1 @@
+d3-path:3.1.0
diff --git a/doc/licenses/d3-shape-3.2.0/LICENSE b/doc/licenses/d3-shape-3.2.0/LICENSE
new file mode 100644
index 0000000000..fbe44bdc9a
--- /dev/null
+++ b/doc/licenses/d3-shape-3.2.0/LICENSE
@@ -0,0 +1,13 @@
+Copyright 2010-2022 Mike Bostock
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
diff --git a/doc/licenses/d3-shape-3.2.0/README b/doc/licenses/d3-shape-3.2.0/README
new file mode 100644
index 0000000000..f0adda5c51
--- /dev/null
+++ b/doc/licenses/d3-shape-3.2.0/README
@@ -0,0 +1,8 @@
+d3-path (https://github.com/d3/d3-shape)
+----------------------------------------------------------
+
+ Version: 3.2.0
+ From: 'Mike Bostock'
+ License(s):
+ BSD 0-clause (bundled/d3-shape-3.2.0/LICENSE)
+
diff --git a/doc/licenses/d3-shape-3.2.0/dep-coordinates.txt b/doc/licenses/d3-shape-3.2.0/dep-coordinates.txt
new file mode 100644
index 0000000000..10973f4897
--- /dev/null
+++ b/doc/licenses/d3-shape-3.2.0/dep-coordinates.txt
@@ -0,0 +1 @@
+d3-shape:3.2.0
diff --git a/guacamole/src/main/frontend/package-lock.json b/guacamole/src/main/frontend/package-lock.json
index e44b0da8dd..b40c26743c 100644
--- a/guacamole/src/main/frontend/package-lock.json
+++ b/guacamole/src/main/frontend/package-lock.json
@@ -14,6 +14,8 @@
"angular-translate-loader-static-files": "^2.19.0",
"blob-polyfill": ">=7.0.20220408",
"csv": "^6.2.5",
+ "d3-path": "^3.1.0",
+ "d3-shape": "^3.2.0",
"datalist-polyfill": "^1.25.1",
"file-saver": "^2.0.5",
"fuzzysort": "^2.0.4",
@@ -4651,6 +4653,25 @@
"resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
"integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A=="
},
+ "node_modules/d3-path": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
+ "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==",
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/d3-shape": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
+ "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
+ "dependencies": {
+ "d3-path": "^3.1.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/datalist-polyfill": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/datalist-polyfill/-/datalist-polyfill-1.25.1.tgz",
@@ -14937,6 +14958,19 @@
"resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
"integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A=="
},
+ "d3-path": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz",
+ "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ=="
+ },
+ "d3-shape": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz",
+ "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==",
+ "requires": {
+ "d3-path": "^3.1.0"
+ }
+ },
"datalist-polyfill": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/datalist-polyfill/-/datalist-polyfill-1.25.1.tgz",
diff --git a/guacamole/src/main/frontend/package.json b/guacamole/src/main/frontend/package.json
index 99d742dcf9..dcb2705072 100644
--- a/guacamole/src/main/frontend/package.json
+++ b/guacamole/src/main/frontend/package.json
@@ -13,6 +13,8 @@
"angular-translate-loader-static-files": "^2.19.0",
"blob-polyfill": ">=7.0.20220408",
"csv": "^6.2.5",
+ "d3-path": "^3.1.0",
+ "d3-shape": "^3.2.0",
"datalist-polyfill": "^1.25.1",
"file-saver": "^2.0.5",
"fuzzysort": "^2.0.4",
diff --git a/guacamole/src/main/frontend/src/app/player/directives/player.js b/guacamole/src/main/frontend/src/app/player/directives/player.js
index b864b0e93a..83acb68ae8 100644
--- a/guacamole/src/main/frontend/src/app/player/directives/player.js
+++ b/guacamole/src/main/frontend/src/app/player/directives/player.js
@@ -81,6 +81,7 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
// Required services
const keyEventDisplayService = $injector.get('keyEventDisplayService');
+ const playerHeatmapService = $injector.get('playerHeatmapService');
const playerTimeService = $injector.get('playerTimeService');
/**
@@ -187,6 +188,65 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
*/
$scope.showKeyLog = false;
+ /**
+ * The height, in pixels, of the SVG heatmap paths. Note that this is not
+ * necessarily the actual rendered height, just the initial size of the
+ * SVG path before any styling is applied.
+ *
+ * @type {!number}
+ */
+ $scope.HEATMAP_HEIGHT = 100;
+
+ /**
+ * The width, in pixels, of the SVG heatmap paths. Note that this is not
+ * necessarily the actual rendered width, just the initial size of the
+ * SVG path before any styling is applied.
+ *
+ * @type {!number}
+ */
+ $scope.HEATMAP_WIDTH = 1000;
+
+ /**
+ * The maximum number of key events per millisecond to display in the
+ * key event heatmap. Any key event rates exceeding this value will be
+ * capped at this rate to ensure that unsually large spikes don't make
+ * swamp the rest of the data.
+ *
+ * Note: This is 6 keys per second (events include both presses and
+ * releases) - equivalent to ~88 words per minute typed.
+ *
+ * @type {!number}
+ */
+ const KEY_EVENT_RATE_CAP = 12 / 1000;
+
+ /**
+ * The maximum number of frames per millisecond to display in the
+ * frame heatmap. Any frame rates exceeding this value will be
+ * capped at this rate to ensure that unsually large spikes don't make
+ * swamp the rest of the data.
+ *
+ * @type {!number}
+ */
+ const FRAME_RATE_CAP = 10 / 1000;
+
+ /**
+ * An SVG path describing a smoothed curve that visualizes the relative
+ * number of frames rendered throughout the recording - i.e. a heatmap
+ * of screen updates.
+ *
+ * @type {!string}
+ */
+ $scope.frameHeatmap = '';
+
+ /**
+ * An SVG path describing a smoothed curve that visualizes the relative
+ * number of key events recorded throughout the recording - i.e. a
+ * heatmap of key events.
+ *
+ * @type {!string}
+ */
+ $scope.keyHeatmap = '';
+
/**
* Whether a seek request is currently in progress. A seek request is
* in progress if the user is attempting to change the current playback
@@ -213,6 +273,22 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
*/
var mouseActivityTimer = null;
+ /**
+ * The recording-relative timestamp of each frame of the recording that
+ * has been processed so far.
+ *
+ * @type {!number[]}
+ */
+ var frameTimestamps = [];
+
+ /**
+ * The recording-relative timestamp of each text event that has been
+ * processed so far.
+ *
+ * @type {!number[]}
+ */
+ var keyTimestamps = [];
+
/**
* Return true if any batches of key event logs are available for this
* recording, or false otherwise.
@@ -355,11 +431,25 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
// Begin downloading the recording
$scope.recording.connect();
- // Notify listeners when the recording is completely loaded
+ // Notify listeners and set any heatmap paths
+ // when the recording is completely loaded
$scope.recording.onload = function recordingLoaded() {
$scope.operationMessage = null;
$scope.$emit('guacPlayerLoaded');
$scope.$evalAsync();
+
+ const recordingDuration = $scope.recording.getDuration();
+
+ // Generate heat maps for rendered frames and typed text
+ $scope.frameHeatmap = (
+ playerHeatmapService.generateHeatmapPath(
+ frameTimestamps, recordingDuration, FRAME_RATE_CAP,
+ $scope.HEATMAP_HEIGHT, $scope.HEATMAP_WIDTH));
+ $scope.keyHeatmap = (
+ playerHeatmapService.generateHeatmapPath(
+ keyTimestamps, recordingDuration, KEY_EVENT_RATE_CAP,
+ $scope.HEATMAP_HEIGHT, $scope.HEATMAP_WIDTH));
+
};
// Notify listeners if an error occurs
@@ -375,6 +465,9 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
$scope.operationProgress = src.size ? current / src.size : 0;
$scope.$emit('guacPlayerProgress', duration, current);
$scope.$evalAsync();
+
+ // Store the timestamp of the just-received frame
+ frameTimestamps.push(duration);
};
// Notify listeners when playback has started/resumed
@@ -396,6 +489,8 @@ angular.module('player').directive('guacPlayer', ['$injector', function guacPlay
$scope.textBatches = (
keyEventDisplayService.parseEvents(events));
+ keyTimestamps = events.map(event => event.timestamp);
+
};
// Notify listeners when current position within the recording
diff --git a/guacamole/src/main/frontend/src/app/player/services/playerHeatmapService.js b/guacamole/src/main/frontend/src/app/player/services/playerHeatmapService.js
new file mode 100644
index 0000000000..9259aa4dc5
--- /dev/null
+++ b/guacamole/src/main/frontend/src/app/player/services/playerHeatmapService.js
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * 'License'); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { curveCatmullRom } from 'd3-shape';
+import { path } from 'd3-path';
+
+/**
+ * A service for generating heat maps of activity levels per time interval,
+ * for session recording playback.
+ */
+angular.module('player').factory('playerHeatmapService', [() => {
+
+ /**
+ * A default, relatively-gentle Gaussian smoothing kernel. This kernel
+ * should help heatmaps look a bit less jagged, while not reducing fidelity
+ * very much.
+ *
+ * @type {!number[]}
+ */
+ const GAUSSIAN_KERNEL = [0.0013, 0.1573, 0.6827, 0.1573, 0.0013];
+
+ /**
+ * The number of buckets that a series of activity timestamps should be
+ * divided into.
+ *
+ * @type {!number}
+ */
+ const NUM_BUCKETS = 100;
+
+ /**
+ * Given a list of values to smooth out, produce a smoothed data set with
+ * the same length as the original provided list.
+ *
+ * @param {!number[]} values
+ * The list of histogram values to smooth out.
+ *
+ * @returns {!number[]}
+ * The smoothed value array.
+ */
+ function smooth(values) {
+
+ // The starting offset into the values array for each calculation
+ const lookBack = Math.floor(GAUSSIAN_KERNEL.length / 2);
+
+ // Apply the smoothing kernel to each value in the provided array
+ return _.map(values, (value, index) => {
+
+ // Total up the weighted values for each position in the kernel
+ return _.reduce(GAUSSIAN_KERNEL, (total, weight, kernelIndex) => {
+
+ // The offset into the original values array for the kernel
+ const valuesOffset = kernelIndex - lookBack;
+
+ // The position inside the original values array to be included
+ const valuesIndex = index + valuesOffset;
+
+ // If the contribution to the final smoothed value would be outside
+ // the bounds of the array, just use the original value instead
+ const contribution = ((valuesIndex >= 0) && valuesIndex < values.length)
+ ? values[valuesIndex] : value;
+
+ // Use the provided weight from the kernel and add to the total
+ return total + (contribution * weight);
+
+ }, 0);
+
+ });
+ }
+
+ /**
+ * Given an array of values, with each value representing an activity count
+ * during a bucket of time, generate a smooth curve, scaled to PATH_HEIGHT
+ * height, and PATH_WIDTH width.
+ *
+ * @param {!number[]} bucketizedData
+ * The bucketized counts to create an SVG path from.
+ *
+ * @param {!number} maxBucketValue
+ * The size of the largest value in the bucketized data.
+ *
+ * @param {!number} height
+ * The target height, in pixels, of the highest point in the heatmap.
+ *
+ * @param {!number} width
+ * The target width, in pixels, of the heatmap.
+ *
+ * @returns {!string}
+ * An SVG path representing a smooth curve, passing through all points
+ * in the provided data.
+ */
+ function createPath(bucketizedData, maxBucketValue, height, width) {
+
+ // Calculate scaling factor to ensure that paths are all the same heigh
+ const yScalingFactor = height / maxBucketValue;
+
+ // Scale a given Y value appropriately
+ const scaleYValue = yValue => height - (yValue * yScalingFactor);
+
+ // Calculate scaling factor to ensure that paths are all the same width
+ const xScalingFactor = width / bucketizedData.length;
+
+ // Construct a continuous curved path
+ const curvedPath = path();
+ const curve = curveCatmullRom(curvedPath);
+
+ curve.lineStart();
+
+ // Add all the data points
+ for (let i = 0; i < bucketizedData.length; i++) {
+
+ // Scale up the x value to hit the target width
+ const x = xScalingFactor * i;
+
+ // Scale and invert the height for display
+ const y = scaleYValue(bucketizedData[i]);
+
+ // Add the scaled point
+ curve.point(x, y);
+
+ }
+
+ // Move back to 0 to complete the path
+ curve.lineEnd();
+ curvedPath.lineTo(width, scaleYValue(0));
+
+ // Generate the SVG path for this curve
+ const rawPathText = curvedPath.toString();
+
+ // The SVG path as generated by D3 starts with a move to the first data
+ // point. This means that when the path ends and the subpath is closed,
+ // it returns to the position of the first data point instead of the
+ // origin. To fix this, the initial move command is removed, and the
+ // path is amended to start at the origin. TODO: Find a better way to
+ // handle this.
+ const startAtOrigin = (
+
+ // Start at origin
+ 'M0,' + scaleYValue(0) +
+
+ // Line to the first point in the curve, to close the shape
+ 'L0,' + scaleYValue(bucketizedData[0])
+
+ );
+
+ // Strip off the first move command from the path
+ const strippedPathText = _.replace(rawPathText, /^[^C]*/, '');
+
+ return startAtOrigin + strippedPathText;
+ }
+
+ const service = {};
+
+ /**
+ * Given a raw array of timestamps indicating when events of a certain type
+ * occured during a record, generate and return a smoothed SVG path
+ * indicating how many events occured during each equal-length bucket.
+ *
+ * @param {!number[]} timestamps
+ * A raw array of timestamps, one for every relevant event. These
+ * must be monotonically increasing.
+ *
+ * @param {!number} duration
+ * The duration over which the heatmap should apply. This value may
+ * be greater than the maximum timestamp value, in which case the path
+ * will drop to 0 after the last timestamp in the provided array.
+ *
+ * @param {number} maxRate
+ * The maximum number of events per millisecond that should be displayed
+ * in the final path. Any rates over this amount will just be capped at
+ * this value.
+ *
+ * @param {!number} height
+ * The target height, in pixels, of the highest point in the heatmap.
+ *
+ * @param {!number} width
+ * The target width, in pixels, of the heatmap.
+ *
+ * @returns {!string}
+ * A smoothed, graphable SVG path representing levels of activity over
+ * time, as extracted from the provided timestamps.
+ */
+ service.generateHeatmapPath = (timestamps, duration, maxRate, height, width) => {
+
+ // The height and width must both be valid in order to create the path
+ if (!height || !width) {
+ console.warn("Heatmap height and width must be positive.");
+ return '';
+ }
+
+ // If no timestamps are available, no path can be created
+ if (!timestamps || !timestamps.length)
+ return '';
+
+ // An initially empty array containing no activity in any bucket
+ const buckets = new Array(NUM_BUCKETS).fill(0);
+
+ // If no events occured, return the an empty path
+ if (!timestamps.length)
+ return '';
+
+ // Determine the bucket granularity
+ const bucketDuration = duration / NUM_BUCKETS;
+
+ // The rate-limited maximum number of events that any bucket can have,
+ const maxPossibleBucketValue = Math.floor(bucketDuration * maxRate);
+
+ // If the duration is invalid, return the still-empty array
+ if (duration <= 0)
+ return '';
+
+ let maxBucketValue = 0;
+
+ // Partition the events into a count of events per bucket
+ let currentBucketIndex = 0;
+ timestamps.forEach(timestamp => {
+
+ // If the current timestamp has passed the end of the current
+ // bucket, move to the appropriate bucket
+ if (timestamp >= (currentBucketIndex + 1) * bucketDuration)
+ currentBucketIndex = Math.min(
+ Math.floor((timestamp / bucketDuration)), NUM_BUCKETS - 1);
+
+ // Do not record events that exceed the maximum allowable rate
+ if (buckets[currentBucketIndex] >= maxPossibleBucketValue)
+ buckets[currentBucketIndex] = maxPossibleBucketValue;
+
+ else
+ // Increment the count for the current bucket
+ buckets[currentBucketIndex]++;
+
+ // Keep track of the maximum value seen so far
+ maxBucketValue = Math.max(
+ maxBucketValue, buckets[currentBucketIndex]);
+
+ });
+
+ // Smooth the data for better aesthetics before creating the path
+ const smoothed = smooth(buckets);
+
+ // Create an SVG path based on the smoothed data
+ return createPath(smoothed, maxBucketValue, height, width);
+
+ }
+
+
+ return service;
+
+}]);
diff --git a/guacamole/src/main/frontend/src/app/player/templates/player.html b/guacamole/src/main/frontend/src/app/player/templates/player.html
index eeb3744b54..f002bd42ca 100644
--- a/guacamole/src/main/frontend/src/app/player/templates/player.html
+++ b/guacamole/src/main/frontend/src/app/player/templates/player.html
@@ -28,6 +28,21 @@
ng-model="playbackPosition"
ng-on-change="commitSeekRequest()">
+