diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 442a34dd2..fb6709fc3 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: browser: ['chrome', 'firefox'] - node-version: [18.x, 20.x] + node-version: [18.x, 20.x, 22.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index da27eff89..efc82ad58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Browsertime changelog (we do [semantic versioning](https://semver.org)) +## 22.5.0 - 2024-06-14 +### Added +* Updated the Docker container to include Chrome 126 and Firefox 127. Chromedriver has been updated to 126 [#2141](https://github.com/sitespeedio/browsertime/pull/2141). + +## 22.4.1 - 2024-06-07 +### Fixed +* Make sure the engine is stopped before the extra video/profile run [#2140](https://github.com/sitespeedio/browsertime/pull/2140). + +## 22.4.0 - 2024-06-06 +### Added +* Use `--enableVideoRun` to get one extra run with a video and visual metrics [#2139](https://github.com/sitespeedio/browsertime/pull/2139) + +## 22.3.0 - 2024-06-04 +### Added +* Add the ability to gather power usage measurements on Android from USB power meters, thank you [Gregory Mierzwinski](https://github.com/gmierz) for PR [#2134](https://github.com/sitespeedio/browsertime/pull/2134). +* Add support to visualmetrics to identify key frames matching the given colors, thank you [aosmond](https://github.com/aosmond) for PR [#2119](https://github.com/sitespeedio/browsertime/pull/2119). + +### Fixed +* Removed DOMContentFlushed for Firefox thank you [florinbilt](https://github.com/florinbilt) for PR [#2138](https://github.com/sitespeedio/browsertime/pull/2138). + ## 22.2.0 - 2024-05-24 ### Added * New command: Mouse single click on a element with a specific id `commands.mouse.singleClick.byId(id)` and `commands.mouse.singleClick.byIdAndWait(id)` [#2135](https://github.com/sitespeedio/browsertime/pull/2135). diff --git a/Dockerfile b/Dockerfile index a3ec4d1a4..e49c6f156 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM sitespeedio/webbrowsers:chrome-125.0-firefox-126.0-edge-125.0 +FROM sitespeedio/webbrowsers:chrome-126.0-firefox-127.0-edge-125.0 ARG TARGETPLATFORM=linux/amd64 diff --git a/bin/browsertime.js b/bin/browsertime.js index 7258afd15..92b53aa2e 100755 --- a/bin/browsertime.js +++ b/bin/browsertime.js @@ -123,27 +123,6 @@ async function run(urls, options) { ); } - if (options.enableProfileRun) { - log.info('Make one extra run to collect trace information'); - options.iterations = 1; - if (options.browser === 'firefox') { - options.firefox.geckoProfiler = true; - } else if (options.browser === 'chrome') { - options.chrome.timeline = true; - options.cpu = true; - options.chrome.enableTraceScreenshots = true; - options.chrome.traceCategory = [ - 'disabled-by-default-v8.cpu_profiler' - ]; - } - options.video = false; - options.visualMetrics = false; - const traceEngine = new Engine(options); - await traceEngine.start(); - await traceEngine.runMultiple(urls, scriptsByCategory); - await traceEngine.stop(); - } - await Promise.all(saveOperations); const resultDirectory = relative(process.cwd(), storageManager.directory); @@ -173,6 +152,41 @@ async function run(urls, options) { process.exitCode = 1; } } + + if (options.enableProfileRun || options.enableVideoRun) { + log.info('Make one extra run to collect trace/video information'); + options.iterations = 1; + if (options.enableProfileRun) { + if (options.browser === 'firefox') { + options.firefox.geckoProfiler = true; + } else if (options.browser === 'chrome') { + options.chrome.timeline = true; + options.cpu = true; + options.chrome.enableTraceScreenshots = true; + options.chrome.traceCategory = [ + 'disabled-by-default-v8.cpu_profiler' + ]; + } + } + if (options.enableVideoRun) { + if (options.video === true) { + log.error( + 'You can only configure video run if you do not collect any video' + ); + // This is a hack to not get an error + options.video = false; + options.visualMetrics = false; + } else { + options.video = true; + options.visualMetrics = true; + } + } + const traceEngine = new Engine(options); + await traceEngine.start(); + await traceEngine.runMultiple(urls, scriptsByCategory); + await traceEngine.stop(); + log.info('Extra run finished'); + } } catch (error) { log.error('Error running browsertime', error); process.exitCode = 1; diff --git a/lib/chrome/webdriver/setupChromiumOptions.js b/lib/chrome/webdriver/setupChromiumOptions.js index 3957dbd3d..b6625ed04 100644 --- a/lib/chrome/webdriver/setupChromiumOptions.js +++ b/lib/chrome/webdriver/setupChromiumOptions.js @@ -172,6 +172,10 @@ export function setupChromiumOptions( } } + if (browserOptions.enableVideoAutoplay) { + seleniumOptions.addArguments('--autoplay-policy=no-user-gesture-required'); + } + // It's a new splash screen introduced in Chrome 98 // for new profiles // disable it with ChromeWhatsNewUI diff --git a/lib/core/engine/iteration.js b/lib/core/engine/iteration.js index 2e4278bc9..916619af9 100644 --- a/lib/core/engine/iteration.js +++ b/lib/core/engine/iteration.js @@ -15,7 +15,10 @@ import { addConnectivity, removeConnectivity } from '../../connectivity/index.js'; -import { jsonifyVisualProgress } from '../../support/util.js'; +import { + jsonifyVisualProgress, + jsonifyKeyColorFrames +} from '../../support/util.js'; import { flushDNS } from '../../support/dns.js'; import { getNumberOfRunningProcesses } from '../../support/processes.js'; @@ -232,6 +235,12 @@ export class Iteration { ); } } + if (videoMetrics.visualMetrics['KeyColorFrames']) { + videoMetrics.visualMetrics['KeyColorFrames'] = + jsonifyKeyColorFrames( + videoMetrics.visualMetrics['KeyColorFrames'] + ); + } result[index_].videoRecordingStart = videoMetrics.videoRecordingStart; result[index_].visualMetrics = videoMetrics.visualMetrics; diff --git a/lib/support/cli.js b/lib/support/cli.js index 3c2005bc5..2214d191c 100644 --- a/lib/support/cli.js +++ b/lib/support/cli.js @@ -274,6 +274,11 @@ export function parseCommandLine() { type: 'boolean', group: 'chrome' }) + .option('chrome.enableVideoAutoplay', { + describe: 'Allow videos to autoplay.', + type: 'boolean', + group: 'chrome' + }) .option('chrome.timeline', { alias: 'chrome.trace', describe: @@ -609,6 +614,11 @@ export function parseCommandLine() { describe: 'Make one extra run that collects the profiling trace log (no other metrics is collected). For Chrome it will collect the timeline trace, for Firefox it will get the Geckoprofiler trace. This means you do not need to get the trace for all runs and can skip the overhead it produces.' }) + .option('enableVideoRun', { + type: 'boolean', + describe: + 'Make one extra run that collects video and visual metrics. This means you can do your runs with --visualMetrics true --video false --enableVideoRun true to collect visual metrics from all runs and save a video from the profile/video run. If you run it together with --enableProfileRun it will also collect profiling trace.' + }) .option('video', { type: 'boolean', describe: @@ -718,6 +728,12 @@ export function parseCommandLine() { describe: 'Use the portable visual-metrics processing script (no ImageMagick dependencies).' }) + .option('visualMetricsKeyColor', { + type: 'array', + nargs: 8, + describe: + 'Collect Key Color frame metrics when you run --visualMetrics. Each --visualMetricsKeyColor supplied must have 8 arguments: key name, red channel (0-255) low and high, green channel (0-255) low and high, blue channel (0-255) low and high, fraction (0.0-1.0) of pixels that must match each channel.' + }) .option('scriptInput.visualElements', { describe: 'Include specific elements in visual elements. Give the element a name and select it with document.body.querySelector. Use like this: --scriptInput.visualElements name:domSelector see https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors. Add multiple instances to measure multiple elements. Visual Metrics will use these elements and calculate when they are visible and fully rendered.' diff --git a/lib/support/util.js b/lib/support/util.js index 058032e8a..489de7721 100644 --- a/lib/support/util.js +++ b/lib/support/util.js @@ -230,6 +230,29 @@ export function jsonifyVisualProgress(visualProgress) { } return visualProgress; } +export function jsonifyKeyColorFrames(keyColorFrames) { + // Original data looks like + // "FrameName1=[0-133 255-300], FrameName2=[133-255] FrameName3=[]" + if (typeof keyColorFrames === 'string') { + const keyColorFramesObject = {}; + for (const keyColorPair of keyColorFrames.split(', ')) { + const [name, values] = keyColorPair.split('='); + keyColorFramesObject[name] = []; + const rangePairs = values.replace('[', '').replace(']', ''); + if (rangePairs) { + for (const rangePair of rangePairs.split(' ')) { + const [start, end] = rangePair.split('-'); + keyColorFramesObject[name].push({ + startTimestamp: Number.parseInt(start, 10), + endTimestamp: Number.parseInt(end, 10) + }); + } + } + } + return keyColorFramesObject; + } + return keyColorFrames; +} export function adjustVisualProgressTimestamps( visualProgress, profilerStartTime, diff --git a/lib/video/postprocessing/visualmetrics/visualMetrics.js b/lib/video/postprocessing/visualmetrics/visualMetrics.js index c821ba480..6f0949fce 100644 --- a/lib/video/postprocessing/visualmetrics/visualMetrics.js +++ b/lib/video/postprocessing/visualmetrics/visualMetrics.js @@ -84,6 +84,15 @@ export async function run( scriptArguments.push('--contentful'); } + if (options.visualMetricsKeyColor) { + for (let i = 0; i < options.visualMetricsKeyColor.length; ++i) { + if (i % 8 == 0) { + scriptArguments.push('--keycolor'); + } + scriptArguments.push(options.visualMetricsKeyColor[i]); + } + } + // There seems to be a bug with --startwhite that makes VM bail out // 11:20:14.950 - Calculating image histograms // 11:20:14.951 - No video frames found in /private/var/folders/27/xpnvcsbs0nlfbb4qq397z3rh0000gn/T/vis-cn_JMf diff --git a/package-lock.json b/package-lock.json index 57df194ac..6532c06bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,17 @@ { "name": "browsertime", - "version": "22.2.0", + "version": "22.5.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "browsertime", - "version": "22.2.0", + "version": "22.5.0", "license": "MIT", "dependencies": { "@cypress/xvfb": "1.2.4", "@devicefarmer/adbkit": "3.2.6", - "@sitespeed.io/chromedriver": "125.0.6422-60", + "@sitespeed.io/chromedriver": "126.0.6478-55", "@sitespeed.io/edgedriver": "125.0.2535-47", "@sitespeed.io/geckodriver": "0.34.0", "@sitespeed.io/throttle": "5.0.0", @@ -1209,9 +1209,9 @@ } }, "node_modules/@sitespeed.io/chromedriver": { - "version": "125.0.6422-60", - "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-125.0.6422-60.tgz", - "integrity": "sha512-UxAxq8eJ5H11nzQScLn7yoce9tTa+E2MOn+yRuDO+gEph+r5JUJkc5f7jKBeSN2dlCUD8Fp3RHLRaWqOX4PeRQ==", + "version": "126.0.6478-55", + "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-126.0.6478-55.tgz", + "integrity": "sha512-+TLK/AFaEcbajJrDSmq+xbVdkcegC9HDhD1m447km1kgvzeukfVpuIAZbWQEuJyX+VUB2McbRB58AcOay4FfYQ==", "hasInstallScript": true, "dependencies": { "node-downloader-helper": "2.1.9", @@ -8161,9 +8161,9 @@ "dev": true }, "@sitespeed.io/chromedriver": { - "version": "125.0.6422-60", - "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-125.0.6422-60.tgz", - "integrity": "sha512-UxAxq8eJ5H11nzQScLn7yoce9tTa+E2MOn+yRuDO+gEph+r5JUJkc5f7jKBeSN2dlCUD8Fp3RHLRaWqOX4PeRQ==", + "version": "126.0.6478-55", + "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-126.0.6478-55.tgz", + "integrity": "sha512-+TLK/AFaEcbajJrDSmq+xbVdkcegC9HDhD1m447km1kgvzeukfVpuIAZbWQEuJyX+VUB2McbRB58AcOay4FfYQ==", "requires": { "node-downloader-helper": "2.1.9", "node-stream-zip": "1.15.0" diff --git a/package.json b/package.json index 77e10b930..32ab0f295 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,14 @@ { "name": "browsertime", "description": "Get performance metrics from your web page using Browsertime.", - "version": "22.2.0", + "version": "22.5.0", "bin": "./bin/browsertime.js", "type": "module", "types": "./types/scripting.d.ts", "dependencies": { "@cypress/xvfb": "1.2.4", "@devicefarmer/adbkit": "3.2.6", - "@sitespeed.io/chromedriver": "125.0.6422-60", + "@sitespeed.io/chromedriver": "126.0.6478-55", "@sitespeed.io/edgedriver": "125.0.2535-47", "@sitespeed.io/geckodriver": "0.34.0", "@sitespeed.io/throttle": "5.0.0", diff --git a/types/android/index.d.ts b/types/android/index.d.ts index 1891eb4d9..b17b3a03c 100644 --- a/types/android/index.d.ts +++ b/types/android/index.d.ts @@ -66,5 +66,10 @@ export class Android { 'full-wifi': number; total: number; }>; + measureUsbPowerUsage(startTime: any, endTime: any): Promise<{ + powerUsage: any; + baselineUsage: number; + }>; + getUsbPowerUsageProfile(index: any, url: any, result: any, options: any, storageManager: any): Promise; } //# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/types/android/index.d.ts.map b/types/android/index.d.ts.map index 443123a1f..72c622c66 100644 --- a/types/android/index.d.ts.map +++ b/types/android/index.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/android/index.js"],"names":[],"mappings":"AAiiBA,2DAWC;AA/hBD;IACE,0BAwBC;IAfC,YAAgC;IAEhC,QAIC;IACD,UAAgC;IAGhC,6BAA6B;IAC7B,yBAA2B;IAC3B,eAAgC;IAKlC,uBAgBC;IALC,YAA4C;IAG1C,YAAoE;IAIxE,wCAEC;IAED,8CAIC;IAED,6CAIC;IAED,2CAUC;IAED,mEAYC;IAED,mEAcC;IAED,uCAEC;IAED,0CAGC;IAED,4CAMC;IAED,4CAMC;IAED,uBAGC;IAED,kCAOC;IAED;;;;;;;OAuBC;IAED,2CAKC;IAED,8BAKC;IAED;;;oDAuBC;IAED,2BAIC;IAED,qCAGC;IAED,iCAGC;IAED,wBASC;IAED,2CAOC;IAED,gCAEC;IAED,0BAIC;IAED,iCAGC;IAED,8CAIC;IAED,4BAEC;IAED,sCAYC;IAED,mDAsBC;IAED,gDA+CC;IAED;;;;;;;OAqDC;IAED,4DAQC;IAED,mCAcC;IAED,kCAQC;IAED,iCAIC;IAED;;;;OAGC;CACF"} \ No newline at end of file +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/android/index.js"],"names":[],"mappings":"AAskBA,2DAWC;AAlkBD;IACE,0BAwBC;IAfC,YAAgC;IAEhC,QAIC;IACD,UAAgC;IAGhC,6BAA6B;IAC7B,yBAA2B;IAC3B,eAAgC;IAKlC,uBAgBC;IALC,YAA4C;IAG1C,YAAoE;IAIxE,wCAEC;IAED,8CAIC;IAED,6CAIC;IAED,2CAUC;IAED,mEAYC;IAED,mEAcC;IAED,uCAEC;IAED,0CAGC;IAED,4CAMC;IAED,4CAMC;IAED,uBAGC;IAED,kCAOC;IAED;;;;;;;OAuBC;IAED,2CAKC;IAED,8BAKC;IAED;;;oDAuBC;IAED,2BAIC;IAED,qCAGC;IAED,iCAGC;IAED,wBASC;IAED,2CAOC;IAED,gCAEC;IAED,0BAIC;IAED,iCAGC;IAED,8CAIC;IAED,4BAEC;IAED,sCAYC;IAED,mDAsBC;IAED,gDA+CC;IAED;;;;;;;OAqDC;IAED,4DAQC;IAED,mCAcC;IAED,kCAQC;IAED,iCAIC;IAED;;;;OAGC;IAED;;;OAEC;IAED,6GAQC;CACF"} \ No newline at end of file diff --git a/types/support/util.d.ts b/types/support/util.d.ts index e8e416102..76dc61672 100644 --- a/types/support/util.d.ts +++ b/types/support/util.d.ts @@ -2,5 +2,6 @@ export function formatMetric(name: any, metric: any, multiple: any, inMs: any, e export function logResultLogLine(results: any): void; export function toArray(arrayLike: any): any[]; export function jsonifyVisualProgress(visualProgress: any): any; +export function jsonifyKeyColorFrames(keyColorFrames: any): any; export function adjustVisualProgressTimestamps(visualProgress: any, profilerStartTime: any, recordingStartTime: any): any; //# sourceMappingURL=util.d.ts.map \ No newline at end of file diff --git a/types/support/util.d.ts.map b/types/support/util.d.ts.map index beeff4cc8..a5873fe17 100644 --- a/types/support/util.d.ts.map +++ b/types/support/util.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../lib/support/util.js"],"names":[],"mappings":"AAIA,oGAeC;AACD,qDA0LC;AACD,+CAQC;AACD,gEAeC;AACD,0HAWC"} \ No newline at end of file +{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../lib/support/util.js"],"names":[],"mappings":"AAIA,oGAeC;AACD,qDA0LC;AACD,+CAQC;AACD,gEAeC;AACD,gEAsBC;AACD,0HAWC"} \ No newline at end of file diff --git a/types/video/postprocessing/visualmetrics/visualMetrics.d.ts.map b/types/video/postprocessing/visualmetrics/visualMetrics.d.ts.map index e296f3986..cfc9a6af7 100644 --- a/types/video/postprocessing/visualmetrics/visualMetrics.d.ts.map +++ b/types/video/postprocessing/visualmetrics/visualMetrics.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"visualMetrics.d.ts","sourceRoot":"","sources":["../../../../lib/video/postprocessing/visualmetrics/visualMetrics.js"],"names":[],"mappings":"AAsCA,mGAEC;AACD,0KAiHC"} \ No newline at end of file +{"version":3,"file":"visualMetrics.d.ts","sourceRoot":"","sources":["../../../../lib/video/postprocessing/visualmetrics/visualMetrics.js"],"names":[],"mappings":"AAsCA,mGAEC;AACD,0KA0HC"} \ No newline at end of file diff --git a/visualmetrics/visualmetrics-portable.py b/visualmetrics/visualmetrics-portable.py index bd044bcd1..1a5289eed 100755 --- a/visualmetrics/visualmetrics-portable.py +++ b/visualmetrics/visualmetrics-portable.py @@ -1104,7 +1104,7 @@ def calculate_histograms(directory, histograms_file, force): m = re.search(match, frame) if m is not None: frame_time = int(m.groupdict().get("ms")) - histogram = calculate_image_histogram(frame) + histogram, total, dropped = calculate_image_histogram(frame) gc.collect() if histogram is not None: histograms.append( @@ -1112,6 +1112,8 @@ def calculate_histograms(directory, histograms_file, force): "time": frame_time, "file": os.path.basename(frame), "histogram": histogram, + "total_pixels": total, + "dropped_pixels": dropped, } ) if os.path.isfile(histograms_file): @@ -1130,12 +1132,14 @@ def calculate_histograms(directory, histograms_file, force): def calculate_image_histogram(file): logging.debug("Calculating histogram for " + file) + dropped = 0 try: from PIL import Image im = Image.open(file) width, height = im.size - colors = im.getcolors(width * height) + total = width * height + colors = im.getcolors(total) histogram = { "r": [0 for i in range(256)], "g": [0 for i in range(256)], @@ -1151,13 +1155,16 @@ def calculate_image_histogram(file): histogram["r"][pixel[0]] += count histogram["g"][pixel[1]] += count histogram["b"][pixel[2]] += count + else: + dropped += 1 except Exception: pass colors = None except Exception: + total = 0 histogram = None logging.exception("Error calculating histogram for " + file) - return histogram + return histogram, total, dropped ########################################################################## @@ -1212,6 +1219,7 @@ def calculate_visual_metrics( dirs, progress_file, hero_elements_file, + key_colors, ): metrics = None histograms = load_histograms(histograms_file, start, end) @@ -1225,6 +1233,7 @@ def calculate_visual_metrics( f = open(progress_file, "w") json.dump(progress, f) f.close() + key_color_frames = calculate_key_color_frames(histograms, key_colors) if len(histograms) > 1: metrics = [ {"name": "First Visual Change", "value": histograms[1]["time"]}, @@ -1315,6 +1324,20 @@ def calculate_visual_metrics( metrics.append({"name": "Perceptual Speed Index", "value": 0}) if contentful: metrics.append({"name": "Contentful Speed Index", "value": 0}) + if key_color_frames: + keysum = "" + for key in key_color_frames: + if len(keysum): + keysum += ", " + framesum = "" + for frame in key_color_frames[key]: + if len(framesum): + framesum += " " + framesum += "{0:d}-{1:d}".format( + frame["start_time"], frame["end_time"] + ) + keysum += "{0}=[{1}]".format(key, framesum) + metrics.append({"name": "Key Color Frames", "value": keysum}) prog = "" for p in progress: if len(prog): @@ -1346,6 +1369,79 @@ def load_histograms(histograms_file, start, end): return histograms +def is_key_color_frame(histogram, key_color): + # The fraction is measured against the entire image, not just the sampled + # pixels. This helps avoid matching frames with only a few pixels that + # happen to be in the acceptable range. + total_fraction = histogram["total_pixels"] * key_color["fraction"] + if total_fraction < histogram["total_pixels"] - histogram["dropped_pixels"]: + for channel in ["r", "g", "b"]: + # Find the acceptable range around the target channel value + max_channel = len(histogram["histogram"][channel]) + low = min(max_channel - 1, max(0, key_color[channel + "_low"])) + high = min(max_channel, max(1, key_color[channel + "_high"] + 1)) + target_total = 0 + for i in histogram["histogram"][channel][low:high]: + target_total += i + if target_total < total_fraction: + return False + return True + + +def calculate_key_color_frames(histograms, key_colors): + if not key_colors: + return {} + + key_color_frames = {} + for key in key_colors: + key_color_frames[key] = [] + + current = None + current_key = None + total = 0 + matched = 0 + buckets = 256 + channels = ["r", "g", "b"] + histograms = histograms.copy() + + while len(histograms) > 0: + histogram = histograms.pop(0) + matching_key = None + for key in key_colors: + if is_key_color_frame(histogram, key_colors[key]): + matching_key = key + break + + if matching_key is None: + continue + + last_histogram = histogram + frame_count = 1 + while len(histograms) > 0: + last_histogram = histograms[0] + if is_key_color_frame(last_histogram, key_colors[matching_key]): + frame_count += 1 + histograms.pop(0) + else: + break + + logging.debug( + "{0:d}ms to {1:d}ms - Matched key color frame {2}".format( + histogram["time"], last_histogram["time"], matching_key + ) + ) + + key_color_frames[matching_key].append( + { + "frame_count": frame_count, + "start_time": histogram["time"], + "end_time": last_histogram["time"], + } + ) + + return key_color_frames + + def calculate_visual_progress(histograms): progress = [] first = histograms[0]["histogram"] @@ -1760,6 +1856,24 @@ def main(): default=False, help="Remove orange-colored frames from the beginning of the video.", ) + parser.add_argument( + "--keycolor", + action="append", + nargs=8, + metavar=( + "key", + "red_low", + "red_high", + "green_low", + "green_high", + "blue_low", + "blue_high", + "fraction", + ), + help="Identify frames that match the given channel (0-255) low and " + "high. Fraction is the percentage of the pixels per channel that " + "must be in the given range (0-1).", + ) parser.add_argument( "-p", "--viewport", @@ -1916,6 +2030,19 @@ def main(): options.full, ) + key_colors = {} + if options.keycolor: + for key_params in options.keycolor: + key_colors[key_params[0]] = { + "r_low": int(key_params[1]), + "r_high": int(key_params[2]), + "g_low": int(key_params[3]), + "g_high": int(key_params[4]), + "b_low": int(key_params[5]), + "b_high": int(key_params[6]), + "fraction": float(key_params[7]), + } + # Calculate the histograms and visual metrics calculate_histograms(directory, histogram_file, options.force) metrics = calculate_visual_metrics( @@ -1927,6 +2054,7 @@ def main(): directory, options.progress, options.herodata, + key_colors, ) if options.screenshot is not None: diff --git a/visualmetrics/visualmetrics.py b/visualmetrics/visualmetrics.py index 4c9709a27..c05b4c7e3 100755 --- a/visualmetrics/visualmetrics.py +++ b/visualmetrics/visualmetrics.py @@ -1339,7 +1339,7 @@ def calculate_histograms(directory, histograms_file, force): m = re.search(match, frame) if m is not None: frame_time = int(m.groupdict().get("ms")) - histogram = calculate_image_histogram(frame) + histogram, total, dropped = calculate_image_histogram(frame) gc.collect() if histogram is not None: histograms.append( @@ -1347,6 +1347,8 @@ def calculate_histograms(directory, histograms_file, force): "time": frame_time, "file": os.path.basename(frame), "histogram": histogram, + "total_pixels": total, + "dropped_pixels": dropped, } ) if os.path.isfile(histograms_file): @@ -1365,12 +1367,14 @@ def calculate_histograms(directory, histograms_file, force): def calculate_image_histogram(file): logging.debug("Calculating histogram for " + file) + dropped = 0 try: from PIL import Image im = Image.open(file) width, height = im.size - colors = im.getcolors(width * height) + total = width * height + colors = im.getcolors(total) histogram = { "r": [0 for i in range(256)], "g": [0 for i in range(256)], @@ -1386,13 +1390,16 @@ def calculate_image_histogram(file): histogram["r"][pixel[0]] += count histogram["g"][pixel[1]] += count histogram["b"][pixel[2]] += count + else: + dropped += 1 except Exception: pass colors = None except Exception: + total = 0 histogram = None logging.exception("Error calculating histogram for " + file) - return histogram + return histogram, total, dropped ########################################################################## @@ -1618,6 +1625,7 @@ def calculate_visual_metrics( dirs, progress_file, hero_elements_file, + key_colors, ): metrics = None histograms = load_histograms(histograms_file, start, end) @@ -1631,6 +1639,7 @@ def calculate_visual_metrics( f = open(progress_file, "w") json.dump(progress, f) f.close() + key_color_frames = calculate_key_color_frames(histograms, key_colors) if len(histograms) > 1: metrics = [ {"name": "First Visual Change", "value": histograms[1]["time"]}, @@ -1720,6 +1729,20 @@ def calculate_visual_metrics( metrics.append({"name": "Perceptual Speed Index", "value": 0}) if contentful: metrics.append({"name": "Contentful Speed Index", "value": 0}) + if key_color_frames: + keysum = "" + for key in key_color_frames: + if len(keysum): + keysum += ", " + framesum = "" + for frame in key_color_frames[key]: + if len(framesum): + framesum += " " + framesum += "{0:d}-{1:d}".format( + frame["start_time"], frame["end_time"] + ) + keysum += "{0}=[{1}]".format(key, framesum) + metrics.append({"name": "Key Color Frames", "value": keysum}) prog = "" for p in progress: if len(prog): @@ -1751,6 +1774,79 @@ def load_histograms(histograms_file, start, end): return histograms +def is_key_color_frame(histogram, key_color): + # The fraction is measured against the entire image, not just the sampled + # pixels. This helps avoid matching frames with only a few pixels that + # happen to be in the acceptable range. + total_fraction = histogram["total_pixels"] * key_color["fraction"] + if total_fraction < histogram["total_pixels"] - histogram["dropped_pixels"]: + for channel in ["r", "g", "b"]: + # Find the acceptable range around the target channel value + max_channel = len(histogram["histogram"][channel]) + low = min(max_channel - 1, max(0, key_color[channel + "_low"])) + high = min(max_channel, max(1, key_color[channel + "_high"] + 1)) + target_total = 0 + for i in histogram["histogram"][channel][low:high]: + target_total += i + if target_total < total_fraction: + return False + return True + + +def calculate_key_color_frames(histograms, key_colors): + if not key_colors: + return {} + + key_color_frames = {} + for key in key_colors: + key_color_frames[key] = [] + + current = None + current_key = None + total = 0 + matched = 0 + buckets = 256 + channels = ["r", "g", "b"] + histograms = histograms.copy() + + while len(histograms) > 0: + histogram = histograms.pop(0) + matching_key = None + for key in key_colors: + if is_key_color_frame(histogram, key_colors[key]): + matching_key = key + break + + if matching_key is None: + continue + + last_histogram = histogram + frame_count = 1 + while len(histograms) > 0: + last_histogram = histograms[0] + if is_key_color_frame(last_histogram, key_colors[matching_key]): + frame_count += 1 + histograms.pop(0) + else: + break + + logging.debug( + "{0:d}ms to {1:d}ms - Matched key color frame {2}".format( + histogram["time"], last_histogram["time"], matching_key + ) + ) + + key_color_frames[matching_key].append( + { + "frame_count": frame_count, + "start_time": histogram["time"], + "end_time": last_histogram["time"], + } + ) + + return key_color_frames + + def calculate_visual_progress(histograms): progress = [] first = histograms[0]["histogram"] @@ -2200,6 +2296,24 @@ def main(): help="Wait for a full white frame after a non-white frame " "at the beginning of the video.", ) + parser.add_argument( + "--keycolor", + action="append", + nargs=8, + metavar=( + "key", + "red_low", + "red_high", + "green_low", + "green_high", + "blue_low", + "blue_high", + "fraction", + ), + help="Identify frames that match the given channel (0-255) low and " + "high. Fraction is the percentage of the pixels per channel that " + "must be in the given range (0-1).", + ) parser.add_argument( "--multiple", action="store_true", @@ -2464,6 +2578,18 @@ def main(): if not options.multiple: if options.render is not None: render_video(directory, options.render) + key_colors = {} + if options.keycolor: + for key_params in options.keycolor: + key_colors[key_params[0]] = { + "r_low": int(key_params[1]), + "r_high": int(key_params[2]), + "g_low": int(key_params[3]), + "g_high": int(key_params[4]), + "b_low": int(key_params[5]), + "b_high": int(key_params[6]), + "fraction": float(key_params[7]), + } # Calculate the histograms and visual metrics calculate_histograms(directory, histogram_file, options.force) @@ -2476,6 +2602,7 @@ def main(): directory, options.progress, options.herodata, + key_colors, ) if options.screenshot is not None: