diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bc07ac..674df60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). - +## Unreleased + +- Add `partition: "measurement"` option for partitioning results table by + measurement (when multiple measurements are in use). ## [0.7.0] 2022-07-15 diff --git a/README.md b/README.md index 4a53845..9e2cbef 100644 --- a/README.md +++ b/README.md @@ -857,7 +857,7 @@ tach http://example.com | `--package-version` / `-p` | _(none)_ | Specify an NPM package version to swap in ([details](#swap-npm-dependencies)) | | `--browser` / `-b` | `chrome` | Which browsers to launch in automatic mode, comma-delimited (chrome, firefox, safari, edge, ie) ([details](#browsers)) | | `--window-size` | `1024,768` | "width,height" in pixels of the browser windows that will be created | -| `--sample-size` / `-n` | `50` | Minimum number of times to run each benchmark ([details](#minimum-sample-size)) | +| `--sample-size` / `-n` | `50` | Minimum number of times to run each benchmark ([details](#minimum-sample-size)) | | `--auto-sample-conditions` | `0%` | The degrees of difference to try and resolve when auto-sampling ("N%" or "Nms", comma-delimited) ([details](#auto-sample-conditions)) | | `--timeout` | `3` | The maximum number of minutes to spend auto-sampling ([details](#auto-sample)) | | `--measure` | `callback` | Which time interval to measure (`callback`, `global`, `fcp`) ([details](#measurement-modes)) | @@ -872,3 +872,4 @@ tach http://example.com | `--trace` | `false` | Enable performance tracing ([details](#performance-traces)) | | `--trace-log-dir` | `${cwd}/logs` | The directory to put tracing log files. Defaults to `${cwd}/logs`. | | `--trace-cat` | [default categories](./src/defaults.ts) | The tracing categories to record. Should be a string of comma-separated category names | +| `--partition` | `"none"` | Use `"measurement"` to partition a single large results table into multiple tables for each [measurement](#measurement-modes). | diff --git a/config.schema.json b/config.schema.json index 41a6619..972f50a 100644 --- a/config.schema.json +++ b/config.schema.json @@ -483,6 +483,10 @@ }, "type": "array" }, + "partition": { + "description": "What to partition the results table by. Use \"measurement\" when multiple\nmeasurements are in use and you want multiple smaller results tables.", + "type": "string" + }, "resolveBareModules": { "description": "Whether to automatically convert ES module imports with bare module\nspecifiers to paths.", "type": "boolean" diff --git a/package.json b/package.json index 5e6d466..9ddc769 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "scripts": { "prepare": "if [ -f './tsconfig.json' ]; then npm run build; fi;", "build": "rimraf lib/ client/lib/ && mkdir lib && npm run generate-json-schema && tsc && tsc -p client/ && npm run lint", + "build:watch": "tsc --watch", "generate-json-schema": "typescript-json-schema tsconfig.json ConfigFile --include src/configfile.ts --required --noExtraProps > config.schema.json", "lint": "eslint .", "format": "prettier --write .", diff --git a/src/config.ts b/src/config.ts index a24f7de..90e42bc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,7 @@ export interface Config { forceCleanNpmInstall: boolean; csvFileStats: string; csvFileRaw: string; + partition: 'none' | 'measurement'; } export async function makeConfig(opts: Opts): Promise { @@ -85,6 +86,9 @@ export async function makeConfig(opts: Opts): Promise { if (opts['window-size'] !== undefined) { throw new Error('--window-size cannot be specified when using --config'); } + if (opts['partition'] !== undefined) { + throw new Error('--partition cannot be specified when using --config'); + } const rawConfigObj = await fsExtra.readJson(opts.config); const validatedConfigObj = await parseConfigFile(rawConfigObj, opts.config); @@ -172,6 +176,8 @@ export function applyDefaults(partial: Partial): Config { : defaults.resolveBareModules, root: partial.root !== undefined ? partial.root : defaults.root, timeout: partial.timeout !== undefined ? partial.timeout : defaults.timeout, + partition: + partial.partition !== undefined ? partial.partition : defaults.partition, }; } diff --git a/src/configfile.ts b/src/configfile.ts index a95315e..d6a321a 100644 --- a/src/configfile.ts +++ b/src/configfile.ts @@ -62,6 +62,12 @@ export interface ConfigFile { */ horizons?: string[]; + /** + * What to partition the results table by. Use "measurement" when multiple + * measurements are in use and you want multiple smaller results tables. + */ + partition?: string; + /** * Benchmarks to run. * @TJS-minItems 1 @@ -380,6 +386,17 @@ export async function parseConfigFile( validated.autoSampleConditions = validated.horizons; } + if (validated.partition !== undefined) { + if ( + validated.partition !== 'measurement' && + validated.partition !== 'none' + ) { + throw new Error( + `The "partition" setting only accepts "measurement" or "none" as an option.` + ); + } + } + return { root, sampleSize: validated.sampleSize, @@ -390,6 +407,7 @@ export async function parseConfigFile( : undefined, benchmarks, resolveBareModules: validated.resolveBareModules, + partition: validated.partition, }; } diff --git a/src/defaults.ts b/src/defaults.ts index 4d698e1..557268b 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -20,6 +20,7 @@ export const mode = 'automatic'; export const resolveBareModules = true; export const forceCleanNpmInstall = false; export const measurementExpression = 'window.tachometerResult'; +export const partition = 'none'; export const traceLogDir = path.join(process.cwd(), 'logs'); export const traceCategories = [ 'blink', diff --git a/src/flags.ts b/src/flags.ts index b3ac716..71eda35 100644 --- a/src/flags.ts +++ b/src/flags.ts @@ -229,6 +229,14 @@ export const optDefs: commandLineUsage.OptionDefinition[] = [ type: String, defaultValue: defaults.traceCategories.join(','), }, + { + name: 'partition', + description: + `What to partition the results table by. Use "measurement" ` + + `when multiple measurements are in use and you want multiple ` + + `smaller results tables.`, + type: String, + }, ]; export interface Opts { @@ -259,6 +267,7 @@ export interface Opts { trace: boolean; 'trace-log-dir': string; 'trace-cat': string; + partition: string; // Extra arguments not associated with a flag are put here. These are our // benchmark names/URLs. diff --git a/src/format.ts b/src/format.ts index 460b778..c112fcd 100644 --- a/src/format.ts +++ b/src/format.ts @@ -16,6 +16,7 @@ import { ResultStatsWithDifferences, } from './stats.js'; import {BenchmarkSpec, BenchmarkResult} from './types.js'; +import {measurementName} from './measure.js'; export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map( (frame) => ansi.format(`[blue]{${frame}}`) @@ -102,6 +103,26 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults { return {fixed: fixedTable, unfixed: unfixedTable}; } +export function partitionResultTableByMeasurement( + results: ResultStatsWithDifferences[] +) { + const collated: {[index: string]: ResultStatsWithDifferences[]} = {}; + results.forEach((result) => { + const meas = measurementName(result.result.measurement); + (collated[meas] || (collated[meas] = [])).push({ + ...result, + differences: result.differences.filter( + (_, i) => measurementName(results[i].result.measurement) === meas + ), + }); + }); + const tables: AutomaticResults[] = []; + for (const results of Object.values(collated)) { + tables.push(automaticResultTable(results)); + } + return tables; +} + /** * Format a terminal text result table where each result is a row: * diff --git a/src/runner.ts b/src/runner.ts index 2af1b15..db9993f 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -33,6 +33,7 @@ import { automaticResultTable, spinner, benchmarkOneLiner, + partitionResultTableByMeasurement, } from './format.js'; import {Config} from './config.js'; import * as github from './github.js'; @@ -420,7 +421,15 @@ export class Runner { console.log(); const {fixed, unfixed} = automaticResultTable(withDifferences); console.log(horizontalTermResultTable(fixed)); - console.log(verticalTermResultTable(unfixed)); + if (config.partition === 'measurement') { + for (const {unfixed} of partitionResultTableByMeasurement( + withDifferences + )) { + console.log(verticalTermResultTable(unfixed)); + } + } else { + console.log(verticalTermResultTable(unfixed)); + } if (hitTimeout === true) { console.log( diff --git a/src/test/config_test.ts b/src/test/config_test.ts index b89cb07..1d185bd 100644 --- a/src/test/config_test.ts +++ b/src/test/config_test.ts @@ -45,6 +45,7 @@ suite('makeConfig', function () { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -90,6 +91,7 @@ suite('makeConfig', function () { csvFileRaw: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -135,6 +137,7 @@ suite('makeConfig', function () { csvFileStats: '', csvFileRaw: '', githubCheck: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -187,6 +190,7 @@ suite('makeConfig', function () { remoteAccessibleHost: '', // TODO(aomarks) Be consistent about undefined vs unset. githubCheck: undefined, + partition: 'none', benchmarks: [ { browser: { @@ -220,6 +224,7 @@ suite('makeConfig', function () { const expected: Config = { mode: 'automatic', csvFileStats: '', + partition: 'none', csvFileRaw: '', jsonFile: '', legacyJsonFile: '', diff --git a/src/test/configfile_test.ts b/src/test/configfile_test.ts index c68e0a2..10b996d 100644 --- a/src/test/configfile_test.ts +++ b/src/test/configfile_test.ts @@ -48,6 +48,7 @@ suite('config', () => { timeout: 7, autoSampleConditions: ['0ms', '1ms', '2%', '+3%'], resolveBareModules: false, + partition: 'measurement', benchmarks: [ { name: 'remote', @@ -101,6 +102,7 @@ suite('config', () => { relative: [-0.02, 0.02, 0.03], }, resolveBareModules: false, + partition: 'measurement', benchmarks: [ { name: 'remote', @@ -184,6 +186,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, + partition: undefined, benchmarks: [ { name: 'http://example.com?foo=bar', @@ -233,6 +236,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, + partition: undefined, resolveBareModules: undefined, benchmarks: [ { @@ -286,6 +290,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, + partition: undefined, benchmarks: [ { name: '/mybench/index.html?foo=a', @@ -365,6 +370,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, + partition: undefined, benchmarks: [ { name: 'http://example.com', @@ -440,6 +446,7 @@ suite('config', () => { timeout: undefined, autoSampleConditions: undefined, resolveBareModules: undefined, + partition: undefined, benchmarks: [ { name: 'http://example.com?foo=bar', @@ -489,6 +496,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, + partition: undefined, resolveBareModules: undefined, benchmarks: [ { @@ -575,6 +583,7 @@ suite('config', () => { sampleSize: undefined, timeout: undefined, autoSampleConditions: undefined, + partition: undefined, resolveBareModules: undefined, benchmarks: [ { diff --git a/src/test/format_test.ts b/src/test/format_test.ts index 874a5b7..511ad78 100644 --- a/src/test/format_test.ts +++ b/src/test/format_test.ts @@ -14,6 +14,7 @@ import { automaticResultTable, horizontalTermResultTable, verticalTermResultTable, + partitionResultTableByMeasurement, } from '../format.js'; import {fakeResults, testData} from './test_helpers.js'; @@ -30,6 +31,22 @@ async function fakeResultTable(configFile: ConfigFile): Promise { ); } +/** + * Given a config file object, generates fake measurement results, and returns + * the partitioned formatted result tables that would be printed (minus color + * etc. formatting). + */ +async function fakePartitionedResultTablesByMeasurement( + configFile: ConfigFile +): Promise { + const results = await fakeResults(configFile); + return partitionResultTableByMeasurement(results) + .map((result) => { + return stripAnsi(verticalTermResultTable(result.unfixed)); + }) + .join('\n'); +} + suite('format', () => { let prevCwd: string; suiteSetup(() => { @@ -209,4 +226,111 @@ suite('format', () => { `; assert.equal(actual, expected.trim() + '\n'); }); + + test('multiple measurements, partition: "none"', async () => { + const config: ConfigFile = { + benchmarks: [ + { + name: 'foo', + url: 'http://foo.com', + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'}, + ], + }, + { + name: 'bar', + url: 'http://bar.com', + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'}, + ], + }, + ], + }; + + const actual = await fakeResultTable(config); + const expected = ` +┌─────────────┬───────────────┐ +│ Version │ │ +├─────────────┼───────────────┤ +│ Browser │ chrome │ +│ │ 75.0.3770.100 │ +├─────────────┼───────────────┤ +│ Sample size │ 50 │ +└─────────────┴───────────────┘ + +┌──────────────┬──────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┐ +│ Benchmark │ Bytes │ Avg time │ vs foo [render] │ vs foo [update] │ vs bar [render] │ vs bar [update] │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ foo [render] │ 1.00 KiB │ 8.56ms - 11.44ms │ │ unsure │ faster │ faster │ +│ │ │ │ - │ -20% - +20% │ 42% - 58% │ 42% - 58% │ +│ │ │ │ │ -2.03ms - +2.03ms │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ foo [update] │ 1.00 KiB │ 8.56ms - 11.44ms │ unsure │ │ faster │ faster │ +│ │ │ │ -20% - +20% │ - │ 42% - 58% │ 42% - 58% │ +│ │ │ │ -2.03ms - +2.03ms │ │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ bar [render] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ slower │ │ unsure │ +│ │ │ │ 68% - 132% │ 68% - 132% │ - │ -10% - +10% │ +│ │ │ │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ │ -2.03ms - +2.03ms │ +├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤ +│ bar [update] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ slower │ unsure │ │ +│ │ │ │ 68% - 132% │ 68% - 132% │ -10% - +10% │ - │ +│ │ │ │ 7.97ms - 12.03ms │ 7.97ms - 12.03ms │ -2.03ms - +2.03ms │ │ +└──────────────┴──────────┴───────────────────┴───────────────────┴───────────────────┴───────────────────┴───────────────────┘ + `; + assert.equal(actual, expected.trim() + '\n'); + }); + + test('multiple measurements, partition: "measurement"', async () => { + const config: ConfigFile = { + benchmarks: [ + { + name: 'foo', + url: 'http://foo.com', + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'}, + ], + }, + { + name: 'bar', + url: 'http://bar.com', + measurement: [ + {name: 'render', mode: 'performance', entryName: 'render'}, + {name: 'update', mode: 'performance', entryName: 'update'}, + ], + }, + ], + }; + + const actual = await fakePartitionedResultTablesByMeasurement(config); + const expected = ` +┌──────────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐ +│ Benchmark │ Bytes │ Avg time │ vs foo [render] │ vs bar [render] │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ foo [render] │ 1.00 KiB │ 8.56ms - 11.44ms │ │ faster │ +│ │ │ │ - │ 42% - 58% │ +│ │ │ │ │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ bar [render] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ │ +│ │ │ │ 68% - 132% │ - │ +│ │ │ │ 7.97ms - 12.03ms │ │ +└──────────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘ + +┌──────────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐ +│ Benchmark │ Bytes │ Avg time │ vs foo [update] │ vs bar [update] │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ foo [update] │ 1.00 KiB │ 8.56ms - 11.44ms │ │ faster │ +│ │ │ │ - │ 42% - 58% │ +│ │ │ │ │ 7.97ms - 12.03ms │ +├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤ +│ bar [update] │ 2.00 KiB │ 18.56ms - 21.44ms │ slower │ │ +│ │ │ │ 68% - 132% │ - │ +│ │ │ │ 7.97ms - 12.03ms │ │ +└──────────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘ + `; + assert.equal(actual, expected.trim() + '\n'); + }); });