Skip to content

Commit

Permalink
First step to a better compare (#4064)
Browse files Browse the repository at this point in the history
  • Loading branch information
@soulgalore
soulgalore committed Feb 11, 2024
1 parent 414870e commit 92b8cf2
Show file tree
Hide file tree
Showing 4 changed files with 263 additions and 157 deletions.
6 changes: 4 additions & 2 deletions lib/plugins/compare/helper.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ function getVisualMetrics(data) {
return results;
}

/*
function getCDPPerformance(data) {
const metricsToKeep = new Set([
'JSEventListeners',
Expand Down Expand Up @@ -261,6 +262,7 @@ function getCDPPerformance(data) {
}
return results;
}
*/

function getCPU(data) {
const cpuMetrics = {
Expand Down Expand Up @@ -351,8 +353,8 @@ export function getMetrics(data) {
...getElementTimings(data),
...getUserTimings(data),
...getCPU(data),
...getBrowserMetrics(data),
...getCDPPerformance(data)
...getBrowserMetrics(data)
// ...getCDPPerformance(data)
};
}

Expand Down
320 changes: 179 additions & 141 deletions lib/plugins/compare/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
}

async open(context, options) {
this.pageXrays = {};
this.browsertimes = {};
this.page = 0;
this.make = context.messageMaker('compare').make;
this.compareOptions = merge({}, defaultConfig, options.compare);
Expand Down Expand Up @@ -93,163 +95,199 @@ export default class ComparePlugin extends SitespeedioPlugin {
break;
}
case 'browsertime.pageSummary': {
this.page++;
const id = this.options.compare.id || urlToId(message.data.info.url);
const baseline = await getBaseline(
id + '-' + this.page,
this.compareOptions
);
if (this.options.compare.id) {
log.info('Using id %s for page baseline', id);
} else {
log.info('Using auto generated id for the baseline: %s ', id);
}
this.browsertimes[message.url] = message;
break;
}
case 'sitespeedio.summarize': {
for (let url of Object.keys(this.browsertimes)) {
this.page++;
const id = this.options.compare.id || urlToId(url);
const baseline = await getBaseline(
id + '-' + this.page,
this.compareOptions
);
if (this.options.compare.id) {
log.info('Using id %s for page baseline', id);
} else {
log.info('Using auto generated id for the baseline: %s ', id);
}

if (baseline) {
if (
baseline &&
this.options.browsertime.iterations !== baseline.timestamps.length
)
log.warning(
'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
baseline.timestamps.length,
this.options.browsertime.iterations
);
log.info('Got a baseline:' + id + '-' + this.page);
const newMetrics = getMetrics(message.data);
const baselineMetrics = getMetrics(baseline);
const metricsInputData = {
options: {
test_type: this.compareOptions.testType,
alternative: this.compareOptions.alternative
},
metrics: {}
};
if (baseline) {
if (
this.options.browsertime.iterations !==
baseline.browsertime.timestamps.length
)
log.warning(
'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
baseline.timestamps.length,
this.options.browsertime.iterations
);
log.info('Got a baseline:' + id + '-' + this.page);
const newMetrics = getMetrics(this.browsertimes[url].data);
const baselineMetrics = getMetrics(baseline.browsertime);
const metricsInputData = {
options: {
test_type: this.compareOptions.testType,
alternative: this.compareOptions.alternative
},
metrics: {}
};

if (this.compareOptions.testType === 'mannwhitneyu') {
metricsInputData.options.use_continuity =
this.compareOptions.mannwhitneyu.useContinuity;
metricsInputData.options.method =
this.compareOptions.mannwhitneyu.method;
metricsInputData.options.nan_policy = 'omit';
} else if (this.compareOptions.testType === 'wilcoxon') {
metricsInputData.options.correction =
this.compareOptions.wilcoxon.correction;
metricsInputData.options.zero_method =
this.compareOptions.wilcoxon.zeroMethod;
}
if (this.compareOptions.testType === 'mannwhitneyu') {
metricsInputData.options.use_continuity =
this.compareOptions.mannwhitneyu.useContinuity;
metricsInputData.options.method =
this.compareOptions.mannwhitneyu.method;
metricsInputData.options.nan_policy = 'omit';
} else if (this.compareOptions.testType === 'wilcoxon') {
metricsInputData.options.correction =
this.compareOptions.wilcoxon.correction;
metricsInputData.options.zero_method =
this.compareOptions.wilcoxon.zeroMethod;
}

for (let group in newMetrics) {
if (baselineMetrics[group]) {
metricsInputData.metrics[group] = {};
for (let metricName in newMetrics[group]) {
// Ensure both current and baseline metrics are available
if (
baselineMetrics[group][metricName] &&
newMetrics[group][metricName]
) {
// Directly access the Metric instance
const currentMetric = newMetrics[group][metricName];
const baselineMetric = baselineMetrics[group][metricName];
for (let group in newMetrics) {
if (baselineMetrics[group]) {
metricsInputData.metrics[group] = {};
for (let metricName in newMetrics[group]) {
// Ensure both current and baseline metrics are available
if (
baselineMetrics[group][metricName] &&
newMetrics[group][metricName]
) {
// Directly access the Metric instance
const currentMetric = newMetrics[group][metricName];
const baselineMetric = baselineMetrics[group][metricName];

// Ensure these are indeed Metric instances
const currentStats = getStatistics(currentMetric.getValues());
const baselineStats = getStatistics(
baselineMetric.getValues()
);
metricsInputData.metrics[group][metricName] = {
baseline: baselineStats.data,
current: currentStats.data
};
} else {
log.info(
`Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.`
);
// Ensure these are indeed Metric instances
const currentStats = getStatistics(
currentMetric.getValues()
);
const baselineStats = getStatistics(
baselineMetric.getValues()
);
metricsInputData.metrics[group][metricName] = {
baseline: baselineStats.data,
current: currentStats.data
};
} else {
log.info(
`Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.`
);
}
}
}
}
}

const results = await runStatisticalTests(metricsInputData);
const finalResult = {};
for (let group in results) {
finalResult[group] = {};
for (let metricName in results[group]) {
const result = results[group][metricName];
// Again, accessing the metricName within the group
const currentStats = getStatistics(
newMetrics[group][metricName].getValues()
);
const baselineStats = getStatistics(
baselineMetrics[group][metricName].getValues()
);
const results = await runStatisticalTests(metricsInputData);
const finalResult = {};
for (let group in results) {
finalResult[group] = {};
for (let metricName in results[group]) {
const result = results[group][metricName];
// Again, accessing the metricName within the group
const currentStats = getStatistics(
newMetrics[group][metricName].getValues()
);
const baselineStats = getStatistics(
baselineMetrics[group][metricName].getValues()
);

const cliffs = cliffsDelta(currentStats.data, baselineStats.data);
finalResult[group][metricName] = {
current: {
stdev: currentStats.stddev(),
mean: currentStats.amean(),
median: currentStats.median(),
values: currentStats.data
},
baseline: {
stdev: baselineStats.stddev(),
mean: baselineStats.amean(),
median: baselineStats.median(),
values: baselineStats.data
},
statisticalTestU: result['p-value'],
cliffsDelta: cliffs,
isSignificant: getIsSignificant(result['p-value'], cliffs)
};
const cliffs = cliffsDelta(
currentStats.data,
baselineStats.data
);
finalResult[group][metricName] = {
current: {
stdev: currentStats.stddev(),
mean: currentStats.amean(),
median: currentStats.median(),
values: currentStats.data
},
baseline: {
stdev: baselineStats.stddev(),
mean: baselineStats.amean(),
median: baselineStats.median(),
values: baselineStats.data
},
statisticalTestU: result['p-value'],
cliffsDelta: cliffs,
isSignificant: getIsSignificant(result['p-value'], cliffs)
};
}
}
}
const meta = {
baseline: {
timestamp: dayjs(baseline.info.timestamp).format(TIME_FORMAT),
url: baseline.info.url,
alias: baseline.info.alias
},
current: {
timestamp: dayjs(message.data.info.timestamp).format(TIME_FORMAT),
url: message.data.info.url,
alias: message.data.info.alias
},
testOptions: this.compareOptions,
iterations: this.options.browsertime.iterations
};

if (this.compareOptions.saveBaseline) {
await saveBaseline(
message.data,
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
)
);
}
const meta = {
baseline: {
timestamp: dayjs(baseline.browsertime.info.timestamp).format(
TIME_FORMAT
),
url: baseline.browsertime.info.url,
alias: baseline.browsertime.info.alias
},
current: {
timestamp: dayjs(
this.browsertimes[url].data.info.timestamp
).format(TIME_FORMAT),
url: url,
alias: this.browsertimes[url].data.info.alias
},
testOptions: this.compareOptions,
iterations: this.options.browsertime.iterations
};

const raw = {
baseline: {
pagexray: baseline.pagexray,
browsertime: baseline.browsertime
},
current: {
pagexray: this.pageXrays[url].data,
browsertime: this.browsertimes[url].data
}
};

super.sendMessage(
'compare.pageSummary',
{ metrics: finalResult, meta },
{
url: message.url,
group: message.group,
runTime: message.runTime
if (this.compareOptions.saveBaseline) {
await saveBaseline(
{
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
)
);
}
);
} else {
if (this.compareOptions.saveBaseline) {
await saveBaseline(
message.data,
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
)

super.sendMessage(
'compare.pageSummary',
{ metrics: finalResult, meta, raw },
{
url: url,
group: this.browsertimes[url].group,
runTime: this.browsertimes[url].runTime
}
);
} else {
if (this.compareOptions.saveBaseline) {
await saveBaseline(
{
browsertime: this.browsertimes[url].data,
pagexray: this.pageXrays[url].data
},
join(
this.compareOptions.baselinePath || process.cwd(),
`${id}-${this.page}.json`
)
);
}
}
}

break;
}
case 'pagexray.pageSummary': {
this.pageXrays[message.url] = message;
break;
}
}
Expand Down
Loading

0 comments on commit 92b8cf2

Please sign in to comment.