Skip to content

Commit efce6e0

Browse files
ig-sinicynig-sinicyn
committed
Perftests:
* updated BenchmarkDotNet to 0.9.6 * validation & logging on errors.
1 parent 2cce095 commit efce6e0

File tree

10 files changed

+94
-26
lines changed

10 files changed

+94
-26
lines changed

‎Experimental/src/RangesV2/RangeInternal.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ internal enum UnsafeOverload
4646
#endregion
4747

4848
#region Range constants
49-
// TODO: uncomment
5049
internal const string KeyPrefixString = "'";
5150
internal const string KeySeparatorString = "': ";
5251
internal const string SeparatorString = "..";

‎Experimental/tests-performance/CodeJam.Experimental-Tests.Performance.csproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@
3737
<AssemblyOriginatorKeyFile>..\..\CodeJam.snk</AssemblyOriginatorKeyFile>
3838
</PropertyGroup>
3939
<ItemGroup>
40-
<Reference Include="BenchmarkDotNet, Version=0.9.5.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
41-
<HintPath>..\..\packages\BenchmarkDotNet.0.9.5\lib\net46\BenchmarkDotNet.dll</HintPath>
40+
<Reference Include="BenchmarkDotNet, Version=0.9.6.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
41+
<HintPath>..\..\packages\BenchmarkDotNet.0.9.6\lib\net46\BenchmarkDotNet.dll</HintPath>
4242
<Private>True</Private>
4343
</Reference>
4444
<Reference Include="Microsoft.Build, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" />

‎Experimental/tests-performance/packages.config‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<packages>
3-
<package id="BenchmarkDotNet" version="0.9.5" targetFramework="net461" />
3+
<package id="BenchmarkDotNet" version="0.9.6" targetFramework="net461" />
44
<package id="NUnit" version="3.2.1" targetFramework="net461" />
55
</packages>

‎Main/tests-performance/BenchmarkDotNet/Core/BenchmarkHelpers.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,11 @@ public static ILookup<KeyValuePair<IJob, ParameterInstances>, Benchmark> SameCon
9393
/// </summary>
9494
public static double GetPercentile(this Summary summary, Benchmark benchmark, double percentileRatio)
9595
{
96-
var values = summary.Reports.Single(r => r.Benchmark == benchmark)
97-
.GetResultRuns()
96+
var benchmarkReport = summary.Reports.SingleOrDefault(r => r.Benchmark == benchmark);
97+
if (benchmarkReport == null)
98+
return 0;
99+
100+
var values = benchmarkReport.GetResultRuns()
98101
.Select(r => r.GetAverageNanoseconds());
99102
return Percentile(values, percentileRatio);
100103
}

‎Main/tests-performance/BenchmarkDotNet/UnitTesting.CompetitionTargets/CompetitionTargetHelpers.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
using BenchmarkDotNet.Reports;
1212
using BenchmarkDotNet.Running;
1313

14+
using CodeJam.Collections;
15+
1416
// ReSharper disable CheckNamespace
1517
using static BenchmarkDotNet.UnitTesting.CompetitionHelpers;
1618

@@ -218,6 +220,8 @@ public static void ValidateSummary(
218220
Summary summary, double defaultMinRatio, double defaultMaxRatio,
219221
CompetitionTargets competitionTargets)
220222
{
223+
ValidatePreconditions(summary);
224+
221225
// Based on 95th percentile
222226
const double percentileRatio = 0.95;
223227

@@ -258,6 +262,40 @@ public static void ValidateSummary(
258262
}
259263
}
260264

265+
private static void ValidatePreconditions(Summary summary)
266+
{
267+
if (summary.HasCriticalValidationErrors)
268+
{
269+
var messages = summary.ValidationErrors
270+
.Where(err => err.IsCritical)
271+
.Select(
272+
err =>
273+
(err.Benchmark == null ? null : err.Benchmark.ShortInfo + ": ") +
274+
err.Message)
275+
.ToArray();
276+
277+
if (messages.Length > 0)
278+
throw new InvalidOperationException(
279+
"Validation failed:\r\n" + string.Join(Environment.NewLine, messages));
280+
281+
throw new InvalidOperationException("Benchmark validation failed.");
282+
}
283+
284+
var benchReports = summary.Reports
285+
.Where(r => r.ExecuteResults.Any())
286+
.Select(r => r.Benchmark)
287+
.ToHashSet();
288+
var benchMissing = summary.Benchmarks
289+
.Where(b => !benchReports.Contains(b))
290+
.Select(b => b.Target.Method.Name)
291+
.Distinct()
292+
.ToArray();
293+
294+
if (benchMissing.Length > 0)
295+
throw new InvalidOperationException(
296+
"No reports for benchmarks: " + string.Join(", ", benchMissing));
297+
}
298+
261299
private static void ValidateBenchmark(
262300
Benchmark benchmark,
263301
double actualRatio, double benchmarkMinRatio, double benchmarkMaxRatio)

‎Main/tests-performance/BenchmarkDotNet/UnitTesting/CompetitionBenchmarkRunner.cs‎

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -81,20 +81,32 @@ private static void RunCompetitionUnderSetup(
8181
// Final config
8282
var competitionConfig = CreateCompetitionConfig(baseConfig, competitionState, logger);
8383

84-
Summary summary = null;
8584
try
8685
{
87-
summary = RunCore(benchmarkType, competitionConfig, competitionState);
86+
Summary summary = null;
87+
try
88+
{
89+
summary = RunCore(benchmarkType, competitionConfig, competitionState);
90+
}
91+
finally
92+
{
93+
DumpOutputSummaryAtTop(summary, logger);
94+
}
95+
96+
competitionState.ValidateSummary(summary, minRatio, maxRatio);
8897
}
89-
finally
98+
catch (Exception ex)
9099
{
91-
DumpOutputSummaryAtTop(summary, logger);
100+
ConsoleLogger.Default.WriteError("Exception:" + ex);
101+
throw;
92102
}
93-
94-
competitionState.ValidateSummary(summary, minRatio, maxRatio);
95103
}
96104

97-
private static void ValidateCompetitionSetup(Type benchmarkType)
105+
private static
106+
void ValidateCompetitionSetup
107+
(
108+
Type
109+
benchmarkType)
98110
{
99111
if (!Debugger.IsAttached)
100112
{
@@ -113,7 +125,9 @@ private static void ValidateCompetitionSetup(Type benchmarkType)
113125
}
114126
}
115127

116-
private static AccumulationLogger InitAccumulationLogger()
128+
private static
129+
AccumulationLogger InitAccumulationLogger
130+
()
117131
{
118132
var logger = new AccumulationLogger();
119133
logger.WriteLine();
@@ -123,12 +137,14 @@ private static AccumulationLogger InitAccumulationLogger()
123137
return logger;
124138
}
125139

126-
private static IConfig CreateCompetitionConfig(
140+
private static
141+
IConfig CreateCompetitionConfig
142+
(
127143
IConfig baseConfig, CompetitionState competitionState,
128144
AccumulationLogger logger)
129145
{
130146
baseConfig = baseConfig ?? DefaultConfig.Instance;
131-
var existingParameters = baseConfig.GetAnalysers()
147+
var existingParameters = baseConfig.GetValidators()
132148
.OfType<CompetitionParameters>()
133149
.SingleOrDefault();
134150

@@ -140,6 +156,7 @@ private static IConfig CreateCompetitionConfig(
140156
result.Add(new CompetitionParameters());
141157
}
142158
result.Add(logger);
159+
result.Add(Validators.JitOptimizationsValidator.FailOnError);
143160
result.Add(
144161
StatisticColumn.Min,
145162
ScaledPercentileColumn.S0Column,
@@ -151,7 +168,9 @@ private static IConfig CreateCompetitionConfig(
151168
return result;
152169
}
153170

154-
private static Summary RunCore(
171+
private static
172+
Summary RunCore
173+
(
155174
Type benchmarkType, IConfig competitionConfig,
156175
CompetitionState competitionState)
157176
{
@@ -176,7 +195,9 @@ private static Summary RunCore(
176195
return summary;
177196
}
178197

179-
private static void DumpOutputSummaryAtTop(Summary summary, AccumulationLogger logger)
198+
private static
199+
void DumpOutputSummaryAtTop
200+
(Summary summary, AccumulationLogger logger)
180201
{
181202
if (summary != null)
182203
{

‎Main/tests-performance/BenchmarkDotNet/UnitTesting/CompetitionHelpers.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ public static void WriteFileContent(string path, string[] lines)
7676

7777
#region Competition benchmarks
7878
public static CompetitionParameters GetCompetitionParameters(this Summary summary) =>
79-
summary.Config.GetAnalysers()
79+
summary.Config.GetValidators()
8080
.OfType<CompetitionParameters>()
8181
.Single();
8282

‎Main/tests-performance/BenchmarkDotNet/UnitTesting/CompetitionParameters.cs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,21 @@
55

66
using BenchmarkDotNet.Analysers;
77
using BenchmarkDotNet.Reports;
8+
using BenchmarkDotNet.Running;
9+
using BenchmarkDotNet.Validators;
810

911
using JetBrains.Annotations;
1012

1113
// ReSharper disable CheckNamespace
1214

1315
namespace BenchmarkDotNet.UnitTesting
1416
{
15-
// TODO: As validator once
16-
// https://github.com/PerfDotNet/BenchmarkDotNet/issues/153
17-
// will be fixed
1817
/// <summary>
1918
/// Competition parameters
2019
/// </summary>
2120
[PublicAPI]
2221
[SuppressMessage("ReSharper", "SuggestVarOrType_BuiltInTypes")]
23-
public class CompetitionParameters : IAnalyser
22+
public class CompetitionParameters : IValidator
2423
{
2524
#region Public API
2625
public bool RerunIfModified { get; set; }
@@ -30,5 +29,12 @@ public class CompetitionParameters : IAnalyser
3029
public IEnumerable<IWarning> Analyse(Summary summary) =>
3130
Enumerable.Empty<IWarning>();
3231
#endregion
32+
33+
#region Implementation of IValidator
34+
IEnumerable<IValidationError> IValidator.Validate(IList<Benchmark> benchmarks) =>
35+
Enumerable.Empty<IValidationError>();
36+
37+
bool IValidator.TreatsWarningsAsErrors => false;
38+
#endregion
3339
}
3440
}

‎Main/tests-performance/CodeJam.Main-Tests.Performance.csproj‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@
3737
<AssemblyOriginatorKeyFile>..\..\CodeJam.snk</AssemblyOriginatorKeyFile>
3838
</PropertyGroup>
3939
<ItemGroup>
40-
<Reference Include="BenchmarkDotNet, Version=0.9.5.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
41-
<HintPath>..\..\packages\BenchmarkDotNet.0.9.5\lib\net46\BenchmarkDotNet.dll</HintPath>
40+
<Reference Include="BenchmarkDotNet, Version=0.9.6.0, Culture=neutral, PublicKeyToken=aa0ca2f9092cefc4, processorArchitecture=MSIL">
41+
<HintPath>..\..\packages\BenchmarkDotNet.0.9.6\lib\net46\BenchmarkDotNet.dll</HintPath>
4242
<Private>True</Private>
4343
</Reference>
4444
<Reference Include="Microsoft.Build, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" />
@@ -56,6 +56,7 @@
5656
<Reference Include="System" />
5757
<Reference Include="System.configuration" />
5858
<Reference Include="System.Core" />
59+
<Reference Include="System.Management" />
5960
<Reference Include="System.Xml" />
6061
<Reference Include="System.Xml.Linq" />
6162
</ItemGroup>

‎Main/tests-performance/packages.config‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<packages>
3-
<package id="BenchmarkDotNet" version="0.9.5" targetFramework="net461" />
3+
<package id="BenchmarkDotNet" version="0.9.6" targetFramework="net461" />
44
<package id="Microsoft.DiaSymReader" version="1.0.7" targetFramework="net452" />
55
<package id="NUnit" version="3.2.1" targetFramework="net452" />
66
</packages>

0 commit comments

Comments
 (0)