diff --git a/README.md b/README.md index 45b003e..23ef98a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,82 @@ -# BenchView -A solution to view and keep track of benchmarking results in C# +# GlassView + +A solution to view and keep track of benchmarking results of dotnet projects. + +This is still a project very much in it's infancy. But when it matures, it is supposed to export [BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet) results such that they can be tracked over time. This should allow teams to keep track of the performance characteristics of a project, similarly to keeping track of the test & coverage results as well as code quality metrics. + +## Vision + +The end product should be some interactive web interface that can run "anywhere" which enables analysis of your benchmark results. + +- [ ] Export benchmark results as json to some local directory. + - [x] Via a library (ToDo #12: nuget pending) + - [ ] Completely decoupled without any dll dependencies +- [ ] Ingestion into some service(?) that creates (static) views. +- [ ] Interactive web interface. +- [ ] IDE integration (Visual Studio Code)? +- [ ] Statistics suite with which, + - [ ] simple queries against past runs can be issued. + - [ ] regressions and trends can be detected and visualised +- [ ] Integration into Ci/Cd systems (pipelines) + +For a complete and more detailed list, please check out our [issues](https://github.com/atmoos/GlassView/issues). + +## Current Set-Up + +Reference the nuget package `Atmoos.GlassView.Export` and modify your benchmarking project as follows: + +```csharp +/* in: YourProject/Program.cs */ +// other usings... +using Atmoos.GlassView.Export; +using Microsoft.Extensions.Configuration; + +var configuration = new ConfigurationBuilder().AddJsonFile("appsettings.json").Build(); + +IGlassView exporter = GlassView.Configure(configuration); + +// dotnet run -c Release --project 'YourProject/' +var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + +await exporter.Export(summary); +``` + +## Configuration + +[Configuration](https://github.com/atmoos/GlassView/blob/main/source/GlassView.Export/Configuration/Configuration.cs) allows for optional configuration of: + +- The export directory. +- Json formatting options. + +Without any configuration, the benchmark results are exported to `./BenchmarkDotNet.Artifacts/GlassView/`. + +A minimal `GlassView` configuration section might look something like this: + +```json +{ + "GlassView": { + "Export": { + "Directory": { + "Path": "Your/Export/Directory/" + } + } + } +} +``` + +## Quick Start + +If you just want to get going and manually want to check for exported data, you can do so by using a `Program.cs` similar to this: + +```csharp +/* in: YourProject/Program.cs */ +// other usings... +using Atmoos.GlassView.Export; + +IGlassView exporter = GlassView.Default(); // exports to ./BenchmarkDotNet.Artifacts/GlassView/ + +// dotnet run -c Release --project 'YourProject/' +var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + +await exporter.Export(summary); +``` diff --git a/documentation/Design.md b/documentation/Design.md index 2a1f86a..47b1994 100644 --- a/documentation/Design.md +++ b/documentation/Design.md @@ -1,4 +1,37 @@ # BenchView Design -**ToDo** +```mermaid +--- +title: High Level Dependency Graph +--- +classDiagram + Core <.. Services + Core <.. Export + Export <.. Benchmark + Library <.. Benchmark + namespace AtmoosGlassView { + class Core{ + + Models + +Serialization() + } + class Services{ + +Run(config) + } + class Export{ + +Export(summary, config) + } + } + namespace User { + class Library{ + + Shared + } + class Benchmark{ + ~RunBenchmark() + } + } + note for Core "Library of shared\ntypes between Export\nand Services" + note for Services "Any service needed\nto realize tracking of\nbenchmarks." + note for Export "User configures how the\nexport of their benchmarks\nis to happen. Configured\nvia appsettings.json" + note for Benchmark "User benchmarking project of Library." +``` diff --git a/source/GlassView.Benchmark/ComplexBenchmark.cs b/source/GlassView.Benchmark/ComplexBenchmark.cs new file mode 100644 index 0000000..8169d27 --- /dev/null +++ b/source/GlassView.Benchmark/ComplexBenchmark.cs @@ -0,0 +1,83 @@ +using System.Numerics; +using BenchmarkDotNet.Attributes; + +namespace Atmoos.GlassView.Benchmark; + +/* This is only a dummy benchmark for us to play around with exports. */ + +[ShortRunJob] +[IterationCount(7)] +[MemoryDiagnoser] +[BenchmarkCategory("Complex")] +public class ComplexBenchmark +{ + private Double[][] left, right; + [Params(54, 42)] + public Int32 Rows { get; set; } + [Params(12, 34, 123)] + public Int32 Center { get; set; } + + [Params(21, 76)] + public Int32 Cols { get; set; } + + [GlobalSetup] + public void Setup() + { + this.left = Matrix(Rows, Center); + // As they're arrays, the dimensions looks the wrong + // way round on the right hand side matrix... + this.right = Matrix(Cols, Center); + } + + + [Benchmark(Baseline = true), BenchmarkCategory("Vectorised")] + public Object VectorisedMultiplication() => MatrixProduct(this.left, this.right, VectorisedDotProduct); + + [Benchmark, BenchmarkCategory("Regular")] + public Object RegularMultiplication() => MatrixProduct(this.left, this.right, DotProduct); + + private static Double[] Vec(Int32 count) + => Enumerable.Range(0, count).Select(i => i - count / 2d).ToArray(); + private static Double[][] Matrix(Int32 rows, Int32 cols) + => Enumerable.Range(0, rows).Select(_ => Vec(cols)).ToArray(); + + + public static Double DotProduct(Double[] left, Double[] right) + { + var sum = 0d; + for (Int32 i = 0; i < left.Length; ++i) { + sum += left[i] * right[i]; + } + return sum; + } + + public static Double VectorisedDotProduct(Double[] left, Double[] right) + { + Int32 index = 0; + var sum = Vector.Zero; + var stride = Vector.Count; + + for (; index < left.Length - stride; index += stride) { + sum += new Vector(left, index) * new Vector(right, index); + } + Double finalSum = Vector.Sum(sum); + for (; index < left.Length; ++index) { + finalSum += left[index] * right[index]; + } + return finalSum; + } + + public static Double[][] MatrixProduct(Double[][] left, Double[][] right, Func dotProduct) + { + var result = new Double[left.Length][]; + for (var row = 0; row < left.Length; ++row) { + var lRow = left[row]; + var resultRow = new Double[right.Length]; + for (var col = 0; col < right.Length; ++col) { + resultRow[col] = dotProduct(lRow, right[col]); + } + result[row] = resultRow; + } + return result; + } +} diff --git a/source/GlassView.Benchmark/GlassView.Benchmark.csproj b/source/GlassView.Benchmark/GlassView.Benchmark.csproj new file mode 100644 index 0000000..c84035f --- /dev/null +++ b/source/GlassView.Benchmark/GlassView.Benchmark.csproj @@ -0,0 +1,35 @@ + + + + + Exe + + + + AnyCPU + pdbonly + true + true + true + Release + false + + disable + + + + + + + + + + + + + + + + + + diff --git a/source/GlassView.Benchmark/Program.cs b/source/GlassView.Benchmark/Program.cs new file mode 100644 index 0000000..99cca82 --- /dev/null +++ b/source/GlassView.Benchmark/Program.cs @@ -0,0 +1,20 @@ +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; +using Microsoft.Extensions.Configuration; +using Atmoos.GlassView.Export; + +using static BenchmarkDotNet.Columns.StatisticColumn; + +var configuration = new ConfigurationBuilder() + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: false) + .Build(); + +IGlassView exporter = GlassView.Configure(configuration); + +// dotnet run -c Release --project GlassView.Benchmark/ +var config = DefaultConfig.Instance.HideColumns(StdDev, Median, Kurtosis, BaselineRatioColumn.RatioStdDev); + +var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); + +await exporter.Export(summary).ConfigureAwait(ConfigureAwaitOptions.None); diff --git a/source/GlassView.Benchmark/SimpleBenchmark.cs b/source/GlassView.Benchmark/SimpleBenchmark.cs new file mode 100644 index 0000000..22537f2 --- /dev/null +++ b/source/GlassView.Benchmark/SimpleBenchmark.cs @@ -0,0 +1,22 @@ +using BenchmarkDotNet.Attributes; + +namespace Atmoos.GlassView.Benchmark; + +/* This is only a dummy benchmark for us to play around with exports. */ + +[ShortRunJob] +[IterationCount(7)] +// So simple, that I don't even add a category... +public class SimpleBenchmark +{ + private static readonly String[] strings = Enumerable.Range(0, 321).Select(i => i.ToString()).ToArray(); + + [Benchmark] + public String ConcatStrings() + => HorribleStringConcatenationNeverEverUseThis(strings); + + private static String HorribleStringConcatenationNeverEverUseThis(IEnumerable strings) + { + return strings.Aggregate(String.Empty, (a, b) => a + b); + } +} diff --git a/source/GlassView.Benchmark/TestBenchmark.cs b/source/GlassView.Benchmark/TestBenchmark.cs new file mode 100644 index 0000000..9aaa205 --- /dev/null +++ b/source/GlassView.Benchmark/TestBenchmark.cs @@ -0,0 +1,23 @@ +using BenchmarkDotNet.Attributes; + +namespace Atmoos.GlassView.Benchmark; + +/* This is only a dummy benchmark for us to play around with exports. */ + +[ShortRunJob] +[IterationCount(7)] +[MemoryDiagnoser] +[BenchmarkCategory("Regular")] +public class TestBenchmark +{ + [Params(54, 42)] + public Int32 Count { get; set; } + + [Benchmark(Baseline = true), BenchmarkCategory("SomeCategory")] + public Int32 SumIntegers() => Integers(Count).Sum(); + + [Benchmark, BenchmarkCategory("OtherCategory")] + public Double SumDoubles() => Doubles(Count).Sum(); + private static Double[] Doubles(Int32 count) => Enumerable.Range(0, count).Select(i => i - count / 2d).ToArray(); + private static Int32[] Integers(Int32 count) => Enumerable.Range(0, count).Select(i => i - count / 2).ToArray(); +} diff --git a/source/GlassView.Benchmark/appsettings.json b/source/GlassView.Benchmark/appsettings.json new file mode 100644 index 0000000..5455888 --- /dev/null +++ b/source/GlassView.Benchmark/appsettings.json @@ -0,0 +1,12 @@ +{ + "GlassView": { + "Export": { + "JsonFormatting": { + "Indented": true + }, + "Directory": { + "Path": "BenchmarkDotNet.Artifacts/GlassView/" + } + } + } +} diff --git a/source/GlassView.Build.props b/source/GlassView.Build.props new file mode 100644 index 0000000..f7f6209 --- /dev/null +++ b/source/GlassView.Build.props @@ -0,0 +1,13 @@ + + + + net8.0 + enable + enable + + + + + + + diff --git a/source/GlassView.Core.Test/Convenience.cs b/source/GlassView.Core.Test/Convenience.cs new file mode 100644 index 0000000..2765668 --- /dev/null +++ b/source/GlassView.Core.Test/Convenience.cs @@ -0,0 +1,17 @@ +using System.Text.Json; + +namespace Atmoos.GlassView.Core.Test; + +public static class Convenience +{ + private static readonly JsonSerializerOptions options = new JsonSerializerOptions() { WriteIndented = true }.EnableGlassView(); + + public static String Serialize(this T value) => JsonSerializer.Serialize(value, options); + public static T Deserialize(this String json) => JsonSerializer.Deserialize(json, options) ?? throw new ArgumentException("Deserialization failed.", nameof(json)); + public static T Deserialize(this FileInfo json) => json.Deserialize(options); + public static T Deserialize(this FileInfo json, JsonSerializerOptions options) + { + using var stream = json.OpenRead(); + return JsonSerializer.Deserialize(stream, options) ?? throw new ArgumentException("Deserialization failed.", nameof(json)); + } +} diff --git a/source/GlassView.Core.Test/GlassView.Core.Test.csproj b/source/GlassView.Core.Test/GlassView.Core.Test.csproj new file mode 100644 index 0000000..79da19c --- /dev/null +++ b/source/GlassView.Core.Test/GlassView.Core.Test.csproj @@ -0,0 +1,17 @@ + + + + + + Atmoos.GlassView.Core.Test + + + + + + + + + + + diff --git a/source/GlassView.Core.Test/GlobalUsings.cs b/source/GlassView.Core.Test/GlobalUsings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/source/GlassView.Core.Test/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/source/GlassView.Core.Test/Resources/TestBenchmark.json b/source/GlassView.Core.Test/Resources/TestBenchmark.json new file mode 100644 index 0000000..522379d --- /dev/null +++ b/source/GlassView.Core.Test/Resources/TestBenchmark.json @@ -0,0 +1,137 @@ +{ + "Name": "TestBenchmark", + "Count": 4, + "Namespace": "Atmoos.GlassView.Benchmark", + "Timestamp": "2024-05-21T17:16:44Z", + "Duration": "00:00:36.9737189", + "Environment": { + "OsVersion": "Arch Linux", + "Dotnet": { + "HasRyuJit": true, + "BuildConfig": "RELEASE", + "DotNetVersion": "8.0.105", + "HasAttachedDebugger": false + }, + "Processor": { + "Name": "Intel(R) Core(TM) i7-8565U CPU @ 1.80GHz", + "Count": 4, + "Architecture": "X64", + "PhysicalCoreCount": 4, + "LogicalCoreCount": 8 + } + }, + "BenchmarkCases": [ + { + "Name": "SumDoubles", + "IsBaseline": false, + "Categories": [ + "OtherCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "42", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 161.38355009896415, + "Median": 161.19704914093018, + "StandardDeviation": 1.8536628217100735, + "SampleSize": 7 + }, + "Allocation": { + "Gen0Collections": 537, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 536 + } + }, + { + "Name": "SumIntegers", + "IsBaseline": true, + "Categories": [ + "SomeCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "42", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 120.6814014116923, + "Median": 120.78095936775208, + "StandardDeviation": 0.7329357120629901, + "SampleSize": 6 + }, + "Allocation": { + "Gen0Collections": 369, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 368 + } + }, + { + "Name": "SumDoubles", + "IsBaseline": false, + "Categories": [ + "OtherCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "54", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 190.1119761126382, + "Median": 191.5717384815216, + "StandardDeviation": 3.343227140810547, + "SampleSize": 7 + }, + "Allocation": { + "Gen0Collections": 633, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 632 + } + }, + { + "Name": "SumIntegers", + "IsBaseline": true, + "Categories": [ + "SomeCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "54", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 137.59001200539726, + "Median": 137.89618372917175, + "StandardDeviation": 1.0305874559096901, + "SampleSize": 7 + }, + "Allocation": { + "Gen0Collections": 417, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 416 + } + } + ] +} \ No newline at end of file diff --git a/source/GlassView.Core.Test/Serialization/SerializationTest.cs b/source/GlassView.Core.Test/Serialization/SerializationTest.cs new file mode 100644 index 0000000..b29b7c8 --- /dev/null +++ b/source/GlassView.Core.Test/Serialization/SerializationTest.cs @@ -0,0 +1,93 @@ +using System.Text.Json; +using Atmoos.GlassView.Core.Models; + +namespace Atmoos.GlassView.Core.Test; + +public class SerializationTest +{ + private static readonly FileInfo testFile = new(Path.Combine("Resources", "TestBenchmark.json")); + + [Fact] + // When issue #10 is resolved, we can remove the call to EnableGlassView(). + public void DeserializationIsPossibleAndDependsOnEnablingGlassView() + { + var options = new JsonSerializerOptions().EnableGlassView(); // required! + BenchmarkSummary actual = testFile.Deserialize(options); + + // The number of cases in the summary could differ if the json was tampered with (not relevant here) + // or deserialization doesn't pick up the case child nodes (which is of relevance here). +#pragma warning disable CA1829 // Use Length/Count property instead of Count() when available + Assert.Equal(actual.Count, actual.Count()); +#pragma warning restore CA1829 // Use Length/Count property instead of Count() when available + } + + /// + /// Values that are sensitive to localisation may change during serialization. + /// This test is a sanity check to ensure that serialization does not have any side effects + /// on the values of the BenchmarkSummary. + /// + /// This far from exhaustive or complete. Making that happen would be fairly involved and + /// is not really relevant for now. + /// e.g: This test was sufficient to pick up the sensitivity to localisation of the time stamp property, + /// which we have since fixed to being UTC. + [Fact] + public void RoundRobinSerializationDoesNotChangeAnyValues() + { + String expected = File.ReadAllText(testFile.FullName); + + // First step: Deserialize + BenchmarkSummary benchmarks = expected.Deserialize(); + + // Second step: Serialize + String actual = benchmarks.Serialize(); + + // Round robin comparison is on strings, where we are only interested in the content (values). + Assert.Equal(expected, actual, ignoreAllWhiteSpace: true, ignoreLineEndingDifferences: true); + } + + /// + /// Similar reasoning to . + /// This time it's just the other way around. (within reason). + /// + [Fact] + public void RoundRobinDeserializationDoesNotChangeAnyValues() + { + BenchmarkSummary expected = testFile.Deserialize(); + + // First step: Serialize + String serializedSummary = expected.Serialize(); + + // Second step: Deserialize + BenchmarkSummary actual = serializedSummary.Deserialize(); + + // Round robin comparison is on the summary object this time :-) + AssertEqual(expected, actual); + } + + // FYI: Although BenchmarkSummary is a record, the collection like properties won't + // transitively compare by value. Hence, we need to do a manual comparison. + // See here: https://stackoverflow.com/questions/63813872/record-types-with-collection-properties-collections-with-value-semantics + private static void AssertEqual(BenchmarkSummary expected, BenchmarkSummary actual) + { + Assert.Equal(expected.Name, actual.Name); + Assert.Equal(expected.Count, actual.Count); + Assert.Equal(expected.Namespace, actual.Namespace); + Assert.Equal(expected.Timestamp, actual.Timestamp); + Assert.Equal(expected.Duration, actual.Duration); + Assert.Equal(expected.Environment, actual.Environment); + + foreach (var (expectedCase, actualCase) in expected.Zip(actual)) { + Equal(expectedCase, actualCase); + } + + static void Equal(BenchmarkCase expected, BenchmarkCase actual) + { + Assert.Equal(expected.Name, actual.Name); + Assert.Equal(expected.IsBaseline, actual.IsBaseline); + Assert.Equal(expected.Categories, actual.Categories); + Assert.Equal(expected.Parameters, actual.Parameters); + Assert.Equal(expected.Statistics, actual.Statistics); + Assert.Equal(expected.Allocation, actual.Allocation); + } + } +} diff --git a/source/GlassView.Core/Extensions.cs b/source/GlassView.Core/Extensions.cs new file mode 100644 index 0000000..20c86f1 --- /dev/null +++ b/source/GlassView.Core/Extensions.cs @@ -0,0 +1,14 @@ +using System.Text.Json; +using Atmoos.GlassView.Core.Serialization; + +namespace Atmoos.GlassView.Core; + +public static class Extensions +{ + public static JsonSerializerOptions EnableGlassView(this JsonSerializerOptions options) + { + options.IgnoreReadOnlyProperties = true; + options.Converters.Add(new BenchmarkSerializer()); + return options; + } +} diff --git a/source/GlassView.Core/GlassView.Core.csproj b/source/GlassView.Core/GlassView.Core.csproj new file mode 100644 index 0000000..43b7ec4 --- /dev/null +++ b/source/GlassView.Core/GlassView.Core.csproj @@ -0,0 +1,8 @@ + + + + + Atmoos.GlassView.Core + + + diff --git a/source/GlassView.Core/ICountable.cs b/source/GlassView.Core/ICountable.cs new file mode 100644 index 0000000..ecd64bb --- /dev/null +++ b/source/GlassView.Core/ICountable.cs @@ -0,0 +1,9 @@ +using System.Collections; + +namespace Atmoos.GlassView.Core; + +public interface ICountable : IEnumerable +{ + Int32 Count { get; } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} diff --git a/source/GlassView.Core/IName.cs b/source/GlassView.Core/IName.cs new file mode 100644 index 0000000..95f8604 --- /dev/null +++ b/source/GlassView.Core/IName.cs @@ -0,0 +1,6 @@ +namespace Atmoos.GlassView.Core; + +public interface IName +{ + String Name { get; } +} diff --git a/source/GlassView.Core/Models/AllocationInfo.cs b/source/GlassView.Core/Models/AllocationInfo.cs new file mode 100644 index 0000000..76dec46 --- /dev/null +++ b/source/GlassView.Core/Models/AllocationInfo.cs @@ -0,0 +1,9 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class AllocationInfo +{ + public required Int32 Gen0Collections { get; init; } + public required Int32 Gen1Collections { get; init; } + public required Int32 Gen2Collections { get; init; } + public required Int64 AllocatedBytes { get; init; } +} diff --git a/source/GlassView.Core/Models/BenchmarkCase.cs b/source/GlassView.Core/Models/BenchmarkCase.cs new file mode 100644 index 0000000..0e5cb89 --- /dev/null +++ b/source/GlassView.Core/Models/BenchmarkCase.cs @@ -0,0 +1,12 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class BenchmarkCase : IName +{ + public required String Name { get; init; } + public required Boolean IsBaseline { get; init; } + public required String[] Categories { get; init; } + public required Parameter[] Parameters { get; init; } + public required String HardwareIntrinsics { get; init; } + public required StatisticsInfo Statistics { get; init; } + public required AllocationInfo Allocation { get; init; } +} diff --git a/source/GlassView.Core/Models/BenchmarkSummary.cs b/source/GlassView.Core/Models/BenchmarkSummary.cs new file mode 100644 index 0000000..9c60769 --- /dev/null +++ b/source/GlassView.Core/Models/BenchmarkSummary.cs @@ -0,0 +1,12 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class BenchmarkSummary(IEnumerable cases) : IName, ICountable +{ + public required String Name { get; init; } + public required Int32 Count { get; init; } + public required String Namespace { get; init; } + public required DateTime Timestamp { get; init; } + public required TimeSpan Duration { get; init; } + public required EnvironmentInfo Environment { get; init; } + public IEnumerator GetEnumerator() => cases.GetEnumerator(); +} diff --git a/source/GlassView.Core/Models/DotnetInfo.cs b/source/GlassView.Core/Models/DotnetInfo.cs new file mode 100644 index 0000000..d21f8c7 --- /dev/null +++ b/source/GlassView.Core/Models/DotnetInfo.cs @@ -0,0 +1,9 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class DotnetInfo +{ + public required Boolean HasRyuJit { get; init; } + public required String BuildConfig { get; init; } + public required String DotNetVersion { get; init; } + public required Boolean HasAttachedDebugger { get; init; } +} diff --git a/source/GlassView.Core/Models/EnvironmentInfo.cs b/source/GlassView.Core/Models/EnvironmentInfo.cs new file mode 100644 index 0000000..128c4de --- /dev/null +++ b/source/GlassView.Core/Models/EnvironmentInfo.cs @@ -0,0 +1,8 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class EnvironmentInfo +{ + public required String OsVersion { get; init; } + public required DotnetInfo Dotnet { get; init; } + public required ProcessorInfo Processor { get; init; } +} diff --git a/source/GlassView.Core/Models/Parameter.cs b/source/GlassView.Core/Models/Parameter.cs new file mode 100644 index 0000000..b4c21ae --- /dev/null +++ b/source/GlassView.Core/Models/Parameter.cs @@ -0,0 +1,8 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class Parameter +{ + public required String Name { get; init; } + public required String Value { get; init; } + public required String Type { get; init; } +} diff --git a/source/GlassView.Core/Models/ProcessorInfo.cs b/source/GlassView.Core/Models/ProcessorInfo.cs new file mode 100644 index 0000000..d543eff --- /dev/null +++ b/source/GlassView.Core/Models/ProcessorInfo.cs @@ -0,0 +1,10 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class ProcessorInfo : IName +{ + public required String Name { get; set; } + public Int32 Count { get; set; } + public required String Architecture { get; set; } + public Int32 PhysicalCoreCount { get; set; } + public Int32 LogicalCoreCount { get; set; } +} diff --git a/source/GlassView.Core/Models/StatisticsInfo.cs b/source/GlassView.Core/Models/StatisticsInfo.cs new file mode 100644 index 0000000..808176c --- /dev/null +++ b/source/GlassView.Core/Models/StatisticsInfo.cs @@ -0,0 +1,10 @@ +namespace Atmoos.GlassView.Core.Models; + +public sealed record class StatisticsInfo +{ + public required Double Mean { get; init; } + public required Double Median { get; init; } + public Double StandardError => StandardDeviation / Math.Sqrt(SampleSize); + public required Double StandardDeviation { get; init; } + public required Int32 SampleSize { get; init; } +} diff --git a/source/GlassView.Core/Serialization/BenchmarkSerializer.cs b/source/GlassView.Core/Serialization/BenchmarkSerializer.cs new file mode 100644 index 0000000..a553288 --- /dev/null +++ b/source/GlassView.Core/Serialization/BenchmarkSerializer.cs @@ -0,0 +1,53 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Atmoos.GlassView.Core.Models; + +namespace Atmoos.GlassView.Core.Serialization; + +// ToDo #10: Remove when upstream issue is resolved. +// --> https://github.com/dotnet/runtime/issues/63791 +internal sealed class BenchmarkSerializer : JsonConverter +{ + public override void Write(Utf8JsonWriter writer, BenchmarkSummary value, JsonSerializerOptions options) + { + var summary = new MutableBenchmarkSummary { + Name = value.Name, + Count = value.Count, + Namespace = value.Namespace, + Timestamp = value.Timestamp, + Duration = value.Duration, + Environment = value.Environment, + BenchmarkCases = [.. value], + }; + JsonSerializer.Serialize(writer, summary, options); + } + + public override BenchmarkSummary? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var summary = JsonSerializer.Deserialize(ref reader, options) ?? throw new ArgumentException("Deserialization failed."); + if (summary.Count != summary.BenchmarkCases.Count) { + throw new ArgumentException($"Expected '{summary.Count}' benchmark cases, but found '{summary.BenchmarkCases.Count}' instead."); + } + + return new BenchmarkSummary(summary.BenchmarkCases) { + Name = summary.Name, + Count = summary.Count, + Namespace = summary.Namespace, + Timestamp = summary.Timestamp, + Duration = summary.Duration, + Environment = summary.Environment, + }; + } +} + +// ToDo #10: Delete this convenience class. (Same as above) +file sealed class MutableBenchmarkSummary : IName +{ + public String Name { get; set; } = String.Empty; + public Int32 Count { get; set; } = 0; + public String Namespace { get; set; } = String.Empty; + public DateTime Timestamp { get; set; } = DateTime.MinValue; + public TimeSpan Duration { get; set; } = TimeSpan.Zero; + public EnvironmentInfo Environment { get; set; } = default!; + public List BenchmarkCases { get; set; } = []; +} diff --git a/source/GlassView.Core/readme.md b/source/GlassView.Core/readme.md new file mode 100644 index 0000000..86009ef --- /dev/null +++ b/source/GlassView.Core/readme.md @@ -0,0 +1,9 @@ +# GlassView Core + +Contains abstract business logic for GlassView. + +- interfaces +- models +- tooling + +Ideally, there is no dependency to [Benchmark.Net](https://benchmarkdotnet.org/). diff --git a/source/GlassView.Export.Test/Convenience.cs b/source/GlassView.Export.Test/Convenience.cs new file mode 100644 index 0000000..51d8b67 --- /dev/null +++ b/source/GlassView.Export.Test/Convenience.cs @@ -0,0 +1,18 @@ +using System.Text.Json; + +namespace Atmoos.GlassView.Export.Test; + +internal static class Convenience +{ + public static T Deserialize(String text, JsonSerializerOptions options) + { + return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException("Failed deserializing a string."); + } + + public static T Deserialize(this FileInfo file, JsonSerializerOptions options) + { + using var stream = file.OpenRead(); + return JsonSerializer.Deserialize(stream, options) ?? throw new InvalidOperationException($"Failed deserializing file '{file.Name}'."); + } +} + diff --git a/source/GlassView.Export.Test/DirectoryExportTest.cs b/source/GlassView.Export.Test/DirectoryExportTest.cs new file mode 100644 index 0000000..e3b244e --- /dev/null +++ b/source/GlassView.Export.Test/DirectoryExportTest.cs @@ -0,0 +1,45 @@ +using System.Text.Json; +using BenchmarkDotNet.Loggers; +using Atmoos.GlassView.Core; +using Atmoos.GlassView.Core.Models; + +namespace Atmoos.GlassView.Export.Test; + +public class DirectoryExportTest +{ + private static readonly FileInfo testFile = new(Path.Combine("Resources", "TestBenchmark.json")); + + [Fact] + public async Task ExportWritesJsonFile() + { + var options = new JsonSerializerOptions().EnableGlassView(); + var summary = testFile.Deserialize(options); + var emptyDirectory = Directory.CreateDirectory(Guid.NewGuid().ToString()); + var export = new DirectoryExport(emptyDirectory, options); + + try { + + await export.Export(summary, NullLogger.Instance, CancellationToken.None); + + var exportedFile = Assert.Single(emptyDirectory.GetFiles()); + Assert.StartsWith(summary.Name, exportedFile.Name); + Assert.EndsWith(".json", exportedFile.Name); + } + finally { + emptyDirectory.Delete(true); + } + } + + [Fact] + public void ToStringContainsExportPath() + { + String[] pathComponents = ["path", "to", "directory"]; + var path = new DirectoryInfo(Path.Combine(pathComponents)); + var export = new DirectoryExport(path, JsonSerializerOptions.Default); + + var result = export.ToString(); + + // Assert + Assert.All(pathComponents, component => Assert.Contains(component, result)); + } +} diff --git a/source/GlassView.Export.Test/ExtensionsTest.cs b/source/GlassView.Export.Test/ExtensionsTest.cs new file mode 100644 index 0000000..67bf42d --- /dev/null +++ b/source/GlassView.Export.Test/ExtensionsTest.cs @@ -0,0 +1,48 @@ +using static Atmoos.GlassView.Export.Extensions; + +namespace Atmoos.GlassView.Export.Test; + +public class ExtensionsTest +{ + + [Fact] + public void FindLeafFindsExistingLeafDirInSomeToplevelDirectory() + { + const String directoryName = "someExistingLeafDir"; + DirectoryInfo expected = MoveUp(CurrentDirectory(), levels: 3).CreateSubdirectory(directoryName); + try { + DirectoryInfo actual = FindLeaf(directoryName); + + Assert.Equal(expected.FullName, actual.FullName); + Assert.True(expected.Exists, "The expected directory should exist after the test."); + } + finally { + expected.Delete(true); + } + } + + [Fact] + public void FindLeafReturnsCreatedLeafDirInCurrentDirectoryWhenItCannotBeFound() + { + const String directoryName = "someLeafDirThatDoesNotExist"; + var expected = new DirectoryInfo(Path.Combine(CurrentDirectory().FullName, directoryName)); + Assert.False(expected.Exists, "The expected directory should not exist before the test."); + + try { + + DirectoryInfo actual = FindLeaf(directoryName); + + Assert.Equal(expected.FullName, actual.FullName); + Assert.True(actual.Exists, "The expected directory should exist after the test."); + } + finally { + expected.Delete(true); + } + } + + private static DirectoryInfo MoveUp(DirectoryInfo directory, Int32 levels) + { + for (; directory.Parent != null && levels-- > 0; directory = directory.Parent) { } + return directory; + } +} diff --git a/source/GlassView.Export.Test/GlassView.Export.Test.csproj b/source/GlassView.Export.Test/GlassView.Export.Test.csproj new file mode 100644 index 0000000..5bf6a9d --- /dev/null +++ b/source/GlassView.Export.Test/GlassView.Export.Test.csproj @@ -0,0 +1,22 @@ + + + + + + Atmoos.GlassView.Export.Test + + + + + + + + + + + + + + + + diff --git a/source/GlassView.Export.Test/GlassViewTest.cs b/source/GlassView.Export.Test/GlassViewTest.cs new file mode 100644 index 0000000..746e617 --- /dev/null +++ b/source/GlassView.Export.Test/GlassViewTest.cs @@ -0,0 +1,63 @@ +using Microsoft.Extensions.Configuration; +using System.Text; + +namespace Atmoos.GlassView.Export.Test; + +public class GlassViewTest +{ + + [Fact] + public void ConfigureWithDirectoryReturnsDirectoryExporter() + { + String path = Path.Combine("some", "export", "path"); + IConfiguration configuration = ConfigFromJson(DirectoryExportJson(path)); + IGlassView publicExporter = GlassView.Configure(configuration); + + var exporter = Assert.IsType(publicExporter); + var directoryExporter = Assert.Single(exporter); + Assert.IsType(directoryExporter); + } + + [Fact] + public void ConfigureWithDirectoryCreatesTheExportPathAsNeeded() + { + const String topPath = "top"; + var path = Path.Combine(topPath, "this", "path", "does", "not", "exist", "yet"); + IConfiguration configuration = ConfigFromJson(DirectoryExportJson(path)); + + Assert.False(Directory.Exists(path), "Precondition failed: directory already exists."); + + try { + var exporter = GlassView.Configure(configuration); + Assert.True(Directory.Exists(path), "The directory was not created."); + Assert.NotNull(exporter); // Avoid warnings & the compiler optimising away the configure step. + } + finally { + Directory.Delete(topPath, recursive: true); + } + } + + private static IConfiguration ConfigFromJson(String json) + { + return new ConfigurationBuilder() + .AddJsonStream(new MemoryStream(Encoding.UTF8.GetBytes(json))) + .Build(); + } + + private static String DirectoryExportJson(String path) + { + return $$""" + { + "GlassView": { + "Export": { + "Directory": { + "Path": "{{Sanitise(path)}}" + } + } + } + } + """; + + static String Sanitise(String value) => value.Replace("\\", "/"); // Windows path to Unix path, which are compatible with JSON. + } +} diff --git a/source/GlassView.Export.Test/GlobalUsings.cs b/source/GlassView.Export.Test/GlobalUsings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/source/GlassView.Export.Test/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/source/GlassView.Export.Test/Resources/TestBenchmark.json b/source/GlassView.Export.Test/Resources/TestBenchmark.json new file mode 100644 index 0000000..522379d --- /dev/null +++ b/source/GlassView.Export.Test/Resources/TestBenchmark.json @@ -0,0 +1,137 @@ +{ + "Name": "TestBenchmark", + "Count": 4, + "Namespace": "Atmoos.GlassView.Benchmark", + "Timestamp": "2024-05-21T17:16:44Z", + "Duration": "00:00:36.9737189", + "Environment": { + "OsVersion": "Arch Linux", + "Dotnet": { + "HasRyuJit": true, + "BuildConfig": "RELEASE", + "DotNetVersion": "8.0.105", + "HasAttachedDebugger": false + }, + "Processor": { + "Name": "Intel(R) Core(TM) i7-8565U CPU @ 1.80GHz", + "Count": 4, + "Architecture": "X64", + "PhysicalCoreCount": 4, + "LogicalCoreCount": 8 + } + }, + "BenchmarkCases": [ + { + "Name": "SumDoubles", + "IsBaseline": false, + "Categories": [ + "OtherCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "42", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 161.38355009896415, + "Median": 161.19704914093018, + "StandardDeviation": 1.8536628217100735, + "SampleSize": 7 + }, + "Allocation": { + "Gen0Collections": 537, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 536 + } + }, + { + "Name": "SumIntegers", + "IsBaseline": true, + "Categories": [ + "SomeCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "42", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 120.6814014116923, + "Median": 120.78095936775208, + "StandardDeviation": 0.7329357120629901, + "SampleSize": 6 + }, + "Allocation": { + "Gen0Collections": 369, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 368 + } + }, + { + "Name": "SumDoubles", + "IsBaseline": false, + "Categories": [ + "OtherCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "54", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 190.1119761126382, + "Median": 191.5717384815216, + "StandardDeviation": 3.343227140810547, + "SampleSize": 7 + }, + "Allocation": { + "Gen0Collections": 633, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 632 + } + }, + { + "Name": "SumIntegers", + "IsBaseline": true, + "Categories": [ + "SomeCategory", + "Regular" + ], + "Parameters": [ + { + "Name": "Count", + "Value": "54", + "Type": "Int32" + } + ], + "HardwareIntrinsics": "AVX2,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256", + "Statistics": { + "Mean": 137.59001200539726, + "Median": 137.89618372917175, + "StandardDeviation": 1.0305874559096901, + "SampleSize": 7 + }, + "Allocation": { + "Gen0Collections": 417, + "Gen1Collections": 0, + "Gen2Collections": 0, + "AllocatedBytes": 416 + } + } + ] +} \ No newline at end of file diff --git a/source/GlassView.Export/Configuration/Configuration.cs b/source/GlassView.Export/Configuration/Configuration.cs new file mode 100644 index 0000000..8012192 --- /dev/null +++ b/source/GlassView.Export/Configuration/Configuration.cs @@ -0,0 +1,40 @@ + +namespace Atmoos.GlassView.Export.Configuration; + +/// +/// Represents the configuration for the GlassView exporter, which +/// is embedded within an application's configuration. +/// +internal sealed class Export +{ + /// + /// Optional configuration for JSON formatting. + /// + public JsonFormatting? JsonFormatting { get; set; } + + /// + /// Optional configuration for the directory to export to. + /// If not specified, a default directory within the working directory is used. + /// + public Directory? Directory { get; set; } +} + +internal sealed class Directory +{ + public String Path { get; set; } = String.Empty; +} + +internal sealed class JsonFormatting +{ + /// + /// Whether to indent the JSON output. + /// If not specified, System.Text.Json's default is used. + /// + public Boolean? Indented { get; set; } + + /// + /// Whether to allow trailing commas in JSON objects and arrays. + /// If not specified, System.Text.Json's default is used. + /// + public Boolean? AllowTrailingCommas { get; set; } +} diff --git a/source/GlassView.Export/Configuration/Extensions.cs b/source/GlassView.Export/Configuration/Extensions.cs new file mode 100644 index 0000000..c1a59ad --- /dev/null +++ b/source/GlassView.Export/Configuration/Extensions.cs @@ -0,0 +1,24 @@ +using System.Text.Json; +using Microsoft.Extensions.Configuration; + +using static Atmoos.GlassView.Export.Extensions; + +namespace Atmoos.GlassView.Export.Configuration; + +internal static class Extensions +{ + public static JsonSerializerOptions Configure(this JsonSerializerOptions options, JsonFormatting? formatting) + { + if (formatting is null) { + return options; + } + SetValue(formatting.Indented, value => options.WriteIndented = value); + SetValue(formatting.AllowTrailingCommas, value => options.AllowTrailingCommas = value); + return options; + } + + /// + /// Safely get a configuration section as a strongly-typed object. + /// + public static T Section(this IConfiguration config) => config.GetSection(typeof(T).Name).Get(); +} diff --git a/source/GlassView.Export/DirectoryExport.cs b/source/GlassView.Export/DirectoryExport.cs new file mode 100644 index 0000000..9f4f6c0 --- /dev/null +++ b/source/GlassView.Export/DirectoryExport.cs @@ -0,0 +1,21 @@ +using System.Text.Json; +using BenchmarkDotNet.Loggers; +using Atmoos.GlassView.Core.Models; + +namespace Atmoos.GlassView.Export; + +internal sealed class DirectoryExport(DirectoryInfo path, JsonSerializerOptions options) : IExport +{ + public async Task Export(BenchmarkSummary summary, ILogger logger, CancellationToken token) + { + FileInfo file = path.AddFile(FileNameFor(summary)); + logger.WriteLineInfo($"file: {file.FullName}"); + using var stream = file.Open(FileMode.Create, FileAccess.Write, FileShare.None); + await JsonSerializer.SerializeAsync(stream, summary, options, token).ConfigureAwait(ConfigureAwaitOptions.None); + } + + public override String ToString() => $"{nameof(Export)}: {path.FullName}"; + + private static String FileNameFor(BenchmarkSummary summary) + => $"{summary.Name}-{summary.Timestamp.ToLocalTime():s}.json".Replace(':', '-'); +} diff --git a/source/GlassView.Export/Extensions.cs b/source/GlassView.Export/Extensions.cs new file mode 100644 index 0000000..1ab3dba --- /dev/null +++ b/source/GlassView.Export/Extensions.cs @@ -0,0 +1,63 @@ +using System.Text.Json; +using Atmoos.GlassView.Core; + +namespace Atmoos.GlassView.Export; + +public static class Extensions +{ + private static readonly EnumerationOptions findLeafOptions = new() { + RecurseSubdirectories = false, + IgnoreInaccessible = true, + MatchType = MatchType.Simple, + MatchCasing = MatchCasing.CaseSensitive, + AttributesToSkip = FileAttributes.Hidden | FileAttributes.System + }; + + internal static String Serialize(this T value) => Serialize(value, Options(new JsonSerializerOptions())); + internal static String Serialize(this T value, JsonSerializerOptions options) => JsonSerializer.Serialize(value, options); + + private static JsonSerializerOptions Options(JsonSerializerOptions options) + { + options.WriteIndented = true; + return options.EnableGlassView(); + } + + /// + /// Recursively looks upward toward parent directories for the leaf directory + /// in the current directory structure staring at the current working directory. + /// + /// The leaf directories name. + /// If found, returns that directory. + /// When not found, returns the leaf directory as subdirectory of the current directory. + internal static DirectoryInfo FindLeaf(String leafDirectoryName) + { + DirectoryInfo? result; + DirectoryInfo cwd, directory; + directory = cwd = new DirectoryInfo(Directory.GetCurrentDirectory()); + while ((result = directory.EnumerateDirectories(leafDirectoryName, findLeafOptions).SingleOrDefault()) == null) { + if (directory.Parent == null) { + return cwd.CreateSubdirectory(leafDirectoryName); + } + directory = directory.Parent; + } + return result; + } + + internal static DirectoryInfo CurrentDirectory() => new(Directory.GetCurrentDirectory()); + internal static FileInfo AddFile(this DirectoryInfo directory, String fileName) => new(Path.Combine(directory.FullName, fileName)); + internal static void SetValue(T? value, Action setter) + where T : struct + { + if (value != null) { + setter(value.Value); + } + } + + internal static void Set(T? value, Action setter) + where T : class + { + if (value != null) { + setter(value); + } + } +} diff --git a/source/GlassView.Export/GlassView.Export.csproj b/source/GlassView.Export/GlassView.Export.csproj new file mode 100644 index 0000000..343a94c --- /dev/null +++ b/source/GlassView.Export/GlassView.Export.csproj @@ -0,0 +1,17 @@ + + + + + Atmoos.GlassView.Export + + + + + + + + + + + + diff --git a/source/GlassView.Export/GlassView.cs b/source/GlassView.Export/GlassView.cs new file mode 100644 index 0000000..e7c7195 --- /dev/null +++ b/source/GlassView.Export/GlassView.cs @@ -0,0 +1,48 @@ +using System.Text.Json; +using Atmoos.GlassView.Core; +using Microsoft.Extensions.Configuration; +using BenchmarkDotNet.Loggers; +using Atmoos.GlassView.Export.Configuration; + +using static System.IO.Directory; +using static Atmoos.GlassView.Export.Extensions; +using static Atmoos.GlassView.Export.Configuration.Extensions; + +namespace Atmoos.GlassView.Export; + +public static class GlassView +{ + private const String glassViewSection = nameof(GlassView); + public static IGlassView Default() => Default(ConsoleLogger.Default); + public static IGlassView Default(ILogger logger) + => new SummaryExporter(new DirectoryExport(GlassViewArtifactsDirectory(), SerializationOptions()), logger); + + public static IGlassView Configure(IConfiguration config) + => Configure(config, ConsoleLogger.Default); + public static IGlassView Configure(IConfiguration config, ILogger logger) + { + logger.WriteLine(); // this helps visually separate us from BenchmarkDotNet's output. + IConfigurationSection section = config.GetSection(glassViewSection); + if (!section.Exists()) { + logger.WriteHint($"No configuration section '{glassViewSection}' found. Using default directory exporter."); + return Default(logger); + } + var exporters = new List(); + var export = section.Section(); + var serializationOptions = SerializationOptions().Configure(export.JsonFormatting); + + // Here's were we can add more exporters in the future. + Set(export.Directory?.Path, path => exporters.Add(new DirectoryExport(CreateDirectory(path), serializationOptions))); + + if (exporters.Count == 0) { + logger.WriteHint($"No exporters configured. Using default directory exporter."); + exporters.Add(new DirectoryExport(GlassViewArtifactsDirectory(), serializationOptions)); + } + + return new SummaryExporter(exporters, logger); + } + + private static DirectoryInfo GlassViewArtifactsDirectory() => FindLeaf("BenchmarkDotNet.Artifacts").CreateSubdirectory(nameof(GlassView)); + + private static JsonSerializerOptions SerializationOptions() => new JsonSerializerOptions().EnableGlassView(); +} diff --git a/source/GlassView.Export/IExport.cs b/source/GlassView.Export/IExport.cs new file mode 100644 index 0000000..252bdef --- /dev/null +++ b/source/GlassView.Export/IExport.cs @@ -0,0 +1,10 @@ +using Atmoos.GlassView.Core.Models; +using BenchmarkDotNet.Loggers; + +namespace Atmoos.GlassView.Export; + +internal interface IExport +{ + Task Export(BenchmarkSummary summary, ILogger logger, CancellationToken token); +} + diff --git a/source/GlassView.Export/IGlassView.cs b/source/GlassView.Export/IGlassView.cs new file mode 100644 index 0000000..35887dc --- /dev/null +++ b/source/GlassView.Export/IGlassView.cs @@ -0,0 +1,9 @@ +using BenchmarkDotNet.Reports; + +namespace Atmoos.GlassView.Export; + +public interface IGlassView +{ + Task Export(Summary inputSummary, CancellationToken token = default); + Task Export(IEnumerable inputSummaries, CancellationToken token = default); +} diff --git a/source/GlassView.Export/Mapping.cs b/source/GlassView.Export/Mapping.cs new file mode 100644 index 0000000..b52085c --- /dev/null +++ b/source/GlassView.Export/Mapping.cs @@ -0,0 +1,113 @@ +using System.Diagnostics.CodeAnalysis; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Parameters; +using BenchmarkDotNet.Portability.Cpu; +using BenchmarkDotNet.Reports; +using Atmoos.GlassView.Core.Models; + +namespace Atmoos.GlassView.Export; + +[ExcludeFromCodeCoverage(Justification = +""" +BenchmarkDotNet does not allow for deserialization of Summary objects. +Hence, creating a useful summary instance for testing is a non trivial error +prone task in itself, making a test flaky at best. +""")] +internal static class Mapping +{ + public static BenchmarkSummary Map(Summary summary) + { + var titleSegments = summary.Title.Split('-'); + var timestamp = ParseTimestamp(titleSegments[1..]); + var (@namespace, name) = ParseName(titleSegments[0]); + + return new BenchmarkSummary(Cases(summary)) { + Name = name, + Namespace = @namespace, + Timestamp = timestamp, + Duration = summary.TotalTime, + Count = summary.BenchmarksCases.Length, + Environment = Map(summary.HostEnvironmentInfo) + }; + + static (String ns, String name) ParseName(String fullName) + { + var namespaceEnd = fullName.LastIndexOf('.'); + return (fullName[..namespaceEnd], fullName[(namespaceEnd + 1)..]); + } + + static DateTime ParseTimestamp(String[] dateTime) + { + var date = DateOnly.ParseExact(dateTime[0], "yyyyMMdd"); + var time = TimeOnly.ParseExact(dateTime[1], "HHmmss"); + return new DateTime(date, time, DateTimeKind.Local).ToUniversalTime(); + } + + static IEnumerable Cases(Summary summary) + { + BenchmarkReport? report; + foreach (var benchmarkCase in summary.BenchmarksCases) { + if ((report = summary[benchmarkCase]) != null && report.Success && report.ResultStatistics != null) { + var hwIntrinsics = report.GetHardwareIntrinsicsInfo() ?? "unknown"; + yield return Map(benchmarkCase, report.ResultStatistics, report.GcStats, hwIntrinsics); + } + } + } + } + + private static BenchmarkCase Map(BenchmarkDotNet.Running.BenchmarkCase benchmarkCase, Statistics statistics, GcStats gcStats, String hwIntrinsics) + => new() { + Name = benchmarkCase.Descriptor.WorkloadMethod.Name, + IsBaseline = benchmarkCase.Descriptor.Baseline, + Categories = benchmarkCase.Descriptor.Categories, + HardwareIntrinsics = hwIntrinsics, + Parameters = benchmarkCase.Parameters.Items.Select(Map).ToArray(), + Allocation = Map(gcStats, gcStats.GetBytesAllocatedPerOperation(benchmarkCase) ?? 0), + Statistics = Map(statistics) + }; + + private static AllocationInfo Map(GcStats gcStats, Int64 allocatedBytes) => new() { + Gen0Collections = gcStats.Gen0Collections, + Gen1Collections = gcStats.Gen1Collections, + Gen2Collections = gcStats.Gen2Collections, + AllocatedBytes = allocatedBytes, + }; + + private static StatisticsInfo Map(Statistics statistics) => new() { + Mean = statistics.Mean, + Median = statistics.Median, + StandardDeviation = statistics.StandardDeviation, + SampleSize = statistics.N, + }; + + private static EnvironmentInfo Map(HostEnvironmentInfo environment) => new() { + + OsVersion = environment.OsVersion.Value, + Processor = Map(environment.CpuInfo.Value, environment.Architecture), + Dotnet = MapDotnet(environment) + }; + + private static ProcessorInfo Map(CpuInfo cpuInfo, String architecture) => new() { + Name = cpuInfo.ProcessorName, + Architecture = architecture, + Count = cpuInfo.PhysicalCoreCount ?? 0, + PhysicalCoreCount = cpuInfo.PhysicalCoreCount ?? 0, + LogicalCoreCount = cpuInfo.LogicalCoreCount ?? 0 + }; + + private static Parameter Map(ParameterInstance parameter) => new() { + Name = parameter.Name, + Value = parameter.Value.ToString() ?? "null", + Type = parameter.Value.GetType().Name + }; + + private static DotnetInfo MapDotnet(HostEnvironmentInfo environment) => new() { + HasRyuJit = environment.HasRyuJit, + BuildConfig = environment.Configuration, + DotNetVersion = environment.DotNetSdkVersion.Value, + HasAttachedDebugger = environment.HasAttachedDebugger + }; +} + diff --git a/source/GlassView.Export/SummaryExporter.cs b/source/GlassView.Export/SummaryExporter.cs new file mode 100644 index 0000000..72720d8 --- /dev/null +++ b/source/GlassView.Export/SummaryExporter.cs @@ -0,0 +1,63 @@ +using Atmoos.GlassView.Core; +using Atmoos.GlassView.Core.Models; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; + +using static Atmoos.GlassView.Export.Mapping; +using static System.Threading.Tasks.ConfigureAwaitOptions; + +namespace Atmoos.GlassView.Export; + +internal sealed class SummaryExporter(List exporters, ILogger logger) : IGlassView, ICountable +{ + public Int32 Count => exporters.Count; + internal SummaryExporter(IExport exporter, ILogger logger) : this([exporter], logger) { } + + public async Task Export(Summary inputSummary, CancellationToken token) + { + logger.WriteLine(); + var summary = Map(inputSummary); + logger.WriteLineHeader($"// * Exporting '{summary.Name}' *"); + await Export(summary, new ExportLogger(logger), token).ConfigureAwait(None); + } + + public async Task Export(IEnumerable inputSummaries, CancellationToken token = default) + { + logger.WriteLine(); + Task export = Task.CompletedTask; + var count = inputSummaries.Count(); // BenchmarkDotNet creates an array. Hence, this is O(1). + var exportLogger = new ExportLogger(logger); + var summaryText = count == 1 ? "summary" : "summaries"; + logger.WriteLineHeader($"// * Exporting {count} {summaryText} *"); + foreach (Summary inputSummary in inputSummaries) { + await export.ConfigureAwait(None); + var summary = Map(inputSummary); + logger.WriteLineInfo($"- '{summary.Name}' to:"); + export = Export(summary, exportLogger, token); + } + await export.ConfigureAwait(None); + } + + private async Task Export(BenchmarkSummary summary, ILogger logger, CancellationToken token) + { + var exportTask = Task.CompletedTask; + foreach (var exporter in exporters) { + await exportTask.ConfigureAwait(None); + exportTask = exporter.Export(summary, logger, token); + } + await exportTask.ConfigureAwait(None); + } + + public IEnumerator GetEnumerator() => exporters.GetEnumerator(); +} + +file sealed class ExportLogger(ILogger logger) : ILogger +{ + public String Id => logger.Id; + public Int32 Priority => logger.Priority; + public void Write(LogKind logKind, String text) => logger.Write(logKind, Indent(text)); + public void WriteLine(LogKind logKind, String text) => logger.WriteLine(logKind, Indent(text)); + public void WriteLine() => logger.WriteLine(); + public void Flush() => logger.Flush(); + private static String Indent(String text) => $" -> {text}"; +} diff --git a/source/GlassView.Export/readme.md b/source/GlassView.Export/readme.md new file mode 100644 index 0000000..6a3b3b2 --- /dev/null +++ b/source/GlassView.Export/readme.md @@ -0,0 +1,7 @@ +# GlassView Export + +This project contains the necessary plumbing to export benchmark results and may depend on [Benchmark.Net](https://benchmarkdotnet.org/). + +It is however **not** an executable. + +See an example use on [github](https://github.com/atmoos/BenchView/blob/main/source/GlassView.Benchmark/Program.cs). diff --git a/source/GlassView.Test.props b/source/GlassView.Test.props new file mode 100644 index 0000000..b5ad4c2 --- /dev/null +++ b/source/GlassView.Test.props @@ -0,0 +1,21 @@ + + + + false + true + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + diff --git a/source/GlassView.sln b/source/GlassView.sln new file mode 100644 index 0000000..dc3894f --- /dev/null +++ b/source/GlassView.sln @@ -0,0 +1,46 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GlassView.Export", "GlassView.Export\GlassView.Export.csproj", "{21ED0893-1012-4035-8350-1D85C0043310}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GlassView.Export.Test", "GlassView.Export.Test\GlassView.Export.Test.csproj", "{8E9EDA36-B32F-4672-9C94-3B9DAA0A9A1D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GlassView.Core", "GlassView.Core\GlassView.Core.csproj", "{9942AAB3-9E9D-43F5-89C6-8226218E7BE3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GlassView.Core.Test", "GlassView.Core.Test\GlassView.Core.Test.csproj", "{2CEC707B-AC13-4333-A81E-A2772287AFA4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GlassView.Benchmark", "GlassView.Benchmark\GlassView.Benchmark.csproj", "{CECBF6CA-3313-4A39-9F26-F4E0BE5D5561}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {21ED0893-1012-4035-8350-1D85C0043310}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {21ED0893-1012-4035-8350-1D85C0043310}.Debug|Any CPU.Build.0 = Debug|Any CPU + {21ED0893-1012-4035-8350-1D85C0043310}.Release|Any CPU.ActiveCfg = Release|Any CPU + {21ED0893-1012-4035-8350-1D85C0043310}.Release|Any CPU.Build.0 = Release|Any CPU + {8E9EDA36-B32F-4672-9C94-3B9DAA0A9A1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8E9EDA36-B32F-4672-9C94-3B9DAA0A9A1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8E9EDA36-B32F-4672-9C94-3B9DAA0A9A1D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8E9EDA36-B32F-4672-9C94-3B9DAA0A9A1D}.Release|Any CPU.Build.0 = Release|Any CPU + {9942AAB3-9E9D-43F5-89C6-8226218E7BE3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9942AAB3-9E9D-43F5-89C6-8226218E7BE3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9942AAB3-9E9D-43F5-89C6-8226218E7BE3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9942AAB3-9E9D-43F5-89C6-8226218E7BE3}.Release|Any CPU.Build.0 = Release|Any CPU + {2CEC707B-AC13-4333-A81E-A2772287AFA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2CEC707B-AC13-4333-A81E-A2772287AFA4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2CEC707B-AC13-4333-A81E-A2772287AFA4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2CEC707B-AC13-4333-A81E-A2772287AFA4}.Release|Any CPU.Build.0 = Release|Any CPU + {CECBF6CA-3313-4A39-9F26-F4E0BE5D5561}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CECBF6CA-3313-4A39-9F26-F4E0BE5D5561}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CECBF6CA-3313-4A39-9F26-F4E0BE5D5561}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CECBF6CA-3313-4A39-9F26-F4E0BE5D5561}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal