From 18079f294b5a4d098a67cd96ac8c22e073c847d6 Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Thu, 3 Dec 2020 14:58:27 +0100 Subject: [PATCH] Change back config to match config of 0.5.0, Update to 0.5.1 --- ...Benchmark.PerformanceBenchmark-barplot.png | 4 +- Docs/screenshot.png | 4 +- NetworkBenchmarkDotNet.sln | 4 +- .../NetworkBenchmarkDotNet.csproj | 12 ++-- .../APredefinedBenchmark.cs | 2 + .../PredefinedBenchmarks/GarbageBenchmark.cs | 2 +- .../PerformanceBenchmark.cs | 2 +- README.md | 55 +++++++++++-------- run-benchmark.sh | 2 +- 9 files changed, 46 insertions(+), 41 deletions(-) diff --git a/Docs/NetCoreNetworkBenchmark.PerformanceBenchmark-barplot.png b/Docs/NetCoreNetworkBenchmark.PerformanceBenchmark-barplot.png index 09d147e..206db34 100644 --- a/Docs/NetCoreNetworkBenchmark.PerformanceBenchmark-barplot.png +++ b/Docs/NetCoreNetworkBenchmark.PerformanceBenchmark-barplot.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d08677f57de92745cc0c0565fbdc4be9992e8f06f6923f4df26c6232c15ddfea -size 84689 +oid sha256:90e0653602dce02614b5e4976877f1ed0a8785e47aef2550171dab9bd8ff61ae +size 83986 diff --git a/Docs/screenshot.png b/Docs/screenshot.png index de9ab9c..f9a3499 100644 --- a/Docs/screenshot.png +++ b/Docs/screenshot.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:750137fc1ec26c132598a3d8b33845f7bc08ca28acf436391435e01bc54df878 -size 158606 +oid sha256:9a9fb2d337ed9646650ee9c31d83a816e0f92605239cbf3bd49dc3633173c9e5 +size 157261 diff --git a/NetworkBenchmarkDotNet.sln b/NetworkBenchmarkDotNet.sln index 4c564ff..c56ac84 100644 --- a/NetworkBenchmarkDotNet.sln +++ b/NetworkBenchmarkDotNet.sln @@ -10,7 +10,7 @@ Global GlobalSection(ProjectConfigurationPlatforms) = postSolution {A1025393-381A-4EB0-98C7-36A0FA7A4685}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A1025393-381A-4EB0-98C7-36A0FA7A4685}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A1025393-381A-4EB0-98C7-36A0FA7A4685}.Release|Any CPU.ActiveCfg = Release|x64 - {A1025393-381A-4EB0-98C7-36A0FA7A4685}.Release|Any CPU.Build.0 = Release|x64 + {A1025393-381A-4EB0-98C7-36A0FA7A4685}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A1025393-381A-4EB0-98C7-36A0FA7A4685}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj index 6a64b8b..010c70b 100644 --- a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj +++ b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj @@ -5,12 +5,12 @@ .Net Core Networking Benchmark Johannes Deml Johannes Deml - DotNetCoreNetworkingBenchmark - 0.5.0 - 0.5.0 + NetworkingBenchmark + 0.5.1 + 0.5.1 en-US Debug;Release - AnyCPU;x64 + AnyCPU netcoreapp3.1 @@ -47,9 +47,5 @@ LICENSE https://github.com/JohannesDeml/NetworkBenchmarkDotNet - - - x64 - diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/APredefinedBenchmark.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/APredefinedBenchmark.cs index aece23f..0e911e8 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/APredefinedBenchmark.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/APredefinedBenchmark.cs @@ -13,6 +13,8 @@ namespace NetworkBenchmark { + [GcServer(true)] + [GcConcurrent(false)] public abstract class APredefinedBenchmark { [ParamsAllValues] diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs index ee9e9a9..a8c633c 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs @@ -13,7 +13,7 @@ namespace NetworkBenchmark { - [Config(typeof(BenchmarkConfig))] + [SimpleJob(launchCount: 1, warmupCount: 1, targetCount: 10, id: "Performance Benchmark")] [EventPipeProfiler(EventPipeProfile.GcVerbose)] public class GarbageBenchmark : APredefinedBenchmark { diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/PerformanceBenchmark.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/PerformanceBenchmark.cs index eaeb150..9d7a265 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/PerformanceBenchmark.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/PerformanceBenchmark.cs @@ -12,7 +12,7 @@ namespace NetworkBenchmark { - [Config(typeof(BenchmarkConfig))] + [SimpleJob(launchCount: 1, warmupCount: 1, targetCount: 10, id: "Performance Benchmark")] [RPlotExporter] public class PerformanceBenchmark : APredefinedBenchmark { diff --git a/README.md b/README.md index 8a95a1e..8f16801 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ![Screenshot](./Docs/screenshot.png) -[![Releases](https://img.shields.io/github/release/JohannesDeml/NetCoreNetworkBenchmark/all.svg)](../../releases) +[![Releases](https://img.shields.io/github/release/JohannesDeml/NetworkBenchmarkDotNet/all.svg)](../../releases) ## Description @@ -17,7 +17,7 @@ NBN is a benchmark for low level networking libraries using UDP and can be used * Max concurrent connections are limited to 4095 due to the protocol * Packetsize overhead: 10 bytes * [Unity Client Example](https://github.com/JohannesDeml/ENetUnityMobile) -* [LiteNetLib](https://github.com/RevenantX/LiteNetLib) (master 252c8eb) +* [LiteNetLib](https://github.com/RevenantX/LiteNetLib) (v 0.9.4) * Very feature-rich library * Packetsize overhead: 1 byte for unreliable, 4 bytes for reliable * [Unity Client Example](https://github.com/RevenantX/NetGameExample) @@ -32,31 +32,38 @@ To reproduce the benchmarks, run `./NetworkBenchmarkDotNet -b All` ``` ini -Results v0.5.0, BenchmarkDotNet v0.12.1, OS ubuntu 20.04 +Results v0.5.1, BenchmarkDotNet=v0.12.1, OS=ubuntu 20.04 Intel Core i5-3570K CPU 3.40GHz (Ivy Bridge), 1 CPU, 4 logical and 4 physical cores -.NET Core SDK=3.1.403 - [Host] : .NET Core 3.1.9 (CoreCLR 4.700.20.47201, CoreFX 4.700.20.47203), X64 RyuJIT +.NET Core SDK=3.1.404 + [Host] : .NET Core 3.1.10 (CoreCLR 4.700.20.51601, CoreFX 4.700.20.51901), X64 RyuJIT + Job-CXRHAN : .NET Core 3.1.10 (CoreCLR 4.700.20.51601, CoreFX 4.700.20.51901), X64 RyuJIT + +Concurrent=False Server=True InvocationCount=1 +IterationCount=10 LaunchCount=1 UnrollFactor=1 +WarmupCount=1 ``` -| Benchmark | ENet | LiteNetLib | NetCoreServer | -| --------------- | ------------------------------ | ------------------------------ | ------------------------------ | -| **Performance** | | | | -| Performance1 | **192,034 msg/s** (5,207.4 ms) | **78,447 msg/s** (12,747.5 ms) | **109,813 msg/s** (9,106.4 ms) | -| Performance2 | **1,075,153 msg/s** (930.1 ms) | **286,566 msg/s** (3,489.6 ms) | **114,481 msg/s** (8,735.1 ms) | -| **Garbage** | | | | -| Alloc | **0.512 MB** | **81.161 MB** | **161.978 MB** | -| GC Pauses | **83.4 ms** (max 8.4 ms) | **101.1 ms** (max 2.9 ms) | **74.5 ms** (max 2.5 ms) | - -![Benchmark Results](./Docs/NetCoreNetworkBenchmark.PerformanceBenchmark-barplot.png) - -### Benchmark Performance1 +| Benchmark | ENet | LiteNetLib | NetCoreServer | +| --------------- | --------------------------- | --------------------------- | --------------------------- | +| **Performance** | | | | +| Performance1 | **167,841 msg/s** (5.958 s) | **68,129 msg/s** (14.678 s) | **90,612 msg/s** (11.036 s) | +| Performance2 | **993,048 msg/s** (1.007 s) | **267,379 msg/s** (3.740 s) | **91,215 msg/s** (10.963 s) | +| **Garbage** | | | | +| Alloc | **0.512 MB** | **81.968 MB** | **161.892 MB** | +| GC Pauses | **107.5 ms** (max 19.6 ms) | **106.6 ms** (max 2.7 ms) | **86.8 ms** (max 10.6 ms) | + +![Benchmark Results](./Docs/NetworkBenchmarkDotNet.PerformanceBenchmark-barplot.png) + +### Benchmark [Performance1](../../NetworkBenchmarkDotNet/PredefinedBenchmarks/PerformanceBenchmark.cs) + Runs the benchmark with **1,000** clients, which pingpong **1 message** each with the server. The benchmark runs until a total of **1 million** messages are sent to the server and back to the clients. Message size is **32 bytes**. This test is for getting an idea of an average roundtrip time. -### Benchmark Performance2 +### Benchmark [Performance2](../../NetworkBenchmarkDotNet/PredefinedBenchmarks/PerformanceBenchmark.cs) + Runs the benchmark with **1,000** clients, which pingpong **10 messages** each with the server. The benchmark runs until a total of **1 million** messages are sent to the server and back to the clients. Message size is **32 bytes**. This test is for multiplexing / message merging performance. -### Benchmark Garbage +### Benchmark [Garbage](../../NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs) Runs the benchmark with **10** clients, which pingpong **10 messages** each with the server. The benchmark runs until a total of **10,000** messages are sent to the server and back to the clients. Message size is **128 bytes**. This test collects information about generated garbage while running the benchmark. @@ -80,7 +87,7 @@ Then just open the solution file with Visual Studio/Rider/Visual Studio Code and ``` Usage: - NetCoreNetworkBenchmark [options] + NetworkBenchmarkDotNet [options] Options: -b, --benchmark Run predefined benchmarks [default: Custom] @@ -118,11 +125,11 @@ Your new proposed library ... #### How to add a library 1. Add a new folder inside the NetworkBenchmarkDotNet solution with the name of your library -2. Add a script called `YourLibraryBenchmark.cs` which implements [INetworkBenchmark](../../blob/master/NetCoreNetworkBenchmark/INetworkBenchmark.cs) -3. Add your library name to the [NetworkLibrary](../../blob/master/NetCoreNetworkBenchmark/NetworkLibrary.cs) enum +2. Add a script called `YourLibraryBenchmark.cs` which implements [INetworkBenchmark](../../blob/master/NetworkBenchmarkDotNet/INetworkBenchmark.cs) +3. Add your library name to the [NetworkLibrary](../../blob/master/NetworkBenchmarkDotNet/NetworkLibrary.cs) enum 4. Add your Implementation Constructor to `INetworkBenchmark.CreateNetworkBenchmark()` -5. Use the `-l ` argument (or `BenchmarkConfiguration.Library`) to test your library and if everything works as expected. -6. Run the benchmarks `./NetCoreNetworkBenchmark -b All` and see if your library runs correct +5. Use the `-l ` argument (or `BenchmarkSetup.Library`) to test your library and if everything works as expected. +6. Run the benchmarks `./NetworkBenchmarkDotNet -b All` and see if your library runs correct 7. Create a PR including your benchmark md results 🎉 ### Rules for adding a benchmark diff --git a/run-benchmark.sh b/run-benchmark.sh index 1b2e0a7..ce7fbfb 100644 --- a/run-benchmark.sh +++ b/run-benchmark.sh @@ -7,4 +7,4 @@ else mode="$1" fi -sudo ./bin/NetworkBenchmarkDotNet-Linux/NetworkBenchmarkDotNet -b "$mode" +./bin/NetworkBenchmarkDotNet-Linux/NetworkBenchmarkDotNet -b "$mode"