-
Notifications
You must be signed in to change notification settings - Fork 52
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Perf tests * Workflow naming * Perf executable name fix * Perf adjust test params
- Loading branch information
Showing
4 changed files
with
235 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| name: Perf | ||
|
|
||
| on: | ||
| pull_request: {} | ||
| push: | ||
| branches: | ||
| - main | ||
|
|
||
| jobs: | ||
| test: | ||
| name: test | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| config: | ||
| - branch: dev | ||
| - branch: main | ||
| runs-on: ubuntu-latest | ||
| env: | ||
| DOTNET_CLI_TELEMETRY_OPTOUT: 1 | ||
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 | ||
| NUGET_XMLDOC_MODE: skip | ||
| steps: | ||
| - name: Install nats | ||
| run: | | ||
| rel=$(curl -s https://api.github.com/repos/nats-io/natscli/releases/latest | jq -r .tag_name | sed s/v//) | ||
| wget https://github.com/nats-io/natscli/releases/download/v$rel/nats-$rel-linux-amd64.zip | ||
| unzip nats-$rel-linux-amd64.zip | ||
| sudo mv nats-$rel-linux-amd64/nats /usr/local/bin | ||
| curl -sf https://binaries.nats.dev/nats-io/nats-server/v2@${{ matrix.config.branch }} | PREFIX=. sh | ||
| sudo mv nats-server /usr/local/bin | ||
| - name: Check nats | ||
| run: | | ||
| nats --version | ||
| nats-server -v | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
|
|
||
| - name: Setup dotnet | ||
| uses: actions/setup-dotnet@v3 | ||
| with: | ||
| dotnet-version: '6.x' | ||
|
|
||
| - name: Release Build | ||
| run: dotnet build -c Release tests/NATS.Client.Perf/NATS.Client.Perf.csproj | ||
|
|
||
| - name: Perf Test | ||
| run: ./tests/NATS.Client.Perf/bin/Release/net6.0/NATS.Client.Perf |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\NATS.Client.TestUtilities\NATS.Client.TestUtilities.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| using System.Buffers; | ||
| using System.Diagnostics; | ||
| using System.Text.RegularExpressions; | ||
| using NATS.Client.Core.Tests; | ||
|
|
||
| var t = new TestParams | ||
| { | ||
| Msgs = 1_000_000, | ||
| Size = 128, | ||
| Subject = "test", | ||
| PubTasks = 10, | ||
| MaxNatsBenchRatio = 0.05, | ||
| MaxMemoryMb = 500, | ||
| MaxAllocatedMb = 750, | ||
| }; | ||
|
|
||
| Console.WriteLine("NATS NET v2 Perf Tests"); | ||
| Console.WriteLine(t); | ||
|
|
||
| await using var server = NatsServer.Start(); | ||
|
|
||
| Console.WriteLine("\nRunning nats bench"); | ||
| var natsBenchTotalMsgs = RunNatsBench(server.ClientUrl, t); | ||
|
|
||
| await using var nats1 = server.CreateClientConnection(); | ||
| await using var nats2 = server.CreateClientConnection(); | ||
|
|
||
| await nats1.PingAsync(); | ||
| await nats2.PingAsync(); | ||
|
|
||
| await using var sub = await nats1.SubscribeAsync(t.Subject); | ||
|
|
||
| var stopwatch = Stopwatch.StartNew(); | ||
|
|
||
| var subReader = Task.Run(async () => | ||
| { | ||
| var count = 0; | ||
| await foreach (var unused in sub.Msgs.ReadAllAsync()) | ||
| { | ||
| if (++count == t.Msgs) | ||
| break; | ||
| } | ||
| }); | ||
|
|
||
| var payload = new ReadOnlySequence<byte>(new byte[t.Size]); | ||
| var sem = new SemaphoreSlim(t.PubTasks); | ||
| for (var i = 0; i < t.Msgs; i++) | ||
| { | ||
| await sem.WaitAsync(); | ||
| var unused = Task.Run(async () => | ||
| { | ||
| try | ||
| { | ||
| await nats2.PublishAsync(t.Subject, payload); | ||
| } | ||
| finally | ||
| { | ||
| sem.Release(); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| Console.WriteLine($"[{stopwatch.Elapsed}]"); | ||
|
|
||
| await subReader; | ||
|
|
||
| Console.WriteLine($"[{stopwatch.Elapsed}]"); | ||
|
|
||
| var seconds = stopwatch.Elapsed.TotalSeconds; | ||
|
|
||
| var meg = Math.Pow(2, 20); | ||
|
|
||
| var totalMsgs = 2.0 * t.Msgs / seconds; | ||
| var totalSizeMb = 2.0 * t.Msgs * t.Size / meg / seconds; | ||
|
|
||
| var memoryMb = Process.GetCurrentProcess().PrivateMemorySize64 / meg; | ||
|
|
||
| Console.WriteLine(); | ||
| Console.WriteLine($"{totalMsgs:n0} msgs/sec ~ {totalSizeMb:n2} MB/sec"); | ||
|
|
||
| var r = totalMsgs / natsBenchTotalMsgs; | ||
| Result.Add($"nats bench comparison: {r:n2} (> {t.MaxNatsBenchRatio})", () => r > t.MaxNatsBenchRatio); | ||
| Result.Add($"memory usage: {memoryMb:n2} MB (< {t.MaxMemoryMb} MB)", () => memoryMb < t.MaxMemoryMb); | ||
|
|
||
| var allocatedMb = GC.GetTotalAllocatedBytes() / meg; | ||
| Result.Add($"allocations: {allocatedMb:n2} MB (< {t.MaxAllocatedMb} MB)", () => allocatedMb < t.MaxAllocatedMb); | ||
|
|
||
| Console.WriteLine(); | ||
| return Result.Eval(); | ||
|
|
||
| double RunNatsBench(string url, TestParams testParams) | ||
| { | ||
| var process = new Process | ||
| { | ||
| StartInfo = new ProcessStartInfo | ||
| { | ||
| FileName = "nats", | ||
| Arguments = $"bench {testParams.Subject} --pub 1 --sub 1 --size={testParams.Size} --msgs={testParams.Msgs} --no-progress", | ||
| RedirectStandardOutput = true, | ||
| UseShellExecute = false, | ||
| Environment = { { "NATS_URL", $"{url}" } }, | ||
| }, | ||
| }; | ||
| process.Start(); | ||
| process.WaitForExit(); | ||
| var output = process.StandardOutput.ReadToEnd(); | ||
| var match = Regex.Match(output, @"^\s*NATS Pub/Sub stats: (\S+) msgs/sec ~ (\S+) (\w+)/sec", RegexOptions.Multiline); | ||
| var total = double.Parse(match.Groups[1].Value); | ||
|
|
||
| Console.WriteLine(output); | ||
| Console.WriteLine($"Parsed nats bench msgs {total:n0}"); | ||
| Console.WriteLine(); | ||
|
|
||
| return total; | ||
| } | ||
|
|
||
| internal class Result | ||
| { | ||
| private static readonly List<Result> Results = new(); | ||
| private readonly string _message; | ||
| private readonly Func<bool> _test; | ||
|
|
||
| private Result(string message, Func<bool> test) | ||
| { | ||
| _message = message; | ||
| _test = test; | ||
| } | ||
|
|
||
| public static void Add(string message, Func<bool> test) => | ||
| Results.Add(new Result(message: message, test: test)); | ||
|
|
||
| public static int Eval() | ||
| { | ||
| var failed = 0; | ||
| foreach (var result in Results) | ||
| { | ||
| var test = result._test(); | ||
| var ok = test ? "OK" : "NOT OK"; | ||
| Console.WriteLine($"[{ok}] {result._message}"); | ||
| if (test == false) failed++; | ||
| } | ||
|
|
||
| Console.WriteLine(failed == 0 ? "PASS" : "FAILED"); | ||
|
|
||
| return failed; | ||
| } | ||
| } | ||
|
|
||
| internal record TestParams | ||
| { | ||
| public int Msgs { get; init; } | ||
|
|
||
| public string Subject { get; init; } = string.Empty; | ||
|
|
||
| public int Size { get; init; } | ||
|
|
||
| public int MaxMemoryMb { get; init; } | ||
|
|
||
| public double MaxNatsBenchRatio { get; init; } | ||
|
|
||
| public int PubTasks { get; init; } | ||
|
|
||
| public int MaxAllocatedMb { get; init; } | ||
| } |