From 12cfee6f5b1601e880db144398d2bedf8f38176d Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Tue, 25 Jul 2023 09:09:34 +0100 Subject: [PATCH] Perf tests (#100) * Perf tests * Workflow naming * Perf executable name fix * Perf adjust test params --- .github/workflows/perf.yml | 50 ++++++ NATS.Client.sln | 7 + .../NATS.Client.Perf/NATS.Client.Perf.csproj | 14 ++ tests/NATS.Client.Perf/Program.cs | 164 ++++++++++++++++++ 4 files changed, 235 insertions(+) create mode 100644 .github/workflows/perf.yml create mode 100644 tests/NATS.Client.Perf/NATS.Client.Perf.csproj create mode 100644 tests/NATS.Client.Perf/Program.cs diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 000000000..8b1cf242e --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,50 @@ +name: Perf + +on: + pull_request: {} + push: + branches: + - main + +jobs: + test: + name: test + strategy: + fail-fast: false + matrix: + config: + - branch: dev + - branch: main + runs-on: ubuntu-latest + env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + NUGET_XMLDOC_MODE: skip + steps: + - name: Install nats + run: | + rel=$(curl -s https://api.github.com/repos/nats-io/natscli/releases/latest | jq -r .tag_name | sed s/v//) + wget https://github.com/nats-io/natscli/releases/download/v$rel/nats-$rel-linux-amd64.zip + unzip nats-$rel-linux-amd64.zip + sudo mv nats-$rel-linux-amd64/nats /usr/local/bin + curl -sf https://binaries.nats.dev/nats-io/nats-server/v2@${{ matrix.config.branch }} | PREFIX=. sh + sudo mv nats-server /usr/local/bin + + - name: Check nats + run: | + nats --version + nats-server -v + + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup dotnet + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.x' + + - name: Release Build + run: dotnet build -c Release tests/NATS.Client.Perf/NATS.Client.Perf.csproj + + - name: Perf Test + run: ./tests/NATS.Client.Perf/bin/Release/net6.0/NATS.Client.Perf diff --git a/NATS.Client.sln b/NATS.Client.sln index b061caa90..d55ba25e5 100644 --- a/NATS.Client.sln +++ b/NATS.Client.sln @@ -63,6 +63,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{BD234E2E EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Schema.Generation", "tools\Schema.Generation\Schema.Generation.csproj", "{B7DD4A9C-2D24-4772-951E-86A665C59ADF}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.Perf", "tests\NATS.Client.Perf\NATS.Client.Perf.csproj", "{ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -153,6 +155,10 @@ Global {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|Any CPU.Build.0 = Debug|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|Any CPU.ActiveCfg = Release|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|Any CPU.Build.0 = Release|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -180,6 +186,7 @@ Global {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145} = {C526E8AB-739A-48D7-8FC4-048978C9B650} {90E5BF38-70C1-460A-9177-CE42815BDBF5} = {C526E8AB-739A-48D7-8FC4-048978C9B650} {B7DD4A9C-2D24-4772-951E-86A665C59ADF} = {BD234E2E-F51A-4B18-B8BE-8AF6D546BF87} + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1} = {C526E8AB-739A-48D7-8FC4-048978C9B650} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8CBB7278-D093-448E-B3DE-B5991209A1AA} diff --git a/tests/NATS.Client.Perf/NATS.Client.Perf.csproj b/tests/NATS.Client.Perf/NATS.Client.Perf.csproj new file mode 100644 index 000000000..354686873 --- /dev/null +++ b/tests/NATS.Client.Perf/NATS.Client.Perf.csproj @@ -0,0 +1,14 @@ + + + + Exe + net6.0 + enable + enable + + + + + + + diff --git a/tests/NATS.Client.Perf/Program.cs b/tests/NATS.Client.Perf/Program.cs new file mode 100644 index 000000000..5782c3448 --- /dev/null +++ b/tests/NATS.Client.Perf/Program.cs @@ -0,0 +1,164 @@ +using System.Buffers; +using System.Diagnostics; +using System.Text.RegularExpressions; +using NATS.Client.Core.Tests; + +var t = new TestParams +{ + Msgs = 1_000_000, + Size = 128, + Subject = "test", + PubTasks = 10, + MaxNatsBenchRatio = 0.05, + MaxMemoryMb = 500, + MaxAllocatedMb = 750, +}; + +Console.WriteLine("NATS NET v2 Perf Tests"); +Console.WriteLine(t); + +await using var server = NatsServer.Start(); + +Console.WriteLine("\nRunning nats bench"); +var natsBenchTotalMsgs = RunNatsBench(server.ClientUrl, t); + +await using var nats1 = server.CreateClientConnection(); +await using var nats2 = server.CreateClientConnection(); + +await nats1.PingAsync(); +await nats2.PingAsync(); + +await using var sub = await nats1.SubscribeAsync(t.Subject); + +var stopwatch = Stopwatch.StartNew(); + +var subReader = Task.Run(async () => +{ + var count = 0; + await foreach (var unused in sub.Msgs.ReadAllAsync()) + { + if (++count == t.Msgs) + break; + } +}); + +var payload = new ReadOnlySequence(new byte[t.Size]); +var sem = new SemaphoreSlim(t.PubTasks); +for (var i = 0; i < t.Msgs; i++) +{ + await sem.WaitAsync(); + var unused = Task.Run(async () => + { + try + { + await nats2.PublishAsync(t.Subject, payload); + } + finally + { + sem.Release(); + } + }); +} + +Console.WriteLine($"[{stopwatch.Elapsed}]"); + +await subReader; + +Console.WriteLine($"[{stopwatch.Elapsed}]"); + +var seconds = stopwatch.Elapsed.TotalSeconds; + +var meg = Math.Pow(2, 20); + +var totalMsgs = 2.0 * t.Msgs / seconds; +var totalSizeMb = 2.0 * t.Msgs * t.Size / meg / seconds; + +var memoryMb = Process.GetCurrentProcess().PrivateMemorySize64 / meg; + +Console.WriteLine(); +Console.WriteLine($"{totalMsgs:n0} msgs/sec ~ {totalSizeMb:n2} MB/sec"); + +var r = totalMsgs / natsBenchTotalMsgs; +Result.Add($"nats bench comparison: {r:n2} (> {t.MaxNatsBenchRatio})", () => r > t.MaxNatsBenchRatio); +Result.Add($"memory usage: {memoryMb:n2} MB (< {t.MaxMemoryMb} MB)", () => memoryMb < t.MaxMemoryMb); + +var allocatedMb = GC.GetTotalAllocatedBytes() / meg; +Result.Add($"allocations: {allocatedMb:n2} MB (< {t.MaxAllocatedMb} MB)", () => allocatedMb < t.MaxAllocatedMb); + +Console.WriteLine(); +return Result.Eval(); + +double RunNatsBench(string url, TestParams testParams) +{ + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "nats", + Arguments = $"bench {testParams.Subject} --pub 1 --sub 1 --size={testParams.Size} --msgs={testParams.Msgs} --no-progress", + RedirectStandardOutput = true, + UseShellExecute = false, + Environment = { { "NATS_URL", $"{url}" } }, + }, + }; + process.Start(); + process.WaitForExit(); + var output = process.StandardOutput.ReadToEnd(); + var match = Regex.Match(output, @"^\s*NATS Pub/Sub stats: (\S+) msgs/sec ~ (\S+) (\w+)/sec", RegexOptions.Multiline); + var total = double.Parse(match.Groups[1].Value); + + Console.WriteLine(output); + Console.WriteLine($"Parsed nats bench msgs {total:n0}"); + Console.WriteLine(); + + return total; +} + +internal class Result +{ + private static readonly List Results = new(); + private readonly string _message; + private readonly Func _test; + + private Result(string message, Func test) + { + _message = message; + _test = test; + } + + public static void Add(string message, Func test) => + Results.Add(new Result(message: message, test: test)); + + public static int Eval() + { + var failed = 0; + foreach (var result in Results) + { + var test = result._test(); + var ok = test ? "OK" : "NOT OK"; + Console.WriteLine($"[{ok}] {result._message}"); + if (test == false) failed++; + } + + Console.WriteLine(failed == 0 ? "PASS" : "FAILED"); + + return failed; + } +} + +internal record TestParams +{ + public int Msgs { get; init; } + + public string Subject { get; init; } = string.Empty; + + public int Size { get; init; } + + public int MaxMemoryMb { get; init; } + + public double MaxNatsBenchRatio { get; init; } + + public int PubTasks { get; init; } + + public int MaxAllocatedMb { get; init; } +}