From bbbd8924f2e2153d1f2816797e90028d85f4c1f1 Mon Sep 17 00:00:00 2001 From: Paul Toffoloni Date: Thu, 9 Jan 2025 09:48:23 +0100 Subject: [PATCH 01/80] Start adding benchmarks --- .github/workflows/benchmark.yml | 66 +++++++++++++++++++++++++++++ Benchmarks/Package.swift | 26 ++++++++++++ Benchmarks/Sources/Benchmarks.swift | 52 +++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 Benchmarks/Package.swift create mode 100644 Benchmarks/Sources/Benchmarks.swift diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..12600b0 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,66 @@ +name: Benchmark PR vs main +on: + workflow_dispatch: + pull_request_review: + types: [submitted] + pull_request: + branches: [ main ] + types: [synchronize] + paths: + - 'Sources/*.swift' + - .github/workflows/benchmark.yml + +jobs: + benchmark-delta-linux: + if: github.event.review.state == 'approved' + runs-on: + - runs-on=${{ github.run_id }} + - runner=2cpu-linux-arm64 + container: swift:jammy + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: jemalloc dependency + run: apt-get update && apt-get install -y libjemalloc-dev + - name: Fix Git config + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Run benchmarks for PR branch + run: | + swift package -c release --package-path Benchmarks --disable-sandbox benchmark baseline update pull_request --no-progress --quiet + echo "### PR Branch Benchmark Results" >> "${GITHUB_STEP_SUMMARY}" + swift package -c release --package-path Benchmarks benchmark list --format markdown >> "${GITHUB_STEP_SUMMARY}" + - name: Check if Benchmarks/Package.swift exists on main + id: check_benchmarks + run: | + git checkout main + if [ -f "Benchmarks/Package.swift" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + git checkout - + - name: Run benchmarks for 'main' branch + if: steps.check_benchmarks.outputs.exists == 'true' + run: | + git stash + git checkout main + swift package -c release --package-path Benchmarks --disable-sandbox benchmark baseline update main --no-progress --quiet + - name: Compare benchmarks + if: steps.check_benchmarks.outputs.exists == 'true' + run: | + echo "### Comparison with Main Branch" >> "${GITHUB_STEP_SUMMARY}" + date >> "${GITHUB_STEP_SUMMARY}" + swift package -c release --package-path Benchmarks benchmark baseline check main pull_request --format markdown >> "${GITHUB_STEP_SUMMARY}" + - name: Get formatted date + if: steps.check_benchmarks.outputs.exists == 'true' + id: get-date + run: echo "date=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_OUTPUT + - uses: thollander/actions-comment-pull-request@v3 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + ${{ format('[PR benchmarks run at {0}]({1}/{2}/actions/runs/{3})', steps.get-date.outputs.date, github.server_url, github.repository, github.run_id) }} + ${{ steps.check_benchmarks.outputs.exists == 'true' && '(includes comparison with main)' || '(main branch benchmarks not available)' }} + comment_tag: 'PR benchmark comparison Linux' diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift new file mode 100644 index 0000000..509eecd --- /dev/null +++ b/Benchmarks/Package.swift @@ -0,0 +1,26 @@ +// swift-tools-version:6.0 + +import PackageDescription + +let package = Package( + name: "Benchmarks", + platforms: [ + .macOS(.v13) + ], + dependencies: [ + .package(path: "../"), + .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.27.0"), + ], + targets: [ + .executableTarget( + name: "Benchmarks", + dependencies: [ + .product(name: "MultipartKit", package: "multipart-kit"), + .product(name: "Benchmark", package: "package-benchmark"), + ], + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark") + ] + ) + ] +) diff --git a/Benchmarks/Sources/Benchmarks.swift b/Benchmarks/Sources/Benchmarks.swift new file mode 100644 index 0000000..60307e2 --- /dev/null +++ b/Benchmarks/Sources/Benchmarks.swift @@ -0,0 +1,52 @@ +import Benchmark +import MultipartKit + +let benchmarks: @Sendable () -> Void = { + Benchmark( + "Parser Allocations", + configuration: .init( + metrics: [.mallocCountTotal] + ) + ) { benchmark in + let boundary = "boundary123" + var message = ArraySlice( + """ + --\(boundary)\r + Content-Disposition: form-data; name="id"\r + Content-Type: text/plain\r + \r + 123e4567-e89b-12d3-a456-426655440000\r + --\(boundary)\r + Content-Disposition: form-data; name="address"\r + Content-Type: application/json\r + \r + {\r + "street": "3, Garden St",\r + "city": "Hillsbery, UT"\r + }\r + --\(boundary)\r + Content-Disposition: form-data; name="profileImage"; filename="image1.png"\r + Content-Type: image/png\r + \r\n + """.utf8) + + message.append(contentsOf: Array(repeating: UInt8.random(in: 0...255), count: 500_000_000)) // 500MB + message.append(contentsOf: "\r\n--\(boundary)--".utf8) + + let stream = AsyncStream { continuation in + var offset = message.startIndex + while offset < message.endIndex { + let endIndex = min(message.endIndex, message.index(offset, offsetBy: 16)) + continuation.yield(message[offset.. Date: Thu, 9 Jan 2025 09:56:38 +0100 Subject: [PATCH 02/80] Remove approval requirement for now --- .github/workflows/benchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 12600b0..db63d44 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,4 +1,4 @@ -name: Benchmark PR vs main +name: benchmark vs main on: workflow_dispatch: pull_request_review: @@ -12,7 +12,6 @@ on: jobs: benchmark-delta-linux: - if: github.event.review.state == 'approved' runs-on: - runs-on=${{ github.run_id }} - runner=2cpu-linux-arm64 From 5046e435b9d78b1a0bc4345cfde0382bbfb6ce83 Mon Sep 17 00:00:00 2001 From: Paul Toffoloni Date: Thu, 9 Jan 2025 09:57:43 +0100 Subject: [PATCH 03/80] Update workflow --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index db63d44..f78d2da 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,4 +1,4 @@ -name: benchmark vs main +name: benchmark on: workflow_dispatch: pull_request_review: @@ -11,7 +11,7 @@ on: - .github/workflows/benchmark.yml jobs: - benchmark-delta-linux: + benchmar-vs-main: runs-on: - runs-on=${{ github.run_id }} - runner=2cpu-linux-arm64 From b74536cafbb40717e74d4fdd93977efdda516091 Mon Sep 17 00:00:00 2001 From: Paul Toffoloni Date: Thu, 9 Jan 2025 09:58:21 +0100 Subject: [PATCH 04/80] Update workflow --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f78d2da..df133cc 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,7 +11,7 @@ on: - .github/workflows/benchmark.yml jobs: - benchmar-vs-main: + benchmark-vs-main: runs-on: - runs-on=${{ github.run_id }} - runner=2cpu-linux-arm64 From 05422f5632131b0c819686df2d6d03d82aecd448 Mon Sep 17 00:00:00 2001 From: Paul Toffoloni Date: Thu, 9 Jan 2025 17:32:07 +0100 Subject: [PATCH 05/80] Add more benchmarks --- .github/workflows/benchmark.yml | 4 +- Benchmarks/Package.swift | 16 ++- Benchmarks/Parser/Parser.swift | 131 +++++++++++++++++++++++++ Benchmarks/Serializer/Serializer.swift | 38 +++++++ Benchmarks/Sources/Benchmarks.swift | 52 ---------- 5 files changed, 185 insertions(+), 56 deletions(-) create mode 100644 Benchmarks/Parser/Parser.swift create mode 100644 Benchmarks/Serializer/Serializer.swift delete mode 100644 Benchmarks/Sources/Benchmarks.swift diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index df133cc..35cb040 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -58,8 +58,8 @@ jobs: run: echo "date=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_OUTPUT - uses: thollander/actions-comment-pull-request@v3 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.GITHUB_TOKEN }} message: | ${{ format('[PR benchmarks run at {0}]({1}/{2}/actions/runs/{3})', steps.get-date.outputs.date, github.server_url, github.repository, github.run_id) }} ${{ steps.check_benchmarks.outputs.exists == 'true' && '(includes comparison with main)' || '(main branch benchmarks not available)' }} - comment_tag: 'PR benchmark comparison Linux' + comment-tag: 'PR benchmark comparison Linux' diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift index 509eecd..656d685 100644 --- a/Benchmarks/Package.swift +++ b/Benchmarks/Package.swift @@ -13,14 +13,26 @@ let package = Package( ], targets: [ .executableTarget( - name: "Benchmarks", + name: "Serializer", dependencies: [ .product(name: "MultipartKit", package: "multipart-kit"), .product(name: "Benchmark", package: "package-benchmark"), ], + path: "Serializer", plugins: [ .plugin(name: "BenchmarkPlugin", package: "package-benchmark") ] - ) + ), + .executableTarget( + name: "Parser", + dependencies: [ + .product(name: "MultipartKit", package: "multipart-kit"), + .product(name: "Benchmark", package: "package-benchmark"), + ], + path: "Parser", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark") + ] + ), ] ) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift new file mode 100644 index 0000000..d9c7ec8 --- /dev/null +++ b/Benchmarks/Parser/Parser.swift @@ -0,0 +1,131 @@ +import Benchmark +import MultipartKit + +// Note: the throughput benchmarks use streams which yield with a delay +// to simulate async work. +let benchmarks: @Sendable () -> Void = { + Benchmark( + "Streaming Parser Allocations", + configuration: .init( + metrics: [.mallocCountTotal] + ) + ) { benchmark in + let boundary = "boundary123" + let bigMessage = makeMessage(boundary: boundary, size: 500_000_000) // 500MB: Big message + let bigMessageStream = makeParsingStream(for: bigMessage, chunkSize: 16 * 1024) // 16KB: Realistic streaming chunk size + + let sequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + + benchmark.startMeasurement() + for try await _ in sequence {} + benchmark.stopMeasurement() + } + + Benchmark( + "Streaming Parser Throughput", + configuration: .init( + metrics: [ + .cpuTotal, + .wallClock, + .throughput + ] + ) + ) { benchmark in + let boundary = "boundary123" + let bigMessage = makeMessage(boundary: boundary, size: 500_000_000) + let bigMessageStream = makeParsingStream(for: bigMessage, chunkSize: 16 * 1024, delay: true) + + let sequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + + benchmark.startMeasurement() + for try await _ in sequence {} + benchmark.stopMeasurement() + } + + Benchmark( + "Collating Parser Allocations", + configuration: .init( + metrics: [.mallocCountTotal] + ) + ) { benchmark in + let boundary = "boundary123" + let bigMessage = makeMessage(boundary: boundary, size: 500_000_000) + let bigMessageStream = makeParsingStream(for: bigMessage, chunkSize: 16 * 1024) + + let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + + benchmark.startMeasurement() + defer { benchmark.stopMeasurement() } + for try await _ in sequence {} + } + + Benchmark( + "Collating Parser Throughput", + configuration: .init( + metrics: [ + .cpuTotal, + .wallClock, + .throughput + ] + ) + ) { benchmark in + let boundary = "boundary123" + let bigMessage = makeMessage(boundary: boundary, size: 500_000_000) + let bigMessageStream = makeParsingStream(for: bigMessage, chunkSize: 16 * 1024, delay: true) + + let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + + benchmark.startMeasurement() + for try await _ in sequence {} + benchmark.stopMeasurement() + } +} + +private func makeParsingStream(for message: Body, chunkSize: Int, delay: Bool = false) -> AsyncStream +where Body.SubSequence: Sendable { + AsyncStream { continuation in + var offset = message.startIndex + while offset < message.endIndex { + let endIndex = min(message.endIndex, message.index(offset, offsetBy: chunkSize)) + + if delay { + // Simulate async work + Task.detached { + try? await Task.sleep(for: .milliseconds(1)) + } + } + + continuation.yield(message[offset.. ArraySlice { + var message = ArraySlice( + """ + --\(boundary)\r + Content-Disposition: form-data; name="id"\r + Content-Type: text/plain\r + \r + 123e4567-e89b-12d3-a456-426655440000\r + --\(boundary)\r + Content-Disposition: form-data; name="address"\r + Content-Type: application/json\r + \r + {\r + "street": "3, Garden St",\r + "city": "Hillsbery, UT"\r + }\r + --\(boundary)\r + Content-Disposition: form-data; name="profileImage"; filename="image1.png"\r + Content-Type: image/png\r + \r\n + """.utf8) + + message.append(contentsOf: Array(repeating: UInt8.random(in: 0...255), count: size)) + message.append(contentsOf: "\r\n--\(boundary)--".utf8) + + return message +} diff --git a/Benchmarks/Serializer/Serializer.swift b/Benchmarks/Serializer/Serializer.swift new file mode 100644 index 0000000..2877071 --- /dev/null +++ b/Benchmarks/Serializer/Serializer.swift @@ -0,0 +1,38 @@ +import Benchmark +import MultipartKit + +let example: MultipartPart = .init( + headerFields: .init([ + .init(name: .contentDisposition, value: "form-data; name=\"file\"; filename=\"hello.txt\""), + .init(name: .contentType, value: "text/plain"), + ]), + body: ArraySlice("Hello, world!".utf8) +) + +let benchmarks: @Sendable () -> Void = { + Benchmark( + "Serializer Allocations", + configuration: .init( + metrics: [.mallocCountTotal] + ) + ) { benchmark in + _ = try MultipartSerializer(boundary: "boundary123").serialize(parts: [example]) + } + + Benchmark( + "Serializer Throughput", + configuration: .init( + metrics: [ + .throughput, + .wallClock, + .cpuTotal + ] + ) + ){ benchmark in + let parts: [MultipartPart] = .init(repeating: example, count: 1000) + + benchmark.startMeasurement() + _ = try MultipartSerializer(boundary: "boundary123").serialize(parts: parts) + benchmark.stopMeasurement() + } +} diff --git a/Benchmarks/Sources/Benchmarks.swift b/Benchmarks/Sources/Benchmarks.swift deleted file mode 100644 index 60307e2..0000000 --- a/Benchmarks/Sources/Benchmarks.swift +++ /dev/null @@ -1,52 +0,0 @@ -import Benchmark -import MultipartKit - -let benchmarks: @Sendable () -> Void = { - Benchmark( - "Parser Allocations", - configuration: .init( - metrics: [.mallocCountTotal] - ) - ) { benchmark in - let boundary = "boundary123" - var message = ArraySlice( - """ - --\(boundary)\r - Content-Disposition: form-data; name="id"\r - Content-Type: text/plain\r - \r - 123e4567-e89b-12d3-a456-426655440000\r - --\(boundary)\r - Content-Disposition: form-data; name="address"\r - Content-Type: application/json\r - \r - {\r - "street": "3, Garden St",\r - "city": "Hillsbery, UT"\r - }\r - --\(boundary)\r - Content-Disposition: form-data; name="profileImage"; filename="image1.png"\r - Content-Type: image/png\r - \r\n - """.utf8) - - message.append(contentsOf: Array(repeating: UInt8.random(in: 0...255), count: 500_000_000)) // 500MB - message.append(contentsOf: "\r\n--\(boundary)--".utf8) - - let stream = AsyncStream { continuation in - var offset = message.startIndex - while offset < message.endIndex { - let endIndex = min(message.endIndex, message.index(offset, offsetBy: 16)) - continuation.yield(message[offset.. Date: Fri, 10 Jan 2025 19:16:08 +0330 Subject: [PATCH 06/80] Update benchmark CI with the jwt-kit one --- .github/workflows/benchmark.yml | 284 +++++++++++++++++++++++++++----- 1 file changed, 240 insertions(+), 44 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 35cb040..2990ff3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,62 +4,258 @@ on: pull_request_review: types: [submitted] pull_request: - branches: [ main ] + branches: [main] types: [synchronize] paths: - - 'Sources/*.swift' - - .github/workflows/benchmark.yml + - Sources/*.swift + - Benchmarks/*.swift + - .github/workflows/benchmark.yml jobs: - benchmark-vs-main: + benchmark-vs-thresholds: + # Run the job only if it's a manual workflow dispatch, or if this event is a pull-request approval event, or if someone has rerun the job. + if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved' || github.run_attempt > 1 + + # https://runs-on.com/features/custom-runners/ runs-on: - - runs-on=${{ github.run_id }} - - runner=2cpu-linux-arm64 - container: swift:jammy + labels: + - runs-on + - runner=2cpu-4ram + - run-id=${{ github.run_id }} + + container: swift:noble + + defaults: + run: + shell: bash + + env: + PR_COMMENT: null # will be populated later + steps: - - uses: actions/checkout@v4 + - name: Check out code + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: jemalloc dependency - run: apt-get update && apt-get install -y libjemalloc-dev - - name: Fix Git config + + - name: Configure git + run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" + + # jemalloc is a dependency of the Benchmarking package + # actions/cache will detect zstd and will become much faster. + - name: Install jemalloc, curl, jq and zstd run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Run benchmarks for PR branch + set -eu + + apt-get update -y + apt-get install -y libjemalloc-dev curl jq zstd + + - name: Restore .build + id: restore-cache + uses: runs-on/cache/restore@v4 + with: + path: Benchmarks/.build + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}" + restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-" + + - name: Run benchmarks for branch ${{ github.head_ref || github.ref_name }} run: | - swift package -c release --package-path Benchmarks --disable-sandbox benchmark baseline update pull_request --no-progress --quiet - echo "### PR Branch Benchmark Results" >> "${GITHUB_STEP_SUMMARY}" - swift package -c release --package-path Benchmarks benchmark list --format markdown >> "${GITHUB_STEP_SUMMARY}" - - name: Check if Benchmarks/Package.swift exists on main - id: check_benchmarks + swift package -c release --disable-sandbox \ + --package-path Benchmarks \ + benchmark baseline update \ + '${{ github.head_ref || github.ref_name }}' + + - name: Read benchmark result + id: read-benchmark run: | - git checkout main - if [ -f "Benchmarks/Package.swift" ]; then - echo "exists=true" >> $GITHUB_OUTPUT + set -eu + + swift package -c release --disable-sandbox \ + --package-path Benchmarks \ + benchmark baseline read \ + '${{ github.head_ref || github.ref_name }}' \ + --no-progress \ + --format markdown \ + >> result.text + + # Read the result to the output of the step + echo 'result<> $GITHUB_OUTPUT + cat result.text >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Compare branch ${{ github.head_ref || github.ref_name }} against thresholds + id: compare-benchmark + run: | + set -eu + + TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + ENCODED_TIMESTAMP=$(date -u +"%Y-%m-%dT%H%%3A%M%%3A%SZ") + TIMESTAMP_LINK="https://www.timeanddate.com/worldclock/fixedtime.html?iso=$ENCODED_TIMESTAMP" + echo "## Benchmark check running at [$TIMESTAMP]($TIMESTAMP_LINK)" >> summary.text + + # Disable 'set -e' to prevent the script from exiting on non-zero exit codes + set +e + swift package -c release --disable-sandbox \ + --package-path Benchmarks \ + benchmark thresholds check \ + '${{ github.head_ref || github.ref_name }}' \ + --path "$PWD/Benchmarks/Thresholds/" \ + --no-progress \ + --format markdown \ + >> summary.text + echo "exit-status=$?" >> "${GITHUB_OUTPUT}" + set -e + + echo 'summary<> $GITHUB_OUTPUT + cat summary.text >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Cache .build + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: runs-on/cache/save@v4 + with: + path: Benchmarks/.build + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}" + + - name: Construct comment + run: | + set -eu + + EXIT_CODE='${{ steps.compare-benchmark.outputs.exit-status }}' + + echo 'PR_COMMENT<> "${GITHUB_ENV}" + + # The fact that the comment starts with " >> "${GITHUB_ENV}" + + echo "" >> "${GITHUB_ENV}" + + echo '## [Benchmark](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) Report' >> "${GITHUB_ENV}" + + case "${EXIT_CODE}" in + 0) + echo '**✅ Pull request has no significant performance differences ✅**' >> "${GITHUB_ENV}" + ;; + 1) + echo '**❌ Pull request has significant performance differences 📊**' >> "${GITHUB_ENV}" + ;; + 2) + echo '**❌ Pull request has significant performance regressions 📉**' >> "${GITHUB_ENV}" + ;; + 4) + echo '**❌ Pull request has significant performance improvements 📈**' >> "${GITHUB_ENV}" + ;; + *) + echo '**❌ Benchmark comparison failed to complete properly with exit code $EXIT_CODE ❌**' >> "${GITHUB_ENV}" + ;; + esac + + echo '
' >> "${GITHUB_ENV}" + echo ' Click to expand comparison result ' >> "${GITHUB_ENV}" + echo '' >> "${GITHUB_ENV}" + echo '${{ steps.compare-benchmark.outputs.summary }}' >> "${GITHUB_ENV}" + echo '' >> "${GITHUB_ENV}" + echo '
' >> "${GITHUB_ENV}" + + echo '' >> "${GITHUB_ENV}" + + echo '
' >> "${GITHUB_ENV}" + echo ' Click to expand benchmark result ' >> "${GITHUB_ENV}" + echo '' >> "${GITHUB_ENV}" + echo '${{ steps.read-benchmark.outputs.result }}' >> "${GITHUB_ENV}" + echo '' >> "${GITHUB_ENV}" + echo '
' >> "${GITHUB_ENV}" + + echo 'EOF' >> "${GITHUB_ENV}" + + - name: Output the comment as job summary + run: echo '${{ env.PR_COMMENT }}' >> "${GITHUB_STEP_SUMMARY}" + + # There is a '' comment at the beginning of the benchamrk report comment. + # The number in that comment is the exit code of the last benchmark run. + - name: Find existing comment ID + if: github.event_name == 'pull_request' + id: existing-comment + run: | + set -eu + + # Known limitation: This only fetches the first 100 comments. This should not + # matter much because a benchmark comment should have been sent early in the PR. + curl -sL \ + -X GET \ + -H 'Accept: application/vnd.github+json' \ + -H 'Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + -o result.json \ + 'https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments?per_page=100' + + # Get the last comment that has a body that starts with '" >> "${GITHUB_ENV}" - echo "" >> "${GITHUB_ENV}" + echo '' >> "${GITHUB_ENV}" echo '## [Benchmark](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) Report' >> "${GITHUB_ENV}" @@ -230,7 +230,7 @@ jobs: echo "body<> "$GITHUB_OUTPUT" + } >> "${GITHUB_OUTPUT}" - name: Comment in PR if: github.event_name == 'pull_request' @@ -245,15 +245,15 @@ jobs: echo "Will update a comment: $EXISTING_COMMENT_ID" ENDPOINT="https://api.github.com/repos/${{ github.repository }}/issues/comments/$EXISTING_COMMENT_ID" else - echo "Will create a new comment" + echo 'Will create a new comment' ENDPOINT="https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments" fi curl -sL \ -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ + -H 'Accept: application/vnd.github+json' \ + -H 'Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ -d "$BODY_JSON" \ "$ENDPOINT" From f3ff5586b9112da7200b60c5719f6188bba910d9 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Sun, 12 Jan 2025 19:03:08 +0330 Subject: [PATCH 38/80] increase iterations for more stability --- Benchmarks/Parser/Parser.swift | 4 ++-- Benchmarks/Serializer/Serializer.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 9bde615..ed98b55 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -27,7 +27,7 @@ let benchmarks: @Sendable () -> Void = { "10xStreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxIterations: 10, + maxIterations: 25, thresholds: [ .cpuUser: .init( /// `8 - 1 == 7`% tolerance. @@ -65,7 +65,7 @@ let benchmarks: @Sendable () -> Void = { "10xCollatingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxIterations: 10, + maxIterations: 25, thresholds: [ .cpuUser: .init( /// `10 - 1 == 9`% tolerance. diff --git a/Benchmarks/Serializer/Serializer.swift b/Benchmarks/Serializer/Serializer.swift index 6732161..647f13e 100644 --- a/Benchmarks/Serializer/Serializer.swift +++ b/Benchmarks/Serializer/Serializer.swift @@ -27,7 +27,7 @@ let benchmarks: @Sendable () -> Void = { "100xSerializerCPUTime", configuration: .init( metrics: [.cpuUser], - maxIterations: 10, + maxIterations: 25, thresholds: [ .cpuUser: .init( /// `6 - 1 == 5`% tolerance. From 8185d07e114ae297d1a90e8ffccc3d3a8eefce47 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Sun, 12 Jan 2025 19:06:45 +0330 Subject: [PATCH 39/80] increase max duration --- Benchmarks/Parser/Parser.swift | 6 ++++-- Benchmarks/Serializer/Serializer.swift | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index ed98b55..dc3e4e4 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -27,7 +27,8 @@ let benchmarks: @Sendable () -> Void = { "10xStreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxIterations: 25, + maxDuration: .seconds(5), + maxIterations: 20, thresholds: [ .cpuUser: .init( /// `8 - 1 == 7`% tolerance. @@ -65,7 +66,8 @@ let benchmarks: @Sendable () -> Void = { "10xCollatingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxIterations: 25, + maxDuration: .seconds(5), + maxIterations: 20, thresholds: [ .cpuUser: .init( /// `10 - 1 == 9`% tolerance. diff --git a/Benchmarks/Serializer/Serializer.swift b/Benchmarks/Serializer/Serializer.swift index 647f13e..7ce462a 100644 --- a/Benchmarks/Serializer/Serializer.swift +++ b/Benchmarks/Serializer/Serializer.swift @@ -27,7 +27,8 @@ let benchmarks: @Sendable () -> Void = { "100xSerializerCPUTime", configuration: .init( metrics: [.cpuUser], - maxIterations: 25, + maxDuration: .seconds(5), + maxIterations: 20, thresholds: [ .cpuUser: .init( /// `6 - 1 == 5`% tolerance. From 8f6f2234d28991f8c984dcf760551d70af023e2b Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Sun, 12 Jan 2025 23:42:10 +0330 Subject: [PATCH 40/80] better thresholds and iterations --- Benchmarks/Parser/Parser.swift | 4 ++-- Benchmarks/Serializer/Serializer.swift | 2 +- .../Thresholds/Parser.10xStreamingParserCPUTime.p90.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index dc3e4e4..4c5017e 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -27,7 +27,7 @@ let benchmarks: @Sendable () -> Void = { "10xStreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxDuration: .seconds(5), + maxDuration: .seconds(10), maxIterations: 20, thresholds: [ .cpuUser: .init( @@ -66,7 +66,7 @@ let benchmarks: @Sendable () -> Void = { "10xCollatingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxDuration: .seconds(5), + maxDuration: .seconds(10), maxIterations: 20, thresholds: [ .cpuUser: .init( diff --git a/Benchmarks/Serializer/Serializer.swift b/Benchmarks/Serializer/Serializer.swift index 7ce462a..ac50435 100644 --- a/Benchmarks/Serializer/Serializer.swift +++ b/Benchmarks/Serializer/Serializer.swift @@ -27,7 +27,7 @@ let benchmarks: @Sendable () -> Void = { "100xSerializerCPUTime", configuration: .init( metrics: [.cpuUser], - maxDuration: .seconds(5), + maxDuration: .seconds(10), maxIterations: 20, thresholds: [ .cpuUser: .init( diff --git a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json index 62fc6a4..76ecdbd 100644 --- a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json +++ b/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 350000000 + "cpuUser": 360000000 } From b691cdec873d6d5a3869e2cd47c80443a50b5ebf Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 00:00:31 +0330 Subject: [PATCH 41/80] more adjustments --- Benchmarks/Parser/Parser.swift | 8 ++++---- .../Thresholds/Parser.10xStreamingParserCPUTime.p90.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 4c5017e..2b9a3dc 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -31,11 +31,11 @@ let benchmarks: @Sendable () -> Void = { maxIterations: 20, thresholds: [ .cpuUser: .init( - /// `8 - 1 == 7`% tolerance. + /// `9 - 1 == 8`% tolerance. /// Will rely on the absolute threshold as the tighter threshold. - relative: [.p90: 8], - /// 21ms of tolerance. - absolute: [.p90: 21_000_000] + relative: [.p90: 9], + /// 26ms of tolerance. + absolute: [.p90: 26_000_000] ) ] ) diff --git a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json index 76ecdbd..17b22a3 100644 --- a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json +++ b/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 360000000 + "cpuUser": 365000000 } From d055d798e4614dad64322ea0563e507e8631c12f Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 02:23:49 +0330 Subject: [PATCH 42/80] allocation benchmarks to only run once --- Benchmarks/Parser/Parser.swift | 6 ++++-- Benchmarks/Serializer/Serializer.swift | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 2b9a3dc..b9b75d3 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -14,7 +14,8 @@ let benchmarks: @Sendable () -> Void = { Benchmark( "StreamingParserAllocations", configuration: .init( - metrics: [.mallocCountTotal] + metrics: [.mallocCountTotal], + maxIterations: 1 ) ) { benchmark in let streamingSequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: streamingStream) @@ -53,7 +54,8 @@ let benchmarks: @Sendable () -> Void = { Benchmark( "CollatingParserAllocations", configuration: .init( - metrics: [.mallocCountTotal] + metrics: [.mallocCountTotal], + maxIterations: 1 ) ) { benchmark in let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: collatingStream) diff --git a/Benchmarks/Serializer/Serializer.swift b/Benchmarks/Serializer/Serializer.swift index ac50435..8c3bb4d 100644 --- a/Benchmarks/Serializer/Serializer.swift +++ b/Benchmarks/Serializer/Serializer.swift @@ -15,7 +15,8 @@ let benchmarks: @Sendable () -> Void = { Benchmark( "SerializerAllocations", configuration: .init( - metrics: [.mallocCountTotal] + metrics: [.mallocCountTotal], + maxIterations: 1 ) ) { benchmark in let serializer = MultipartSerializer(boundary: "boundary123") From e3197b0a827819c415b9243e564ea63d989b8902 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 02:25:50 +0330 Subject: [PATCH 43/80] increase big message size to what it should be --- Benchmarks/Parser/Parser.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index b9b75d3..b090639 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -5,7 +5,7 @@ import MultipartKit // to simulate async work. let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - let bigMessage = makeMessage(boundary: boundary, size: 1 << 24) // 400MiB: Big message + let bigMessage = makeMessage(boundary: boundary, size: 1 << 29) // 512MiB: Big message var messageStreams = (0..<500).map { _ in makeParsingStream(for: bigMessage, chunkSize: 1 << 14) // 16KiB: Realistic streaming chunk size } From 009f2e9e2523f19769c88ea45e9ed2e5785d0c69 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 02:37:30 +0330 Subject: [PATCH 44/80] decrease rounds --- Benchmarks/Parser/Parser.swift | 34 ++++++++----------- ...=> Parser.CollatingParserCPUTime.p90.json} | 0 ...=> Parser.StreamingParserCPUTime.p90.json} | 0 3 files changed, 14 insertions(+), 20 deletions(-) rename Benchmarks/Thresholds/{Parser.10xCollatingParserCPUTime.p90.json => Parser.CollatingParserCPUTime.p90.json} (100%) rename Benchmarks/Thresholds/{Parser.10xStreamingParserCPUTime.p90.json => Parser.StreamingParserCPUTime.p90.json} (100%) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index b090639..a9e9c62 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -1,8 +1,6 @@ import Benchmark import MultipartKit -// Note: the `cpuUser` benchmarks use streams which yield with a delay -// to simulate async work. let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" let bigMessage = makeMessage(boundary: boundary, size: 1 << 29) // 512MiB: Big message @@ -25,11 +23,11 @@ let benchmarks: @Sendable () -> Void = { } Benchmark( - "10xStreamingParserCPUTime", + "StreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxDuration: .seconds(10), - maxIterations: 20, + maxDuration: .seconds(20), + maxIterations: 10, thresholds: [ .cpuUser: .init( /// `9 - 1 == 8`% tolerance. @@ -41,12 +39,10 @@ let benchmarks: @Sendable () -> Void = { ] ) ) { benchmark in - for _ in 0..<10 { - let bigMessageStream = messageStreams.removeFirst() - let streamingSequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) - for try await part in streamingSequence { - blackHole(part) - } + let bigMessageStream = messageStreams.removeFirst() + let streamingSequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + for try await part in streamingSequence { + blackHole(part) } } @@ -65,11 +61,11 @@ let benchmarks: @Sendable () -> Void = { } Benchmark( - "10xCollatingParserCPUTime", + "CollatingParserCPUTime", configuration: .init( metrics: [.cpuUser], - maxDuration: .seconds(10), - maxIterations: 20, + maxDuration: .seconds(20), + maxIterations: 10, thresholds: [ .cpuUser: .init( /// `10 - 1 == 9`% tolerance. @@ -81,12 +77,10 @@ let benchmarks: @Sendable () -> Void = { ] ) ) { benchmark in - for _ in 0..<10 { - let bigMessageStream = messageStreams.removeFirst() - let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) - for try await part in sequence { - blackHole(part) - } + let bigMessageStream = messageStreams.removeFirst() + let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + for try await part in sequence { + blackHole(part) } } } diff --git a/Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json diff --git a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json From 74a0aea72719a4575d5ed0ebf8a39a7c19faf4c8 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 02:49:01 +0330 Subject: [PATCH 45/80] minor refinements --- Benchmarks/Parser/Parser.swift | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index a9e9c62..c4c0377 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -4,11 +4,8 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" let bigMessage = makeMessage(boundary: boundary, size: 1 << 29) // 512MiB: Big message - var messageStreams = (0..<500).map { - _ in makeParsingStream(for: bigMessage, chunkSize: 1 << 14) // 16KiB: Realistic streaming chunk size - } - let streamingStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + let streamingParserAllocationsStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( "StreamingParserAllocations", configuration: .init( @@ -16,12 +13,16 @@ let benchmarks: @Sendable () -> Void = { maxIterations: 1 ) ) { benchmark in - let streamingSequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: streamingStream) + let streamingSequence = StreamingMultipartParserAsyncSequence( + boundary: boundary, + buffer: streamingParserAllocationsStream + ) for try await part in streamingSequence { blackHole(part) } } + let streamingParserCPUTimeStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( "StreamingParserCPUTime", configuration: .init( @@ -39,14 +40,16 @@ let benchmarks: @Sendable () -> Void = { ] ) ) { benchmark in - let bigMessageStream = messageStreams.removeFirst() - let streamingSequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + let streamingSequence = StreamingMultipartParserAsyncSequence( + boundary: boundary, + buffer: streamingParserCPUTimeStream + ) for try await part in streamingSequence { blackHole(part) } } - let collatingStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + let collatingParserAllocationsStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( "CollatingParserAllocations", configuration: .init( @@ -54,12 +57,16 @@ let benchmarks: @Sendable () -> Void = { maxIterations: 1 ) ) { benchmark in - let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: collatingStream) + let sequence = MultipartParserAsyncSequence( + boundary: boundary, + buffer: collatingParserAllocationsStream + ) for try await part in sequence { blackHole(part) } } + let collatingParserCPUTimeStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( "CollatingParserCPUTime", configuration: .init( @@ -77,17 +84,20 @@ let benchmarks: @Sendable () -> Void = { ] ) ) { benchmark in - let bigMessageStream = messageStreams.removeFirst() - let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bigMessageStream) + let sequence = MultipartParserAsyncSequence( + boundary: boundary, + buffer: collatingParserCPUTimeStream + ) for try await part in sequence { blackHole(part) } } } -private func makeParsingStream(for message: Body, chunkSize: Int) -> AsyncStream< - Body.SubSequence -> +private func makeParsingStream( + for message: Body, + chunkSize: Int +) -> AsyncStream where Body.SubSequence: Sendable { AsyncStream { continuation in var offset = message.startIndex From 9b748cc1ef8c3cf8c39e0d9527e03f1af80563c6 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 03:29:21 +0330 Subject: [PATCH 46/80] try something weird --- Benchmarks/Parser/Parser.swift | 153 +++++++++++++----- ...Parser.10xCollatingParserCPUTime.p90.json} | 0 ...Parser.10xStreamingParserCPUTime.p90.json} | 0 3 files changed, 112 insertions(+), 41 deletions(-) rename Benchmarks/Thresholds/{Parser.CollatingParserCPUTime.p90.json => Parser.10xCollatingParserCPUTime.p90.json} (100%) rename Benchmarks/Thresholds/{Parser.StreamingParserCPUTime.p90.json => Parser.10xStreamingParserCPUTime.p90.json} (100%) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index c4c0377..6a5460e 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -3,28 +3,36 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - let bigMessage = makeMessage(boundary: boundary, size: 1 << 29) // 512MiB: Big message + let bigMessage = makeMessage(boundary: boundary, size: 1 << 27) // 128MiB: Big message + nonisolated(unsafe) var bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream2 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream3 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream4 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream5 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream6 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream7 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream8 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream9 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + nonisolated(unsafe) var bufferStream10 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - let streamingParserAllocationsStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( "StreamingParserAllocations", configuration: .init( metrics: [.mallocCountTotal], - maxIterations: 1 + maxIterations: 1, + teardown: { + bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + } ) ) { benchmark in - let streamingSequence = StreamingMultipartParserAsyncSequence( - boundary: boundary, - buffer: streamingParserAllocationsStream - ) + let streamingSequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) for try await part in streamingSequence { blackHole(part) } } - let streamingParserCPUTimeStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( - "StreamingParserCPUTime", + "10xStreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], maxDuration: .seconds(20), @@ -37,38 +45,68 @@ let benchmarks: @Sendable () -> Void = { /// 26ms of tolerance. absolute: [.p90: 26_000_000] ) - ] + ], + teardown: { + bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream2 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream3 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream4 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream5 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream6 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream7 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream8 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream9 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream10 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + } ) ) { benchmark in - let streamingSequence = StreamingMultipartParserAsyncSequence( - boundary: boundary, - buffer: streamingParserCPUTimeStream - ) - for try await part in streamingSequence { - blackHole(part) - } + let sequence1 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) + for try await part in sequence1 { blackHole(part) } + + let sequence2 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream2) + for try await part in sequence2 { blackHole(part) } + + let sequence3 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream3) + for try await part in sequence3 { blackHole(part) } + + let sequence4 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream4) + for try await part in sequence4 { blackHole(part) } + + let sequence5 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream5) + for try await part in sequence5 { blackHole(part) } + + let sequence6 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream6) + for try await part in sequence6 { blackHole(part) } + + let sequence7 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream7) + for try await part in sequence7 { blackHole(part) } + + let sequence8 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream8) + for try await part in sequence8 { blackHole(part) } + + let sequence9 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream9) + for try await part in sequence9 { blackHole(part) } + + let sequence10 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream10) + for try await part in sequence10 { blackHole(part) } } - let collatingParserAllocationsStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( "CollatingParserAllocations", configuration: .init( metrics: [.mallocCountTotal], - maxIterations: 1 + maxIterations: 1, + teardown: { + bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + } ) ) { benchmark in - let sequence = MultipartParserAsyncSequence( - boundary: boundary, - buffer: collatingParserAllocationsStream - ) - for try await part in sequence { - blackHole(part) - } + let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) + for try await part in sequence { blackHole(part) } } - let collatingParserCPUTimeStream = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) Benchmark( - "CollatingParserCPUTime", + "10xCollatingParserCPUTime", configuration: .init( metrics: [.cpuUser], maxDuration: .seconds(20), @@ -81,25 +119,58 @@ let benchmarks: @Sendable () -> Void = { /// 26ms of tolerance. absolute: [.p90: 26_000_000] ) - ] + ], + teardown: { + bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream2 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream3 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream4 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream5 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream6 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream7 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream8 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream9 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStream10 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + } ) ) { benchmark in - let sequence = MultipartParserAsyncSequence( - boundary: boundary, - buffer: collatingParserCPUTimeStream - ) - for try await part in sequence { - blackHole(part) - } + let sequence1 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) + for try await part in sequence1 { blackHole(part) } + + let sequence2 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream2) + for try await part in sequence2 { blackHole(part) } + + let sequence3 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream3) + for try await part in sequence3 { blackHole(part) } + + let sequence4 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream4) + for try await part in sequence4 { blackHole(part) } + + let sequence5 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream5) + for try await part in sequence5 { blackHole(part) } + + let sequence6 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream6) + for try await part in sequence6 { blackHole(part) } + + let sequence7 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream7) + for try await part in sequence7 { blackHole(part) } + + let sequence8 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream8) + for try await part in sequence8 { blackHole(part) } + + let sequence9 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream9) + for try await part in sequence9 { blackHole(part) } + + let sequence10 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream10) + for try await part in sequence10 { blackHole(part) } } } -private func makeParsingStream( - for message: Body, +private func makeParsingStream( + for message: ArraySlice, chunkSize: Int -) -> AsyncStream -where Body.SubSequence: Sendable { - AsyncStream { continuation in +) -> AsyncStream> { + AsyncStream { continuation in var offset = message.startIndex while offset < message.endIndex { let endIndex = min(message.endIndex, message.index(offset, offsetBy: chunkSize)) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json From 6128a4a75c1a99a50959fdf4bb60d41d7e9ff62a Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 03:58:40 +0330 Subject: [PATCH 47/80] try different method --- .github/workflows/benchmark.yml | 2 +- Benchmarks/Parser/Parser.swift | 128 ++++-------------- ...arser.100xCollatingParserCPUTime.p90.json} | 0 ...arser.100xStreamingParserCPUTime.p90.json} | 0 4 files changed, 27 insertions(+), 103 deletions(-) rename Benchmarks/Thresholds/{Parser.10xCollatingParserCPUTime.p90.json => Parser.100xCollatingParserCPUTime.p90.json} (100%) rename Benchmarks/Thresholds/{Parser.10xStreamingParserCPUTime.p90.json => Parser.100xStreamingParserCPUTime.p90.json} (100%) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5c32b13..f148667 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,7 +20,7 @@ jobs: runs-on: labels: - runs-on - - runner=2cpu-4ram + - runner=2cpu-8ram - run-id=${{ github.run_id }} container: swift:noble diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 6a5460e..28d69cc 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -3,36 +3,28 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - let bigMessage = makeMessage(boundary: boundary, size: 1 << 27) // 128MiB: Big message - nonisolated(unsafe) var bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream2 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream3 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream4 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream5 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream6 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream7 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream8 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream9 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - nonisolated(unsafe) var bufferStream10 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + let bigMessage = makeMessage(boundary: boundary, size: 1 << 26) // 64MiB: Big message + var bufferStreams = (0..<100).map { _ in + // 100 empty streams + AsyncStream> { $0.finish() } + } Benchmark( "StreamingParserAllocations", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1, - teardown: { - bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + setup: { + bufferStreams[0] = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) } ) ) { benchmark in - let streamingSequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) - for try await part in streamingSequence { - blackHole(part) - } + let sequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStreams[0]) + for try await part in sequence { blackHole(part) } } Benchmark( - "10xStreamingParserCPUTime", + "100xStreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], maxDuration: .seconds(20), @@ -46,49 +38,15 @@ let benchmarks: @Sendable () -> Void = { absolute: [.p90: 26_000_000] ) ], - teardown: { - bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream2 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream3 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream4 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream5 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream6 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream7 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream8 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream9 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream10 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + setup: { + bufferStreams = (0..<100).map { _ in makeParsingStream(for: bigMessage, chunkSize: 1 << 14) } } ) ) { benchmark in - let sequence1 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) - for try await part in sequence1 { blackHole(part) } - - let sequence2 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream2) - for try await part in sequence2 { blackHole(part) } - - let sequence3 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream3) - for try await part in sequence3 { blackHole(part) } - - let sequence4 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream4) - for try await part in sequence4 { blackHole(part) } - - let sequence5 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream5) - for try await part in sequence5 { blackHole(part) } - - let sequence6 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream6) - for try await part in sequence6 { blackHole(part) } - - let sequence7 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream7) - for try await part in sequence7 { blackHole(part) } - - let sequence8 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream8) - for try await part in sequence8 { blackHole(part) } - - let sequence9 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream9) - for try await part in sequence9 { blackHole(part) } - - let sequence10 = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream10) - for try await part in sequence10 { blackHole(part) } + for bufferStream in bufferStreams { + let sequence = StreamingMultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream) + for try await part in sequence { blackHole(part) } + } } Benchmark( @@ -96,17 +54,17 @@ let benchmarks: @Sendable () -> Void = { configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1, - teardown: { - bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + setup: { + bufferStreams[0] = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) } ) ) { benchmark in - let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) + let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStreams[0]) for try await part in sequence { blackHole(part) } } Benchmark( - "10xCollatingParserCPUTime", + "100xCollatingParserCPUTime", configuration: .init( metrics: [.cpuUser], maxDuration: .seconds(20), @@ -120,49 +78,15 @@ let benchmarks: @Sendable () -> Void = { absolute: [.p90: 26_000_000] ) ], - teardown: { - bufferStream1 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream2 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream3 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream4 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream5 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream6 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream7 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream8 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream9 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) - bufferStream10 = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + setup: { + bufferStreams = (0..<100).map { _ in makeParsingStream(for: bigMessage, chunkSize: 1 << 14) } } ) ) { benchmark in - let sequence1 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream1) - for try await part in sequence1 { blackHole(part) } - - let sequence2 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream2) - for try await part in sequence2 { blackHole(part) } - - let sequence3 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream3) - for try await part in sequence3 { blackHole(part) } - - let sequence4 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream4) - for try await part in sequence4 { blackHole(part) } - - let sequence5 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream5) - for try await part in sequence5 { blackHole(part) } - - let sequence6 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream6) - for try await part in sequence6 { blackHole(part) } - - let sequence7 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream7) - for try await part in sequence7 { blackHole(part) } - - let sequence8 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream8) - for try await part in sequence8 { blackHole(part) } - - let sequence9 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream9) - for try await part in sequence9 { blackHole(part) } - - let sequence10 = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream10) - for try await part in sequence10 { blackHole(part) } + for bufferStream in bufferStreams { + let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream) + for try await part in sequence { blackHole(part) } + } } } diff --git a/Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.100xCollatingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.100xCollatingParserCPUTime.p90.json diff --git a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.100xStreamingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.100xStreamingParserCPUTime.p90.json From a89c63836a0c968b93bf5c6b47e0c2bdfa4114a2 Mon Sep 17 00:00:00 2001 From: Paul Toffoloni Date: Mon, 13 Jan 2025 10:48:21 +0100 Subject: [PATCH 48/80] Use `unsafeUninitializedCapacity` initialiser --- Benchmarks/Parser/Parser.swift | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 28d69cc..5068b38 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -4,10 +4,7 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" let bigMessage = makeMessage(boundary: boundary, size: 1 << 26) // 64MiB: Big message - var bufferStreams = (0..<100).map { _ in - // 100 empty streams - AsyncStream> { $0.finish() } - } + var bufferStreams: [AsyncStream>] = .init(unsafeUninitializedCapacity: 100) { _, _ in } Benchmark( "StreamingParserAllocations", From 79dc04df96745550fe038351d57c4f6cdb6444f0 Mon Sep 17 00:00:00 2001 From: Paul Toffoloni Date: Mon, 13 Jan 2025 11:04:02 +0100 Subject: [PATCH 49/80] Revert "Use `unsafeUninitializedCapacity` initialiser" This reverts commit a89c63836a0c968b93bf5c6b47e0c2bdfa4114a2. --- Benchmarks/Parser/Parser.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 5068b38..28d69cc 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -4,7 +4,10 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" let bigMessage = makeMessage(boundary: boundary, size: 1 << 26) // 64MiB: Big message - var bufferStreams: [AsyncStream>] = .init(unsafeUninitializedCapacity: 100) { _, _ in } + var bufferStreams = (0..<100).map { _ in + // 100 empty streams + AsyncStream> { $0.finish() } + } Benchmark( "StreamingParserAllocations", From 75282372ca946cd4ca5e2522fb0a4b1125eb636e Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 20:43:47 +0330 Subject: [PATCH 50/80] Try something --- Benchmarks/.gitignore | 1 + Benchmarks/Package.swift | 4 ++ Benchmarks/Parser/Parser.swift | 40 ++++++------------- ...Parser.10xCollatingParserCPUTime.p90.json} | 0 ...Parser.10xStreamingParserCPUTime.p90.json} | 0 5 files changed, 18 insertions(+), 27 deletions(-) create mode 120000 Benchmarks/.gitignore rename Benchmarks/Thresholds/{Parser.100xCollatingParserCPUTime.p90.json => Parser.10xCollatingParserCPUTime.p90.json} (100%) rename Benchmarks/Thresholds/{Parser.100xStreamingParserCPUTime.p90.json => Parser.10xStreamingParserCPUTime.p90.json} (100%) diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore new file mode 120000 index 0000000..5a19b83 --- /dev/null +++ b/Benchmarks/.gitignore @@ -0,0 +1 @@ +../.gitignore \ No newline at end of file diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift index 656d685..beafa40 100644 --- a/Benchmarks/Package.swift +++ b/Benchmarks/Package.swift @@ -10,6 +10,8 @@ let package = Package( dependencies: [ .package(path: "../"), .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.27.0"), + .package(url: "https://github.com/apple/swift-async-algorithms.git", from: "1.0.3"), + .package(url: "https://github.com/apple/swift-algorithms.git", from: "1.2.0"), ], targets: [ .executableTarget( @@ -28,6 +30,8 @@ let package = Package( dependencies: [ .product(name: "MultipartKit", package: "multipart-kit"), .product(name: "Benchmark", package: "package-benchmark"), + .product(name: "Algorithms", package: "swift-algorithms"), + .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), ], path: "Parser", plugins: [ diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 28d69cc..86d903a 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -1,13 +1,14 @@ +import Algorithms +import AsyncAlgorithms import Benchmark import MultipartKit let benchmarks: @Sendable () -> Void = { + print("Starting benchmarks") let boundary = "boundary123" - let bigMessage = makeMessage(boundary: boundary, size: 1 << 26) // 64MiB: Big message - var bufferStreams = (0..<100).map { _ in - // 100 empty streams - AsyncStream> { $0.finish() } - } + // 64MiB: Big message, 16KiB: Chunk size + let chunkedMessage = makeMessage(boundary: boundary, size: 1 << 26).chunks(ofCount: 1 << 14) + var bufferStreams: [AsyncSyncSequence>>] = .init(unsafeUninitializedCapacity: 10) { _, _ in } Benchmark( "StreamingParserAllocations", @@ -15,7 +16,7 @@ let benchmarks: @Sendable () -> Void = { metrics: [.mallocCountTotal], maxIterations: 1, setup: { - bufferStreams[0] = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStreams[0] = chunkedMessage.async } ) ) { benchmark in @@ -24,7 +25,7 @@ let benchmarks: @Sendable () -> Void = { } Benchmark( - "100xStreamingParserCPUTime", + "10xStreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], maxDuration: .seconds(20), @@ -39,7 +40,7 @@ let benchmarks: @Sendable () -> Void = { ) ], setup: { - bufferStreams = (0..<100).map { _ in makeParsingStream(for: bigMessage, chunkSize: 1 << 14) } + bufferStreams = (0..<10).map { _ in chunkedMessage.async } } ) ) { benchmark in @@ -55,7 +56,7 @@ let benchmarks: @Sendable () -> Void = { metrics: [.mallocCountTotal], maxIterations: 1, setup: { - bufferStreams[0] = makeParsingStream(for: bigMessage, chunkSize: 1 << 14) + bufferStreams[0] = chunkedMessage.async } ) ) { benchmark in @@ -64,9 +65,9 @@ let benchmarks: @Sendable () -> Void = { } Benchmark( - "100xCollatingParserCPUTime", + "10xCollatingParserCPUTime", configuration: .init( - metrics: [.cpuUser], + metrics: [.cpuUser, .peakMemoryResident], maxDuration: .seconds(20), maxIterations: 10, thresholds: [ @@ -79,7 +80,7 @@ let benchmarks: @Sendable () -> Void = { ) ], setup: { - bufferStreams = (0..<100).map { _ in makeParsingStream(for: bigMessage, chunkSize: 1 << 14) } + bufferStreams = (0..<10).map { _ in chunkedMessage.async } } ) ) { benchmark in @@ -90,21 +91,6 @@ let benchmarks: @Sendable () -> Void = { } } -private func makeParsingStream( - for message: ArraySlice, - chunkSize: Int -) -> AsyncStream> { - AsyncStream { continuation in - var offset = message.startIndex - while offset < message.endIndex { - let endIndex = min(message.endIndex, message.index(offset, offsetBy: chunkSize)) - continuation.yield(message[offset.. ArraySlice { var message = ArraySlice( """ diff --git a/Benchmarks/Thresholds/Parser.100xCollatingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.100xCollatingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json diff --git a/Benchmarks/Thresholds/Parser.100xStreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.100xStreamingParserCPUTime.p90.json rename to Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json From 0ec09f1d1fa3235eddf9c0c9b42aebe291c65ac9 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 20:48:37 +0330 Subject: [PATCH 51/80] minor fixes --- .github/workflows/benchmark.yml | 2 +- Benchmarks/Parser/Parser.swift | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f148667..5c32b13 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,7 +20,7 @@ jobs: runs-on: labels: - runs-on - - runner=2cpu-8ram + - runner=2cpu-4ram - run-id=${{ github.run_id }} container: swift:noble diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 86d903a..bd747fb 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -4,7 +4,6 @@ import Benchmark import MultipartKit let benchmarks: @Sendable () -> Void = { - print("Starting benchmarks") let boundary = "boundary123" // 64MiB: Big message, 16KiB: Chunk size let chunkedMessage = makeMessage(boundary: boundary, size: 1 << 26).chunks(ofCount: 1 << 14) From 9b300454d8937869978ba6b72de39614167b62e3 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 20:55:09 +0330 Subject: [PATCH 52/80] minor --- Benchmarks/Parser/Parser.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index bd747fb..3c773df 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -7,7 +7,7 @@ let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" // 64MiB: Big message, 16KiB: Chunk size let chunkedMessage = makeMessage(boundary: boundary, size: 1 << 26).chunks(ofCount: 1 << 14) - var bufferStreams: [AsyncSyncSequence>>] = .init(unsafeUninitializedCapacity: 10) { _, _ in } + var bufferStreams = (0..<10).map { _ in chunkedMessage.async } Benchmark( "StreamingParserAllocations", From 4c4cc779bfa72bd4ed318add379d9c72a18728e6 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 21:35:56 +0330 Subject: [PATCH 53/80] refinements --- Benchmarks/Parser/Parser.swift | 10 +++++----- .../Parser.10xCollatingParserCPUTime.p90.json | 3 --- .../Parser.10xStreamingParserCPUTime.p90.json | 3 --- .../Thresholds/Parser.CollatingParserCPUTime.p90.json | 3 +++ .../Thresholds/Parser.StreamingParserCPUTime.p90.json | 4 ++++ 5 files changed, 12 insertions(+), 11 deletions(-) delete mode 100644 Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json delete mode 100644 Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json create mode 100644 Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json create mode 100644 Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 3c773df..f54087e 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -5,8 +5,8 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - // 64MiB: Big message, 16KiB: Chunk size - let chunkedMessage = makeMessage(boundary: boundary, size: 1 << 26).chunks(ofCount: 1 << 14) + // 512MiB: Big message, 16KiB: Chunk size + let chunkedMessage = makeMessage(boundary: boundary, size: 1 << 29).chunks(ofCount: 1 << 14) var bufferStreams = (0..<10).map { _ in chunkedMessage.async } Benchmark( @@ -24,7 +24,7 @@ let benchmarks: @Sendable () -> Void = { } Benchmark( - "10xStreamingParserCPUTime", + "StreamingParserCPUTime", configuration: .init( metrics: [.cpuUser], maxDuration: .seconds(20), @@ -64,9 +64,9 @@ let benchmarks: @Sendable () -> Void = { } Benchmark( - "10xCollatingParserCPUTime", + "CollatingParserCPUTime", configuration: .init( - metrics: [.cpuUser, .peakMemoryResident], + metrics: [.cpuUser], maxDuration: .seconds(20), maxIterations: 10, thresholds: [ diff --git a/Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json deleted file mode 100644 index 4c3c2ba..0000000 --- a/Benchmarks/Thresholds/Parser.10xCollatingParserCPUTime.p90.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "cpuUser": 385000000 -} diff --git a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json deleted file mode 100644 index 17b22a3..0000000 --- a/Benchmarks/Thresholds/Parser.10xStreamingParserCPUTime.p90.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "cpuUser": 365000000 -} diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json new file mode 100644 index 0000000..0567f41 --- /dev/null +++ b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json @@ -0,0 +1,3 @@ +{ + "cpuUser": 1100000000 +} diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json new file mode 100644 index 0000000..8e1d77f --- /dev/null +++ b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json @@ -0,0 +1,4 @@ +{ + "cpuUser": 1100000000 +} + From 3259d8cb1a01dc52ae46d93dd18328fa4f59be8b Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 21:46:51 +0330 Subject: [PATCH 54/80] smaller chunk --- Benchmarks/Parser/Parser.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index f54087e..80133c6 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -5,8 +5,8 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - // 512MiB: Big message, 16KiB: Chunk size - let chunkedMessage = makeMessage(boundary: boundary, size: 1 << 29).chunks(ofCount: 1 << 14) + // 128MiB: Big message, 16KiB: Chunk size + let chunkedMessage = ArraySlice(makeMessage(boundary: boundary, size: 1 << 27).chunks(ofCount: 1 << 14)) var bufferStreams = (0..<10).map { _ in chunkedMessage.async } Benchmark( From dcb061a9e0753957857e5701cf0cf93cfe04ad9d Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 21:59:30 +0330 Subject: [PATCH 55/80] use array --- Benchmarks/Parser/Parser.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 80133c6..578ada5 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -6,7 +6,7 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" // 128MiB: Big message, 16KiB: Chunk size - let chunkedMessage = ArraySlice(makeMessage(boundary: boundary, size: 1 << 27).chunks(ofCount: 1 << 14)) + let chunkedMessage = Array(makeMessage(boundary: boundary, size: 1 << 27).chunks(ofCount: 1 << 14)) var bufferStreams = (0..<10).map { _ in chunkedMessage.async } Benchmark( From e946cb5b071cd2b984bddb324adb862ce572b857 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 22:58:06 +0330 Subject: [PATCH 56/80] more fixes --- Benchmarks/Package.swift | 2 - Benchmarks/Parser/AsyncSyncSequence.swift | 49 +++++++++++ Benchmarks/Parser/Parser.swift | 84 +++++++++++-------- .../Parser.CollatingParserCPUTime.p90.json | 2 +- .../Parser.StreamingParserCPUTime.p90.json | 2 +- 5 files changed, 102 insertions(+), 37 deletions(-) create mode 100644 Benchmarks/Parser/AsyncSyncSequence.swift diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift index beafa40..67a8d2b 100644 --- a/Benchmarks/Package.swift +++ b/Benchmarks/Package.swift @@ -10,7 +10,6 @@ let package = Package( dependencies: [ .package(path: "../"), .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.27.0"), - .package(url: "https://github.com/apple/swift-async-algorithms.git", from: "1.0.3"), .package(url: "https://github.com/apple/swift-algorithms.git", from: "1.2.0"), ], targets: [ @@ -31,7 +30,6 @@ let package = Package( .product(name: "MultipartKit", package: "multipart-kit"), .product(name: "Benchmark", package: "package-benchmark"), .product(name: "Algorithms", package: "swift-algorithms"), - .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), ], path: "Parser", plugins: [ diff --git a/Benchmarks/Parser/AsyncSyncSequence.swift b/Benchmarks/Parser/AsyncSyncSequence.swift new file mode 100644 index 0000000..bc383b8 --- /dev/null +++ b/Benchmarks/Parser/AsyncSyncSequence.swift @@ -0,0 +1,49 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Async Algorithms open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +extension Array> { + var async: AsyncSyncSequence { + AsyncSyncSequence(self) + } +} + +/// An asynchronous sequence composed from a synchronous sequence, that releases the reference to +/// the base sequence when an iterator is created. So you can only iterate once. +/// +/// Not safe. Only for testing purposes. +/// Use `swift-algorithms`'s `AsyncSyncSequence`` instead if you're looking for something like this. +final class AsyncSyncSequence: AsyncSequence { + typealias Base = Array> + typealias Element = Base.Element + + struct Iterator: AsyncIteratorProtocol { + var iterator: Base.Iterator? + + init(_ iterator: Base.Iterator) { + self.iterator = iterator + } + + mutating func next() async -> Base.Element? { + iterator?.next() + } + } + + var base: Base? + + init(_ base: Base) { + self.base = base + } + + func makeAsyncIterator() -> Iterator { + defer { self.base = nil } // release the reference so no CoW is triggered + return Iterator(base.unsafelyUnwrapped.makeIterator()) + } +} diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 578ada5..8a7192a 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -1,74 +1,90 @@ import Algorithms -import AsyncAlgorithms import Benchmark import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - // 128MiB: Big message, 16KiB: Chunk size - let chunkedMessage = Array(makeMessage(boundary: boundary, size: 1 << 27).chunks(ofCount: 1 << 14)) - var bufferStreams = (0..<10).map { _ in chunkedMessage.async } + // 256MiB: Big message, 16KiB: Chunk size + let chunkedMessage = Array(makeMessage(boundary: boundary, size: 1 << 28).chunks(ofCount: 1 << 14)) + let cpuBenchsWarmupIterations = 1 + let cpuBenchsMaxIterations = 10 + let cpuBenchsTotalIterations = cpuBenchsWarmupIterations + cpuBenchsMaxIterations + var bufferStreams = (0.. Void = { /// 26ms of tolerance. absolute: [.p90: 26_000_000] ) - ], - setup: { - bufferStreams = (0..<10).map { _ in chunkedMessage.async } - } + ] ) ) { benchmark in - for bufferStream in bufferStreams { - let sequence = MultipartParserAsyncSequence(boundary: boundary, buffer: bufferStream) - for try await part in sequence { blackHole(part) } + defer { collatingParserIterated += 1 } + + let sequence = MultipartParserAsyncSequence( + boundary: boundary, + buffer: bufferStreams[collatingParserIterated] + ) + for try await part in sequence { + blackHole(part) } } } diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json index 0567f41..33e46c4 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 1100000000 + "cpuUser": 600000000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json index 8e1d77f..a8d6ac8 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json @@ -1,4 +1,4 @@ { - "cpuUser": 1100000000 + "cpuUser": 600000000 } From b23f6c7af9525f1b6c6f0f4c9bb845e2f7381dda Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 23:00:15 +0330 Subject: [PATCH 57/80] minor fixes --- Benchmarks/Parser/AsyncSyncSequence.swift | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Benchmarks/Parser/AsyncSyncSequence.swift b/Benchmarks/Parser/AsyncSyncSequence.swift index bc383b8..46e6ac4 100644 --- a/Benchmarks/Parser/AsyncSyncSequence.swift +++ b/Benchmarks/Parser/AsyncSyncSequence.swift @@ -1,14 +1,3 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Async Algorithms open source project -// -// Copyright (c) 2022 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// -//===----------------------------------------------------------------------===// - extension Array> { var async: AsyncSyncSequence { AsyncSyncSequence(self) From c6d84ad0460613fa6589d689b64ee6bddd6dd5ef Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Mon, 13 Jan 2025 23:06:19 +0330 Subject: [PATCH 58/80] refinements --- Benchmarks/Parser/Parser.swift | 16 ++++++++++++---- .../Parser.CollatingParserCPUTime.p90.json | 3 --- ...Parser.CollatingParserCPUTime_256MiB.p90.json | 3 +++ .../Parser.StreamingParserCPUTime.p90.json | 4 ---- ...Parser.StreamingParserCPUTime_256MiB.p90.json | 4 ++++ 5 files changed, 19 insertions(+), 11 deletions(-) delete mode 100644 Benchmarks/Thresholds/Parser.CollatingParserCPUTime.p90.json create mode 100644 Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json delete mode 100644 Benchmarks/Thresholds/Parser.StreamingParserCPUTime.p90.json create mode 100644 Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 8a7192a..3e55b59 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -4,8 +4,16 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - // 256MiB: Big message, 16KiB: Chunk size - let chunkedMessage = Array(makeMessage(boundary: boundary, size: 1 << 28).chunks(ofCount: 1 << 14)) + let sizeInMiB = 256 + let chunkSizeInKiB = 16 + let chunkedMessage = Array( + makeMessage( + boundary: boundary, + size: sizeInMiB << 20 + ).chunks( + ofCount: chunkSizeInKiB << 10 + ) + ) let cpuBenchsWarmupIterations = 1 let cpuBenchsMaxIterations = 10 let cpuBenchsTotalIterations = cpuBenchsWarmupIterations + cpuBenchsMaxIterations @@ -31,7 +39,7 @@ let benchmarks: @Sendable () -> Void = { var streamingParserIterated = 0 bufferStreams = (0.. Void = { var collatingParserIterated = 0 bufferStreams = (0.. Date: Tue, 14 Jan 2025 01:32:49 +0330 Subject: [PATCH 59/80] just use AsyncStream --- Benchmarks/Parser/AsyncSyncSequence.swift | 38 -------------------- Benchmarks/Parser/Parser.swift | 43 ++++++++++++++++------- 2 files changed, 31 insertions(+), 50 deletions(-) delete mode 100644 Benchmarks/Parser/AsyncSyncSequence.swift diff --git a/Benchmarks/Parser/AsyncSyncSequence.swift b/Benchmarks/Parser/AsyncSyncSequence.swift deleted file mode 100644 index 46e6ac4..0000000 --- a/Benchmarks/Parser/AsyncSyncSequence.swift +++ /dev/null @@ -1,38 +0,0 @@ -extension Array> { - var async: AsyncSyncSequence { - AsyncSyncSequence(self) - } -} - -/// An asynchronous sequence composed from a synchronous sequence, that releases the reference to -/// the base sequence when an iterator is created. So you can only iterate once. -/// -/// Not safe. Only for testing purposes. -/// Use `swift-algorithms`'s `AsyncSyncSequence`` instead if you're looking for something like this. -final class AsyncSyncSequence: AsyncSequence { - typealias Base = Array> - typealias Element = Base.Element - - struct Iterator: AsyncIteratorProtocol { - var iterator: Base.Iterator? - - init(_ iterator: Base.Iterator) { - self.iterator = iterator - } - - mutating func next() async -> Base.Element? { - iterator?.next() - } - } - - var base: Base? - - init(_ base: Base) { - self.base = base - } - - func makeAsyncIterator() -> Iterator { - defer { self.base = nil } // release the reference so no CoW is triggered - return Iterator(base.unsafelyUnwrapped.makeIterator()) - } -} diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 3e55b59..b31bb16 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -6,20 +6,24 @@ let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" let sizeInMiB = 256 let chunkSizeInKiB = 16 - let chunkedMessage = Array( - makeMessage( - boundary: boundary, - size: sizeInMiB << 20 - ).chunks( - ofCount: chunkSizeInKiB << 10 - ) + let message = makeMessage( + boundary: boundary, + size: sizeInMiB << 20 ) + + func makeStream() -> AsyncStream> { + makeParsingStream( + for: message, + chunkSize: chunkSizeInKiB << 10 + ) + } + let cpuBenchsWarmupIterations = 1 let cpuBenchsMaxIterations = 10 let cpuBenchsTotalIterations = cpuBenchsWarmupIterations + cpuBenchsMaxIterations - var bufferStreams = (0.. Void = { } var streamingParserIterated = 0 - bufferStreams = (0.. Void = { } } - bufferStreams[0] = chunkedMessage.async + bufferStreams[0] = makeStream() Benchmark( "CollatingParserAllocations", configuration: .init( @@ -85,7 +89,7 @@ let benchmarks: @Sendable () -> Void = { } var collatingParserIterated = 0 - bufferStreams = (0.. Void = { } } +private func makeParsingStream( + for message: ArraySlice, + chunkSize: Int +) -> AsyncStream> { + AsyncStream { continuation in + var offset = message.startIndex + while offset < message.endIndex { + let endIndex = min(message.endIndex, message.index(offset, offsetBy: chunkSize)) + continuation.yield(message[offset.. ArraySlice { var message = ArraySlice( """ From 8bfa20b30adaf96ea7c9af5fd84783addd087898 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 01:38:38 +0330 Subject: [PATCH 60/80] better thresholds --- Benchmarks/Parser/Parser.swift | 16 ++++++++-------- ...Parser.CollatingParserCPUTime_256MiB.p90.json | 2 +- ...Parser.StreamingParserCPUTime_256MiB.p90.json | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index b31bb16..b6c2db6 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -51,11 +51,11 @@ let benchmarks: @Sendable () -> Void = { maxIterations: cpuBenchsMaxIterations, thresholds: [ .cpuUser: .init( - /// `10 - 1 == 9`% tolerance. + /// `5 - 1 == 4`% tolerance. /// Will rely on the absolute threshold as the tighter threshold. - relative: [.p90: 9], - /// 26ms of tolerance. - absolute: [.p90: 26_000_000] + relative: [.p90: 5], + /// 21ms of tolerance. + absolute: [.p90: 21_000_000] ) ] ) @@ -99,11 +99,11 @@ let benchmarks: @Sendable () -> Void = { maxIterations: cpuBenchsMaxIterations, thresholds: [ .cpuUser: .init( - /// `10 - 1 == 9`% tolerance. + /// `5 - 1 == 4`% tolerance. /// Will rely on the absolute threshold as the tighter threshold. - relative: [.p90: 9], - /// 26ms of tolerance. - absolute: [.p90: 26_000_000] + relative: [.p90: 5], + /// 21ms of tolerance. + absolute: [.p90: 21_000_000] ) ] ) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json index 53a4a5d..087acdb 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 565000000 + "cpuUser": 570000000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json index 8dcef4a..5523a7c 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json @@ -1,4 +1,4 @@ { - "cpuUser": 555000000 + "cpuUser": 540000000 } From 263c36ce9146efe1d9a49d337c4a7fc47cbf937b Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:17:19 +0330 Subject: [PATCH 61/80] more benchs + revert to `AsyncSyncStream` for max accuracy --- Benchmarks/Package.swift | 2 - Benchmarks/Parser/AsyncSyncSequence.swift | 37 ++++++ Benchmarks/Parser/Parser.swift | 117 ++++++++++++------ ....CollatingParserAllocations_0MiB.p90.json} | 0 ...CollatingParserAllocations_256MiB.p90.json | 3 + ....StreamingParserAllocations_0MiB.p90.json} | 0 ...StreamingParserAllocations_256MiB.p90.json | 3 + 7 files changed, 123 insertions(+), 39 deletions(-) create mode 100644 Benchmarks/Parser/AsyncSyncSequence.swift rename Benchmarks/Thresholds/{Parser.CollatingParserAllocations.p90.json => Parser.CollatingParserAllocations_0MiB.p90.json} (100%) create mode 100644 Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json rename Benchmarks/Thresholds/{Parser.StreamingParserAllocations.p90.json => Parser.StreamingParserAllocations_0MiB.p90.json} (100%) create mode 100644 Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift index 67a8d2b..656d685 100644 --- a/Benchmarks/Package.swift +++ b/Benchmarks/Package.swift @@ -10,7 +10,6 @@ let package = Package( dependencies: [ .package(path: "../"), .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.27.0"), - .package(url: "https://github.com/apple/swift-algorithms.git", from: "1.2.0"), ], targets: [ .executableTarget( @@ -29,7 +28,6 @@ let package = Package( dependencies: [ .product(name: "MultipartKit", package: "multipart-kit"), .product(name: "Benchmark", package: "package-benchmark"), - .product(name: "Algorithms", package: "swift-algorithms"), ], path: "Parser", plugins: [ diff --git a/Benchmarks/Parser/AsyncSyncSequence.swift b/Benchmarks/Parser/AsyncSyncSequence.swift new file mode 100644 index 0000000..f593ad3 --- /dev/null +++ b/Benchmarks/Parser/AsyncSyncSequence.swift @@ -0,0 +1,37 @@ +extension Sequence { + var async: AsyncSyncSequence { + AsyncSyncSequence(self) + } +} + +/// An asynchronous sequence composed from a synchronous sequence, that releases the reference to +/// the base sequence when an iterator is created. So you can only iterate once. +/// +/// Not safe. Only for testing purposes. +/// Use `swift-algorithms`'s `AsyncSyncSequence`` instead if you're looking for something like this. +final class AsyncSyncSequence: AsyncSequence { + typealias Element = Base.Element + + struct Iterator: AsyncIteratorProtocol { + var iterator: Base.Iterator? + + init(_ iterator: Base.Iterator) { + self.iterator = iterator + } + + mutating func next() async -> Base.Element? { + iterator?.next() + } + } + + var base: Base? + + init(_ base: Base) { + self.base = base + } + + func makeAsyncIterator() -> Iterator { + defer { self.base = nil } // release the reference so no CoW is triggered + return Iterator(base.unsafelyUnwrapped.makeIterator()) + } +} diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index b6c2db6..4d27e86 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -4,28 +4,28 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - let sizeInMiB = 256 + let approxSizeInMiB = 256 let chunkSizeInKiB = 16 let message = makeMessage( boundary: boundary, - size: sizeInMiB << 20 + size: approxSizeInMiB << 20 ) - - func makeStream() -> AsyncStream> { - makeParsingStream( - for: message, - chunkSize: chunkSizeInKiB << 10 - ) - } + let chunkedMessage = makeChunks(for: message, chunkSize: chunkSizeInKiB << 10) let cpuBenchsWarmupIterations = 1 let cpuBenchsMaxIterations = 10 - let cpuBenchsTotalIterations = cpuBenchsWarmupIterations + cpuBenchsMaxIterations - var bufferStreams = (0..]>] = [] + var benchmarkIterated = 0 + + func refreshBufferStreams() { + bufferStreams = (0.. Void = { ) { benchmark in let sequence = StreamingMultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreams[0] + buffer: [ArraySlice]().async ) for try await part in sequence { blackHole(part) } } - var streamingParserIterated = 0 - bufferStreams = (0.. Void = { ] ) ) { benchmark in - defer { streamingParserIterated += 1 } + defer { benchmarkIterated += 1 } let sequence = StreamingMultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreams[streamingParserIterated] + buffer: bufferStreams[benchmarkIterated] ) for try await part in sequence { blackHole(part) } } - bufferStreams[0] = makeStream() Benchmark( - "CollatingParserAllocations", + "CollatingParserAllocations_0MiB", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1 @@ -81,17 +100,37 @@ let benchmarks: @Sendable () -> Void = { ) { benchmark in let sequence = MultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreams[0] + buffer: [ArraySlice]().async ) for try await part in sequence { blackHole(part) } } - var collatingParserIterated = 0 - bufferStreams = (0.. Void = { ] ) ) { benchmark in - defer { collatingParserIterated += 1 } + defer { benchmarkIterated += 1 } let sequence = MultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreams[collatingParserIterated] + buffer: bufferStreams[benchmarkIterated] ) for try await part in sequence { blackHole(part) @@ -120,19 +159,23 @@ let benchmarks: @Sendable () -> Void = { } } -private func makeParsingStream( +private func makeChunks( for message: ArraySlice, chunkSize: Int -) -> AsyncStream> { - AsyncStream { continuation in - var offset = message.startIndex - while offset < message.endIndex { - let endIndex = min(message.endIndex, message.index(offset, offsetBy: chunkSize)) - continuation.yield(message[offset.. [ArraySlice] { + var chunks: [ArraySlice] = [] + let approxChunksCount = (message.count / chunkSize) + 2 + chunks.reserveCapacity(approxChunksCount) + + var offset = message.startIndex + let endIndex = message.endIndex + while offset < endIndex { + let chunkEndIndex = min(endIndex, message.index(offset, offsetBy: chunkSize)) + chunks.append(message[offset.. ArraySlice { diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_0MiB.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.CollatingParserAllocations.p90.json rename to Benchmarks/Thresholds/Parser.CollatingParserAllocations_0MiB.p90.json diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json new file mode 100644 index 0000000..37f76dd --- /dev/null +++ b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal": 97 +} diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_0MiB.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.StreamingParserAllocations.p90.json rename to Benchmarks/Thresholds/Parser.StreamingParserAllocations_0MiB.p90.json diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json new file mode 100644 index 0000000..37f76dd --- /dev/null +++ b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal": 97 +} From 33efdfdb849233295396ffe4453989927af7a89e Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:24:34 +0330 Subject: [PATCH 62/80] refinements --- Benchmarks/Parser/Parser.swift | 22 +++++++++++++++---- ...CollatingParserAllocations_Empty.p90.json} | 0 ...StreamingParserAllocations_Empty.p90.json} | 0 3 files changed, 18 insertions(+), 4 deletions(-) rename Benchmarks/Thresholds/{Parser.CollatingParserAllocations_0MiB.p90.json => Parser.CollatingParserAllocations_Empty.p90.json} (100%) rename Benchmarks/Thresholds/{Parser.StreamingParserAllocations_0MiB.p90.json => Parser.StreamingParserAllocations_Empty.p90.json} (100%) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 4d27e86..5120b3d 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -24,16 +24,26 @@ let benchmarks: @Sendable () -> Void = { } } + func refreshBufferStreamsWithNoContents() { + bufferStreams = (0..]().async + } + } + + benchmarkIterated = 0 + refreshBufferStreamsWithNoContents() Benchmark( - "StreamingParserAllocations_0MiB", + "StreamingParserAllocations_Empty", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1 ) ) { benchmark in + defer { benchmarkIterated += 1 } + let sequence = StreamingMultipartParserAsyncSequence( boundary: boundary, - buffer: [ArraySlice]().async + buffer: bufferStreams[benchmarkIterated] ) for try await part in sequence { blackHole(part) @@ -91,16 +101,20 @@ let benchmarks: @Sendable () -> Void = { } } + benchmarkIterated = 0 + refreshBufferStreamsWithNoContents() Benchmark( - "CollatingParserAllocations_0MiB", + "CollatingParserAllocations_Empty", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1 ) ) { benchmark in + defer { benchmarkIterated += 1 } + let sequence = MultipartParserAsyncSequence( boundary: boundary, - buffer: [ArraySlice]().async + buffer: bufferStreams[benchmarkIterated] ) for try await part in sequence { blackHole(part) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_0MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.CollatingParserAllocations_0MiB.p90.json rename to Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_0MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json similarity index 100% rename from Benchmarks/Thresholds/Parser.StreamingParserAllocations_0MiB.p90.json rename to Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json From b71aceb11416e66a989546587d4e18ee60317d68 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:33:37 +0330 Subject: [PATCH 63/80] more benchmarks and refinements --- Benchmarks/Serializer/Serializer.swift | 28 ++++++++++++++----- ....100xSerializerCPUTime_1024Parts.p90.json} | 0 .../Serializer.SerializerAllocations.p90.json | 3 -- ...izer.SerializerAllocations_0Parts.p90.json | 3 ++ ...r.SerializerAllocations_1024Parts.p90.json | 3 ++ 5 files changed, 27 insertions(+), 10 deletions(-) rename Benchmarks/Thresholds/{Serializer.100xSerializerCPUTime.p90.json => Serializer.100xSerializerCPUTime_1024Parts.p90.json} (100%) delete mode 100644 Benchmarks/Thresholds/Serializer.SerializerAllocations.p90.json create mode 100644 Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json create mode 100644 Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json diff --git a/Benchmarks/Serializer/Serializer.swift b/Benchmarks/Serializer/Serializer.swift index 8c3bb4d..ce8e7fb 100644 --- a/Benchmarks/Serializer/Serializer.swift +++ b/Benchmarks/Serializer/Serializer.swift @@ -2,6 +2,7 @@ import Benchmark import MultipartKit let benchmarks: @Sendable () -> Void = { + let boundary = "boundary123" let examplePart: MultipartPart = .init( headerFields: .init([ .init(name: .contentDisposition, value: "form-data; name=\"file\"; filename=\"hello.txt\""), @@ -9,23 +10,36 @@ let benchmarks: @Sendable () -> Void = { ]), body: ArraySlice("Hello, world!".utf8) ) - let onePart: [MultipartPart] = [examplePart] - let repeatedParts: [MultipartPart] = .init(repeating: examplePart, count: 1 << 10) + let emptyParts: [MultipartPart>] = [] + let partCount = 1 << 10 + let repeatedParts: [MultipartPart] = .init(repeating: examplePart, count: partCount) Benchmark( - "SerializerAllocations", + "SerializerAllocations_0Parts", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1 ) ) { benchmark in - let serializer = MultipartSerializer(boundary: "boundary123") - let serialized = try serializer.serialize(parts: onePart) + let serializer = MultipartSerializer(boundary: boundary) + let serialized = try serializer.serialize(parts: emptyParts) blackHole(serialized) } Benchmark( - "100xSerializerCPUTime", + "SerializerAllocations_\(partCount)Parts", + configuration: .init( + metrics: [.mallocCountTotal], + maxIterations: 1 + ) + ) { benchmark in + let serializer = MultipartSerializer(boundary: boundary) + let serialized = try serializer.serialize(parts: repeatedParts) + blackHole(serialized) + } + + Benchmark( + "100xSerializerCPUTime_\(partCount)Parts", configuration: .init( metrics: [.cpuUser], maxDuration: .seconds(10), @@ -42,7 +56,7 @@ let benchmarks: @Sendable () -> Void = { ) ) { benchmark in for _ in 0..<100 { - let serializer = MultipartSerializer(boundary: "boundary123") + let serializer = MultipartSerializer(boundary: boundary) let serialized = try serializer.serialize(parts: repeatedParts) blackHole(serialized) } diff --git a/Benchmarks/Thresholds/Serializer.100xSerializerCPUTime.p90.json b/Benchmarks/Thresholds/Serializer.100xSerializerCPUTime_1024Parts.p90.json similarity index 100% rename from Benchmarks/Thresholds/Serializer.100xSerializerCPUTime.p90.json rename to Benchmarks/Thresholds/Serializer.100xSerializerCPUTime_1024Parts.p90.json diff --git a/Benchmarks/Thresholds/Serializer.SerializerAllocations.p90.json b/Benchmarks/Thresholds/Serializer.SerializerAllocations.p90.json deleted file mode 100644 index 17d7b17..0000000 --- a/Benchmarks/Thresholds/Serializer.SerializerAllocations.p90.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mallocCountTotal": 8 -} diff --git a/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json b/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json new file mode 100644 index 0000000..f91c250 --- /dev/null +++ b/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal": 3 +} diff --git a/Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json b/Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json new file mode 100644 index 0000000..229600a --- /dev/null +++ b/Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal": 3086 +} From bc3c03aebc8bab0095af06f92e38efc2ed2119ec Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:43:16 +0330 Subject: [PATCH 64/80] fixes --- Benchmarks/Parser/Parser.swift | 10 ++++++---- .../Parser.CollatingParserAllocations_256MiB.p90.json | 2 +- .../Parser.CollatingParserAllocations_Empty.p90.json | 2 +- .../Parser.StreamingParserAllocations_256MiB.p90.json | 2 +- .../Parser.StreamingParserAllocations_Empty.p90.json | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 5120b3d..3a1ca90 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -15,7 +15,9 @@ let benchmarks: @Sendable () -> Void = { let cpuBenchsWarmupIterations = 1 let cpuBenchsMaxIterations = 10 let maxBufferStreamsUsedInBenchs = cpuBenchsWarmupIterations + cpuBenchsMaxIterations + var bufferStreams: [AsyncSyncSequence<[ArraySlice]>] = [] + var bufferStreamsWithNoContents: [AsyncStream>] = [] var benchmarkIterated = 0 func refreshBufferStreams() { @@ -25,8 +27,8 @@ let benchmarks: @Sendable () -> Void = { } func refreshBufferStreamsWithNoContents() { - bufferStreams = (0..]().async + bufferStreamsWithNoContents = (0.. Void = { let sequence = StreamingMultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreams[benchmarkIterated] + buffer: bufferStreamsWithNoContents[benchmarkIterated] ) for try await part in sequence { blackHole(part) @@ -114,7 +116,7 @@ let benchmarks: @Sendable () -> Void = { let sequence = MultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreams[benchmarkIterated] + buffer: bufferStreamsWithNoContents[benchmarkIterated] ) for try await part in sequence { blackHole(part) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json index 37f76dd..f0d90b4 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 97 + "mallocCountTotal": 98000 } diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json index 17d7b17..36d82e7 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 8 + "mallocCountTotal": 9 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json index 37f76dd..f0d90b4 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 97 + "mallocCountTotal": 98000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json index 14184fb..d3eb4e9 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 6 + "mallocCountTotal": 7 } From b5d2aa6468b1bc0b576f946f1acbb2a79fb40e63 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:48:16 +0330 Subject: [PATCH 65/80] adjust thresholds --- .../Parser.CollatingParserAllocations_256MiB.p90.json | 2 +- .../Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json | 2 +- .../Parser.StreamingParserAllocations_256MiB.p90.json | 2 +- .../Thresholds/Parser.StreamingParserAllocations_Empty.p90.json | 2 +- .../Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json | 2 +- .../Thresholds/Serializer.SerializerAllocations_0Parts.p90.json | 2 +- .../Serializer.SerializerAllocations_1024Parts.p90.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json index f0d90b4..c4f80c1 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_256MiB.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 98000 + "mallocCountTotal": 82000 } diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json index 087acdb..33e46c4 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 570000000 + "cpuUser": 600000000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json index f0d90b4..c4f80c1 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_256MiB.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 98000 + "mallocCountTotal": 82000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json index d3eb4e9..17d7b17 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 7 + "mallocCountTotal": 8 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json index 5523a7c..a8d6ac8 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json @@ -1,4 +1,4 @@ { - "cpuUser": 540000000 + "cpuUser": 600000000 } diff --git a/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json b/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json index f91c250..c5fe12c 100644 --- a/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json +++ b/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 3 + "mallocCountTotal": 4 } diff --git a/Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json b/Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json index 229600a..f8c6bdc 100644 --- a/Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json +++ b/Benchmarks/Thresholds/Serializer.SerializerAllocations_1024Parts.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 3086 + "mallocCountTotal": 3087 } From 668bb5d551127c3070883cba9ec54c0d80045915 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:50:43 +0330 Subject: [PATCH 66/80] minor refinement --- Benchmarks/Parser/Parser.swift | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 3a1ca90..86fe4a3 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -6,11 +6,13 @@ let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" let approxSizeInMiB = 256 let chunkSizeInKiB = 16 - let message = makeMessage( - boundary: boundary, - size: approxSizeInMiB << 20 + let chunkedMessage = makeChunks( + for: makeMessage( + boundary: boundary, + size: approxSizeInMiB << 20 + ), + chunkSize: chunkSizeInKiB << 10 ) - let chunkedMessage = makeChunks(for: message, chunkSize: chunkSizeInKiB << 10) let cpuBenchsWarmupIterations = 1 let cpuBenchsMaxIterations = 10 From 2a9532847acd0fbc5ce80ebcb847d350a7285e08 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:52:43 +0330 Subject: [PATCH 67/80] thresholds --- Benchmarks/Parser/Parser.swift | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 86fe4a3..dba279f 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -60,7 +60,16 @@ let benchmarks: @Sendable () -> Void = { "StreamingParserAllocations_\(approxSizeInMiB)MiB", configuration: .init( metrics: [.mallocCountTotal], - maxIterations: 1 + maxIterations: 1, + thresholds: [ + .mallocCountTotal: .init( + /// `2 - 1 == 1`% tolerance. + /// Will rely on the absolute threshold as the tighter threshold. + relative: [.p90: 2], + /// 500 allocations of tolerance. + absolute: [.p90: 500] + ) + ] ) ) { benchmark in defer { benchmarkIterated += 1 } @@ -111,7 +120,16 @@ let benchmarks: @Sendable () -> Void = { "CollatingParserAllocations_Empty", configuration: .init( metrics: [.mallocCountTotal], - maxIterations: 1 + maxIterations: 1, + thresholds: [ + .mallocCountTotal: .init( + /// `2 - 1 == 1`% tolerance. + /// Will rely on the absolute threshold as the tighter threshold. + relative: [.p90: 2], + /// 500 allocations of tolerance. + absolute: [.p90: 500] + ) + ] ) ) { benchmark in defer { benchmarkIterated += 1 } From e2f37e536c0cee75052a4b88543a94da5dc24118 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 02:56:37 +0330 Subject: [PATCH 68/80] thresholds --- Benchmarks/Parser/Parser.swift | 8 ++++---- .../Parser.CollatingParserCPUTime_256MiB.p90.json | 2 +- .../Parser.StreamingParserCPUTime_256MiB.p90.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index dba279f..82175cd 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -66,8 +66,8 @@ let benchmarks: @Sendable () -> Void = { /// `2 - 1 == 1`% tolerance. /// Will rely on the absolute threshold as the tighter threshold. relative: [.p90: 2], - /// 500 allocations of tolerance. - absolute: [.p90: 500] + /// 1000 allocations of tolerance. + absolute: [.p90: 1000] ) ] ) @@ -126,8 +126,8 @@ let benchmarks: @Sendable () -> Void = { /// `2 - 1 == 1`% tolerance. /// Will rely on the absolute threshold as the tighter threshold. relative: [.p90: 2], - /// 500 allocations of tolerance. - absolute: [.p90: 500] + /// 1000 allocations of tolerance. + absolute: [.p90: 1000] ) ] ) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json index 33e46c4..087acdb 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 600000000 + "cpuUser": 570000000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json index a8d6ac8..34351db 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json @@ -1,4 +1,4 @@ { - "cpuUser": 600000000 + "cpuUser": 550000000 } From 3599f3179cfbe2293e660ebcc860f67ab62e8947 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 03:01:51 +0330 Subject: [PATCH 69/80] thresholds --- Benchmarks/Parser/Parser.swift | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 82175cd..85a42ba 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -66,8 +66,8 @@ let benchmarks: @Sendable () -> Void = { /// `2 - 1 == 1`% tolerance. /// Will rely on the absolute threshold as the tighter threshold. relative: [.p90: 2], - /// 1000 allocations of tolerance. - absolute: [.p90: 1000] + /// 500 allocations of tolerance. + absolute: [.p90: 500] ) ] ) @@ -120,16 +120,7 @@ let benchmarks: @Sendable () -> Void = { "CollatingParserAllocations_Empty", configuration: .init( metrics: [.mallocCountTotal], - maxIterations: 1, - thresholds: [ - .mallocCountTotal: .init( - /// `2 - 1 == 1`% tolerance. - /// Will rely on the absolute threshold as the tighter threshold. - relative: [.p90: 2], - /// 1000 allocations of tolerance. - absolute: [.p90: 1000] - ) - ] + maxIterations: 1 ) ) { benchmark in defer { benchmarkIterated += 1 } @@ -149,7 +140,16 @@ let benchmarks: @Sendable () -> Void = { "CollatingParserAllocations_\(approxSizeInMiB)MiB", configuration: .init( metrics: [.mallocCountTotal], - maxIterations: 1 + maxIterations: 1, + thresholds: [ + .mallocCountTotal: .init( + /// `2 - 1 == 1`% tolerance. + /// Will rely on the absolute threshold as the tighter threshold. + relative: [.p90: 2], + /// 500 allocations of tolerance. + absolute: [.p90: 500] + ) + ] ) ) { benchmark in defer { benchmarkIterated += 1 } From 3378157d148f5af7234e9d638c2b03b1e1681bf3 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 03:03:00 +0330 Subject: [PATCH 70/80] increase iterations --- Benchmarks/Parser/Parser.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 85a42ba..6aa4cbe 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -15,7 +15,7 @@ let benchmarks: @Sendable () -> Void = { ) let cpuBenchsWarmupIterations = 1 - let cpuBenchsMaxIterations = 10 + let cpuBenchsMaxIterations = 20 let maxBufferStreamsUsedInBenchs = cpuBenchsWarmupIterations + cpuBenchsMaxIterations var bufferStreams: [AsyncSyncSequence<[ArraySlice]>] = [] From d634658cb751bf83faed941304bedd0c9b566c4c Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 15:00:53 +0330 Subject: [PATCH 71/80] thresholds --- Benchmarks/Parser/NoOpAsyncSequence.swift | 15 +++++++++++++ Benchmarks/Parser/Parser.swift | 21 +++---------------- ....CollatingParserAllocations_Empty.p90.json | 2 +- ...ser.CollatingParserCPUTime_256MiB.p90.json | 2 +- ....StreamingParserAllocations_Empty.p90.json | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 Benchmarks/Parser/NoOpAsyncSequence.swift diff --git a/Benchmarks/Parser/NoOpAsyncSequence.swift b/Benchmarks/Parser/NoOpAsyncSequence.swift new file mode 100644 index 0000000..a55d2ec --- /dev/null +++ b/Benchmarks/Parser/NoOpAsyncSequence.swift @@ -0,0 +1,15 @@ +struct NoOpAsyncSequence: AsyncSequence { + typealias Element = ArraySlice + + struct Iterator: AsyncIteratorProtocol { + mutating func next() async -> ArraySlice? { + nil + } + } + + init() {} + + func makeAsyncIterator() -> Iterator { + return Iterator() + } +} diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 6aa4cbe..eef00e4 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -14,12 +14,11 @@ let benchmarks: @Sendable () -> Void = { chunkSize: chunkSizeInKiB << 10 ) - let cpuBenchsWarmupIterations = 1 + let cpuBenchsWarmupIterations = 5 let cpuBenchsMaxIterations = 20 let maxBufferStreamsUsedInBenchs = cpuBenchsWarmupIterations + cpuBenchsMaxIterations var bufferStreams: [AsyncSyncSequence<[ArraySlice]>] = [] - var bufferStreamsWithNoContents: [AsyncStream>] = [] var benchmarkIterated = 0 func refreshBufferStreams() { @@ -28,14 +27,6 @@ let benchmarks: @Sendable () -> Void = { } } - func refreshBufferStreamsWithNoContents() { - bufferStreamsWithNoContents = (0.. Void = { maxIterations: 1 ) ) { benchmark in - defer { benchmarkIterated += 1 } - let sequence = StreamingMultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreamsWithNoContents[benchmarkIterated] + buffer: NoOpAsyncSequence() ) for try await part in sequence { blackHole(part) @@ -114,8 +103,6 @@ let benchmarks: @Sendable () -> Void = { } } - benchmarkIterated = 0 - refreshBufferStreamsWithNoContents() Benchmark( "CollatingParserAllocations_Empty", configuration: .init( @@ -123,11 +110,9 @@ let benchmarks: @Sendable () -> Void = { maxIterations: 1 ) ) { benchmark in - defer { benchmarkIterated += 1 } - let sequence = MultipartParserAsyncSequence( boundary: boundary, - buffer: bufferStreamsWithNoContents[benchmarkIterated] + buffer: NoOpAsyncSequence() ) for try await part in sequence { blackHole(part) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json index 36d82e7..d3eb4e9 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 9 + "mallocCountTotal": 7 } diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json index 087acdb..f66ef7b 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 570000000 + "cpuUser": 610000000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json index 17d7b17..8ba9cd4 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 8 + "mallocCountTotal": 5 } From 7db849ae635f7a787c289c30c190c00a97bc89d8 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 15:24:42 +0330 Subject: [PATCH 72/80] thresholds --- Benchmarks/Parser/AsyncSyncSequence.swift | 2 +- Benchmarks/Parser/NoOpAsyncSequence.swift | 4 +--- Benchmarks/Parser/Parser.swift | 5 +++-- .../Parser.CollatingParserAllocations_Empty.p90.json | 2 +- .../Parser.StreamingParserAllocations_Empty.p90.json | 2 +- .../Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json | 2 +- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Benchmarks/Parser/AsyncSyncSequence.swift b/Benchmarks/Parser/AsyncSyncSequence.swift index f593ad3..bfa9d48 100644 --- a/Benchmarks/Parser/AsyncSyncSequence.swift +++ b/Benchmarks/Parser/AsyncSyncSequence.swift @@ -24,7 +24,7 @@ final class AsyncSyncSequence: AsyncSequence { } } - var base: Base? + private var base: Base? init(_ base: Base) { self.base = base diff --git a/Benchmarks/Parser/NoOpAsyncSequence.swift b/Benchmarks/Parser/NoOpAsyncSequence.swift index a55d2ec..e67b9c8 100644 --- a/Benchmarks/Parser/NoOpAsyncSequence.swift +++ b/Benchmarks/Parser/NoOpAsyncSequence.swift @@ -2,13 +2,11 @@ struct NoOpAsyncSequence: AsyncSequence { typealias Element = ArraySlice struct Iterator: AsyncIteratorProtocol { - mutating func next() async -> ArraySlice? { + mutating func next() async -> Element? { nil } } - init() {} - func makeAsyncIterator() -> Iterator { return Iterator() } diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index eef00e4..adbf942 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -14,6 +14,7 @@ let benchmarks: @Sendable () -> Void = { chunkSize: chunkSizeInKiB << 10 ) + let cpuBenchsMaxDuration: Duration = .seconds(20) let cpuBenchsWarmupIterations = 5 let cpuBenchsMaxIterations = 20 let maxBufferStreamsUsedInBenchs = cpuBenchsWarmupIterations + cpuBenchsMaxIterations @@ -79,7 +80,7 @@ let benchmarks: @Sendable () -> Void = { configuration: .init( metrics: [.cpuUser], warmupIterations: cpuBenchsWarmupIterations, - maxDuration: .seconds(20), + maxDuration: cpuBenchsMaxDuration, maxIterations: cpuBenchsMaxIterations, thresholds: [ .cpuUser: .init( @@ -155,7 +156,7 @@ let benchmarks: @Sendable () -> Void = { configuration: .init( metrics: [.cpuUser], warmupIterations: cpuBenchsWarmupIterations, - maxDuration: .seconds(20), + maxDuration: cpuBenchsMaxDuration, maxIterations: cpuBenchsMaxIterations, thresholds: [ .cpuUser: .init( diff --git a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json index d3eb4e9..17d7b17 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserAllocations_Empty.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 7 + "mallocCountTotal": 8 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json index 8ba9cd4..14184fb 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserAllocations_Empty.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal": 5 + "mallocCountTotal": 6 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json index 34351db..13a0791 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json @@ -1,4 +1,4 @@ { - "cpuUser": 550000000 + "cpuUser": 590000000 } From 464d97e4573b01959616d1c7534d1d28d7704218 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 15:26:23 +0330 Subject: [PATCH 73/80] minor --- Benchmarks/Parser/Parser.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index adbf942..26d8660 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -14,9 +14,9 @@ let benchmarks: @Sendable () -> Void = { chunkSize: chunkSizeInKiB << 10 ) - let cpuBenchsMaxDuration: Duration = .seconds(20) let cpuBenchsWarmupIterations = 5 let cpuBenchsMaxIterations = 20 + let cpuBenchsMaxDuration: Duration = .seconds(cpuBenchsMaxIterations + cpuBenchsWarmupIterations) let maxBufferStreamsUsedInBenchs = cpuBenchsWarmupIterations + cpuBenchsMaxIterations var bufferStreams: [AsyncSyncSequence<[ArraySlice]>] = [] From 9b7ef8d2d566d6454f116c09024655e84fd3fbc2 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 15:37:20 +0330 Subject: [PATCH 74/80] better message --- Benchmarks/Parser/Parser.swift | 2 +- Benchmarks/Serializer/Serializer.swift | 2 +- ...p90.json => Serializer.SerializerAllocations_Empty.p90.json} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename Benchmarks/Thresholds/{Serializer.SerializerAllocations_0Parts.p90.json => Serializer.SerializerAllocations_Empty.p90.json} (100%) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index 26d8660..e073230 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -222,7 +222,7 @@ private func makeMessage(boundary: String, size: Int) -> ArraySlice { \r\n """.utf8) - message.append(contentsOf: Array(repeating: UInt8.random(in: .min ... .max), count: size)) + message.append(contentsOf: (0.. Void = { let repeatedParts: [MultipartPart] = .init(repeating: examplePart, count: partCount) Benchmark( - "SerializerAllocations_0Parts", + "SerializerAllocations_Empty", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1 diff --git a/Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json b/Benchmarks/Thresholds/Serializer.SerializerAllocations_Empty.p90.json similarity index 100% rename from Benchmarks/Thresholds/Serializer.SerializerAllocations_0Parts.p90.json rename to Benchmarks/Thresholds/Serializer.SerializerAllocations_Empty.p90.json From 1651b6314607f1d989250d737051e919de9d32af Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 16:31:38 +0330 Subject: [PATCH 75/80] thresholds --- .../Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json | 2 +- .../Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json index f66ef7b..087acdb 100644 --- a/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.CollatingParserCPUTime_256MiB.p90.json @@ -1,3 +1,3 @@ { - "cpuUser": 610000000 + "cpuUser": 570000000 } diff --git a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json index 13a0791..34351db 100644 --- a/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json +++ b/Benchmarks/Thresholds/Parser.StreamingParserCPUTime_256MiB.p90.json @@ -1,4 +1,4 @@ { - "cpuUser": 590000000 + "cpuUser": 550000000 } From 6a706b9b999950fd1ea328c8747482534587e351 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 16:34:09 +0330 Subject: [PATCH 76/80] minor --- Benchmarks/Parser/Parser.swift | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index e073230..a885edc 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -4,12 +4,12 @@ import MultipartKit let benchmarks: @Sendable () -> Void = { let boundary = "boundary123" - let approxSizeInMiB = 256 + let fileSizeInMiB = 256 let chunkSizeInKiB = 16 let chunkedMessage = makeChunks( for: makeMessage( boundary: boundary, - size: approxSizeInMiB << 20 + fileSize: fileSizeInMiB << 20 ), chunkSize: chunkSizeInKiB << 10 ) @@ -47,7 +47,7 @@ let benchmarks: @Sendable () -> Void = { benchmarkIterated = 0 refreshBufferStreams() Benchmark( - "StreamingParserAllocations_\(approxSizeInMiB)MiB", + "StreamingParserAllocations_\(fileSizeInMiB)MiB", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1, @@ -76,7 +76,7 @@ let benchmarks: @Sendable () -> Void = { benchmarkIterated = 0 refreshBufferStreams() Benchmark( - "StreamingParserCPUTime_\(approxSizeInMiB)MiB", + "StreamingParserCPUTime_\(fileSizeInMiB)MiB", configuration: .init( metrics: [.cpuUser], warmupIterations: cpuBenchsWarmupIterations, @@ -123,7 +123,7 @@ let benchmarks: @Sendable () -> Void = { benchmarkIterated = 0 refreshBufferStreams() Benchmark( - "CollatingParserAllocations_\(approxSizeInMiB)MiB", + "CollatingParserAllocations_\(fileSizeInMiB)MiB", configuration: .init( metrics: [.mallocCountTotal], maxIterations: 1, @@ -152,7 +152,7 @@ let benchmarks: @Sendable () -> Void = { benchmarkIterated = 0 refreshBufferStreams() Benchmark( - "CollatingParserCPUTime_\(approxSizeInMiB)MiB", + "CollatingParserCPUTime_\(fileSizeInMiB)MiB", configuration: .init( metrics: [.cpuUser], warmupIterations: cpuBenchsWarmupIterations, @@ -200,7 +200,7 @@ private func makeChunks( return chunks } -private func makeMessage(boundary: String, size: Int) -> ArraySlice { +private func makeMessage(boundary: String, fileSize: Int) -> ArraySlice { var message = ArraySlice( """ --\(boundary)\r From 82cba28dff740367f53233a004b856db49cf41bd Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 16:36:34 +0330 Subject: [PATCH 77/80] fix --- Benchmarks/Parser/Parser.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Benchmarks/Parser/Parser.swift b/Benchmarks/Parser/Parser.swift index a885edc..8f227f7 100644 --- a/Benchmarks/Parser/Parser.swift +++ b/Benchmarks/Parser/Parser.swift @@ -222,7 +222,7 @@ private func makeMessage(boundary: String, fileSize: Int) -> ArraySlice { \r\n """.utf8) - message.append(contentsOf: (0.. Date: Tue, 14 Jan 2025 19:16:06 +0330 Subject: [PATCH 78/80] better detect PR events --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5c32b13..93bf0f9 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -179,7 +179,7 @@ jobs: # There is a '' comment at the beginning of the benchamrk report comment. # The number in that comment is the exit code of the last benchmark run. - name: Find existing comment ID - if: github.event_name == 'pull_request' + if: startsWith(github.event_name, 'pull_request') id: existing-comment run: | set -eu @@ -233,7 +233,7 @@ jobs: } >> "${GITHUB_OUTPUT}" - name: Comment in PR - if: github.event_name == 'pull_request' + if: startsWith(github.event_name, 'pull_request') run: | set -eu From bcea652213f72caa9885521074d63318019309e8 Mon Sep 17 00:00:00 2001 From: MahdiBM Date: Tue, 14 Jan 2025 19:22:31 +0330 Subject: [PATCH 79/80] Try thollander/actions-comment-pull-request --- .github/workflows/benchmark.yml | 82 ++------------------------------- 1 file changed, 4 insertions(+), 78 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 93bf0f9..2e3a208 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -176,86 +176,12 @@ jobs: - name: Output the comment as job summary run: echo '${{ env.PR_COMMENT }}' >> "${GITHUB_STEP_SUMMARY}" - # There is a '' comment at the beginning of the benchamrk report comment. - # The number in that comment is the exit code of the last benchmark run. - - name: Find existing comment ID - if: startsWith(github.event_name, 'pull_request') - id: existing-comment - run: | - set -eu - - # Known limitation: This only fetches the first 100 comments. This should not - # matter much because a benchmark comment should have been sent early in the PR. - curl -sL \ - -X GET \ - -H 'Accept: application/vnd.github+json' \ - -H 'Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - -o result.json \ - 'https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments?per_page=100' - - # Get the last comment that has a body that starts with '" >> "${GITHUB_ENV}" - - echo '' >> "${GITHUB_ENV}" - echo '## [Benchmark](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) Report' >> "${GITHUB_ENV}" case "${EXIT_CODE}" in