feat: remove earthly usage for benchmarks (#654)

* feat: remove earthly usage for benchmarks

* chore: remove commited js files

* feat: use ephemeral postgres for benchmarks (cause regression error)

* feat(benchmarks): use compare recipe instead of run on prs

* feat: properly clean postgres servers

* fix: create output directory

* feat: increase benchmarks timeout

* feat: benchmarks and add inputs on workflow dispatch

* feat: use nix to setup just

* fix: nix usage

Author: gfyrag

.github/workflows/benchmark-comparison.yml

@@ -0,0 +1,55 @@
+name: Benchmark comparison
+on:
+  workflow_dispatch:
+    inputs:
+      bench:
+        description: 'Benchmarks to run'
+        required: false
+        default: '.'
+      parallelism:
+        description: 'Number of parallel benchmarks to run'
+        required: false
+        default: 5
+      duration:
+        description: 'Duration of each benchmark'
+        required: false
+        default: '10s'
+      count:
+        description: 'Number of times to run each benchmark '
+        required: false
+        default: 1
+  pull_request:
+    types: [ assigned, opened, synchronize, reopened, labeled ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  BenchmarkCompare:
+    runs-on: "github-001"
+    if: contains(github.event.pull_request.labels.*.name, 'benchmarks')
+    steps:
+      - uses: 'actions/checkout@v4'
+        with:
+          fetch-depth: 0
+      - name: Setup Env
+        uses: ./.github/actions/env
+        with:
+          token: ${{ secrets.NUMARY_GITHUB_TOKEN }}
+      - run: >
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
+          develop --impure --command just
+          --justfile ./test/performance/justfile
+          --working-directory ./test/performance 
+          compare ${{ inputs.bench }} ${{ inputs.parallelism }} ${{ inputs.duration }} ${{ inputs.count }}
+      - run: >
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
+          develop --impure --command just 
+          --justfile ./test/performance/justfile 
+          --working-directory ./test/performance 
+          graphs
+      - uses: actions/upload-artifact@v4
+        with:
+          name: graphs
+          path: test/performance/report

.github/workflows/benchmark.yml

@@ -1,43 +1,48 @@
 name: Benchmark
 on:
   workflow_dispatch:
-  pull_request:
-    types: [ assigned, opened, synchronize, reopened, labeled ]
+    inputs:
+      bench:
+        description: 'Benchmarks to run'
+        required: false
+        default: '.'
+      parallelism:
+        description: 'Number of parallel benchmarks to run'
+        required: false
+        default: 5
+      duration:
+        description: 'Duration of each benchmark'
+        required: false
+        default: '10s'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   Benchmark:
     runs-on: "github-001"
-    if: contains(github.event.pull_request.labels.*.name, 'benchmarks') || github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/main'
     steps:
       - uses: 'actions/checkout@v4'
         with:
           fetch-depth: 0
-      - run: go build -o /tmp/ledger ./
-      - run: echo "running actions as ${USER}"
-      - run: >
-          /tmp/ledger serve 
-          --postgres-uri=postgres://formance:formance@127.0.0.1/ledger 
-          --postgres-conn-max-idle-time=120s 
-          --postgres-max-open-conns=500 
-          --postgres-max-idle-conns=100 
-          --experimental-features 
-          --otel-metrics-keep-in-memory &
+      - name: Setup Env
+        uses: ./.github/actions/env
+        with:
+          token: ${{ secrets.NUMARY_GITHUB_TOKEN }}
       - run: >
-          earthly
-          --allow-privileged
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          ./test/performance+run --args="-benchtime 10s --ledger.url=http://localhost:3068 --parallelism=5" --locally=yes
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
+          develop --impure --command just
+          --justfile ./test/performance/justfile
+          --working-directory ./test/performance 
+          run ${{ inputs.bench }} ${{ inputs.parallelism }} ${{ inputs.duration }} 1
       - run: >
-          earthly
-          --allow-privileged
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          ./test/performance+generate-graphs
-      - run: kill -9 $(ps aux | grep "ledger serve"| grep -v "grep" | awk '{print $2}')
-        if: always()
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
+          develop --impure --command just 
+          --justfile ./test/performance/justfile 
+          --working-directory ./test/performance 
+          graphs
       - uses: actions/upload-artifact@v4
         with:
           name: graphs

flake.nix

@@ -101,6 +101,7 @@
               just
               nodejs_22
               self.packages.${system}.speakeasy
+              goperf
             ];
           };
         }

test/performance/Earthfile

@@ -10,25 +10,25 @@ Scripts can be found in directory [scripts](./scripts).
 ## Run locally
 
 ```shell
-earthly +run
+just run
 ```
 
 You can pass additional arguments (the underlying command is a standard `go test -bench=.`) using the flag `--args`.
 For example:
 ```shell
-earthly +run --args="-benchtime 10s"
+just run "-benchtime 10s"
 ```
 
 ## Run on a remote stack
 
 ```shell
-earthly +run --args="--stack.url=XXX --client.id=XXX --client.secret=XXX"
+just run "--stack.url=XXX --client.id=XXX --client.secret=XXX"
 ```
 
 ## Run on a remote ledger
 
 ```shell
-earthly +run --args="--ledger.url=XXX --auth.url=XXX --client.id=XXX --client.secret=XXX"
+just run "--ledger.url=XXX --auth.url=XXX --client.id=XXX --client.secret=XXX"
 ```
 
 ## Results
@@ -37,7 +37,7 @@ TPS is included as a benchmark metrics.
 
 You can generate some graphs using the command: 
 ```
-earthly +generate-graphs
+just graphs
 ```
 
 See generated files in `report` directory.

test/performance/README.md

@@ -0,0 +1 @@
+*.js

test/performance/charts/.gitignore

@@ -6,15 +6,15 @@ const main = async () => {
     let buffer = fs.readFileSync('../report/report.json', 'utf-8');
     let reports = JSON.parse(buffer);
     await exportTPSGraph({
-        output: 'tps.png',
+        output: '../report/tps.png',
     }, reports);
 
-    await exportDatabaseStats('database_connections.png', reports);
+    await exportDatabaseStats('../report/database_connections.png', reports);
 
     const ps: (keyof MetricsTime)[] = ['P99', 'P95', 'P75', 'Avg']
     for (let p of ps) {
         await exportLatencyGraph({
-            output: p.toLowerCase() + '.png'
+            output: '../report/' + p.toLowerCase() + '.png'
         }, p, reports);
     }
 }