Display difference in tx-cost outputs as a comment on PRs (#1703)

This adds a simple diff calculation to the PR comment, just like the
present "Transaction costs" comment we get, but instead a diff between
the PR and the master branch.

Positive numbers are bad - it means the cost/size/etc got larger with
the introduction of the PR. These are coloured black.

Negative numbers are good - it means a reduction in size/cost/etc, so
these are coloured green.

Values that are the same between PR and branch are simply listed as `-`;
i.e. unchanged.

> [!note]
> It seems that some differences are expected _even when nothing
relevant has changed_, as in this PR itself. So some care will need to
be taken in learning how to interpret these numbers, as we see them
develop over furture PRs.

Author: locallycompact

.github/workflows/ci-nix.yaml

@@ -288,6 +288,7 @@ jobs:
         path: |
           ./*.pdf
 
+
   documentation:
     name: Documentation
     needs: [haddock,benchmarks,build-test,build-specification]
@@ -359,3 +360,81 @@ jobs:
       run: |
         yarn
         yarn build-dev
+
+
+  # Compute the cost difference between this branch and master.
+  tx-cost-diff:
+    name: Compute cost differences
+    runs-on: ubuntu-latest
+    # Only run on PR
+    if: github.event_name == 'pull_request'
+    steps:
+    - name: "Checkout the PR as the 'new' source"
+      uses: actions/checkout@v4
+      with:
+        path: ./new
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: "Checkout `master` as the 'old' source"
+      uses: actions/checkout@v4
+      with:
+        path: ./old
+        ref: master
+
+    - name: ❄ Prepare nix
+      uses: cachix/install-nix-action@v30
+      with:
+        extra_nix_config: |
+          accept-flake-config = true
+          log-lines = 1000
+
+    - name: ❄ Cachix cache of nix derivations
+      uses: cachix/cachix-action@v15
+      with:
+        name: cardano-scaling
+        authToken: '${{ secrets.CACHIX_CARDANO_SCALING_AUTH_TOKEN }}'
+
+    - name: Set up and use the "ci" devShell
+      uses: nicknovitski/nix-develop@v1
+      with:
+        arguments: "./new#costDifferences"
+
+    - name: "Compute costs on both old and new"
+      run: |
+        nix run ./new/#tx-cost >new.md
+        nix run ./old/#tx-cost >old.md
+
+    - name: "Compute the difference markdown"
+      run: |
+        # Convert to HTML, as that's the easiest way to read the markdown
+        # tables via pandas (!!!)
+        pandoc -i new.md -o new.html
+        pandoc -i old.md -o old.html
+
+        cat new.md | grep '##' >new-headers.txt
+        cat old.md | grep '##' >old-headers.txt
+
+        # Stop if the heading columns aren't the same.
+        cmp old-headers.txt new-headers.txt
+
+        # Run the diff script; note that it's located in the "new" folder.
+        ./new/.github/workflows/cost-differences/diff.py \
+          old-headers.txt \
+          old.html \
+          new.html >diff.md
+
+    - name: 🔎 Find Comment
+      uses: peter-evans/find-comment@v3
+      id: find-comment
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: Transaction cost differences
+
+    - name: ✏ Create or update comment
+      uses: peter-evans/create-or-update-comment@v4
+      with:
+        comment-id: ${{ steps.find-comment.outputs.comment-id }}
+        edit-mode: replace
+        issue-number: ${{ github.event.pull_request.number }}
+        body-file: diff.md

.github/workflows/cost-differences/diff.py

@@ -0,0 +1,80 @@
+#! /usr/bin/env python
+
+import pandas as pd
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("header_file", type=str)
+parser.add_argument("old_file", type=str)
+parser.add_argument("new_file", type=str)
+args = parser.parse_args()
+
+with open(args.header_file, "r") as f:
+    headers = [ l[3:].strip() for l in f.readlines() ]
+
+base   = pd.read_html(args.old_file, flavor="html5lib")
+branch = pd.read_html(args.new_file, flavor="html5lib")
+
+def script_size(df):
+  return df.drop(columns=["Hash"]).set_index("Name")
+
+def parties(df):
+  return df.set_index("Parties")
+
+def utxo(df):
+  return df.set_index("UTxO")
+
+def compare_to_md(f, old, new):
+  # New should be better, so we compare to that.
+  df = f(new) - f(old)
+
+  # Don't keep what we couldn't compare.
+  df = df.dropna()
+
+  # Round everything to 2 decimals
+  df = df.round(2)
+
+  # Add colour
+  def update_colour(x):
+    if x == 0:
+      return "-"
+
+    if type(x) is object:
+      return x
+
+    if x > 0:
+      return f"+{x}"
+    else:
+      return f"$${{\\color{{green}}{x:.2f}}}$$"
+
+  df = df.map(update_colour)
+
+  return df.to_markdown()
+
+print("Transaction cost differences")
+
+# First is the script size
+
+print(f"## {headers[0]}")
+print("")
+print( compare_to_md( script_size, base[1], branch[1]) )
+
+# Then Init,
+print("")
+print(f"## {headers[1]}")
+print("")
+print( compare_to_md(parties, base[2], branch[2]) )
+
+# Then Commit is different; it doesn't have a "Parties" column
+
+print("")
+print(f"## {headers[2]}")
+print("")
+print( compare_to_md(utxo, base[3], branch[3]) )
+
+# The remaining are all the same as Init.
+for i in range(4, 9 + 1):
+  print("")
+  print(f"## {headers[i - 1]}")
+  print("")
+  print( compare_to_md(parties, base[i], branch[i]) )

nix/hydra/shell.nix

@@ -150,6 +150,15 @@ let
     ];
   };
 
+  # Shell for computing tx-cost-differences
+  costDifferencesShell = pkgs.mkShell {
+    name = "tx-cost-differences-shell";
+    buildInputs = [
+      pkgs.pandoc
+      (pkgs.python3.withPackages (ps: with ps; [ pandas html5lib beautifulsoup4 tabulate ]))
+    ];
+  };
+
   # If you want to modify `Python` code add `libtmux` and pyyaml to the
   # `buildInputs` then enter it and then run `Python` module directly so you
   # have fast devel cycle.
@@ -165,4 +174,5 @@ in
   exes = exeShell;
   demo = demoShell;
   ci = ciShell;
+  costDifferences = costDifferencesShell;
 }