Script to Compare Packages in Benchpark Against Spack Tag

.github/workflows/ci.yml

@@ -23,10 +23,12 @@ jobs:
       run: ${{ steps.filter.outputs.run }}
       coverage: ${{ steps.filter.outputs.coverage }}
       license: ${{ steps.filter.outputs.license }}
+      diffpackages: ${{ steps.filter.outputs.diffpackages }}
+      modified_repo_dirs: ${{ steps.get_modified_repo_dirs.outputs.modified_repo_dirs }}
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # @v2
-        if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }}
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'pull_request' }}
         with:
           fetch-depth: 0
 
@@ -96,6 +98,23 @@ jobs:
               - 'repo/**'
               - 'systems/**'
               - 'var/**'
+            diffpackages:
+              - 'repo/**/package.py'
+
+      - name: Get modified directories in repo/
+        id: get_modified_repo_dirs
+        run: |
+          # Fetch develop branch
+          git fetch origin develop:develop
+
+          # Get the modified directories under 'repo/'
+          MODIFIED_REPO_DIRS=$(git diff --name-only develop | grep 'repo/' | xargs -n1 dirname | sed 's|^repo/||' | tr '\n' ' ')
+
+          # Debugging: Print the modified directories
+          echo "Modified directories: $MODIFIED_REPO_DIRS"
+
+          # Output the modified directories as a GitHub Actions output
+          echo "modified_repo_dirs=$MODIFIED_REPO_DIRS" >> $GITHUB_OUTPUT
 
   docs:
     if: ${{ needs.changes.outputs.docs == 'true' }}
@@ -127,6 +146,13 @@ jobs:
     needs: changes
     uses: ./.github/workflows/license.yml
 
+  diffpackages:
+    if: ${{ needs.changes.outputs.diffpackages == 'true' }}
+    needs: changes
+    uses: ./.github/workflows/diffpackages.yml
+    with:
+      modified_repo_dirs: ${{ needs.changes.outputs.modified_repo_dirs }}
+
   all:
     needs:
       - changes
@@ -135,6 +161,7 @@ jobs:
       - run
       - coverage
       - license
+      - diffpackages
     if: always()
     runs-on: ubuntu-latest
     steps:

.github/workflows/diffpackages.yml

@@ -0,0 +1,21 @@
+on:
+  workflow_call:
+    inputs:
+      modified_repo_dirs:
+        required: true
+        type: string
+
+jobs:
+  diffPackages:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+
+      - name: Add needed Python libs
+        run: |
+          pip install -r ./requirements.txt
+
+      - name: Check Packages Against Spack
+        run: |
+          ./bin/benchpark-python lib/scripts/diffPackages.py --packages "${{ inputs.modified_repo_dirs }}"

bin/benchpark-python

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Copyright 2023 Lawrence Livermore National Security, LLC and other
 # Benchpark Project Developers. See the top-level COPYRIGHT file for details.
@@ -18,4 +18,11 @@
 #
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 export PYTHONPATH="${SCRIPT_DIR}/../lib":$PYTHONPATH
-exec python3 -i "$@"
+# Check if the first argument is -i
+if [ "$1" = "-i" ]; then
+    # Shift the arguments to remove the first one (-i)
+    shift
+    exec python3 -i "$@"
+else
+    exec python3 "$@"
+fi

lib/benchpark/cmd/unit_test.py

@@ -221,7 +221,7 @@ def command(args, unknown_args):
                 "--dist",
                 "loadfile",
                 "--tx",
-                f"{args.numprocesses}*popen//python=benchpark-tmpconfig benchpark-python",
+                f"{args.numprocesses}*popen//python=benchpark-tmpconfig benchpark-python -i",
             ]
         )
 

lib/scripts/diffExperiments.py

@@ -201,7 +201,7 @@ def main():
         # Path to the Spack setup script
         spack_setup_script = f"{old_name}/wkp/setup.sh"
         # Define the ramble command
-        cmd = f"{old_name}/wkp/spack/bin/spack-python altdiff.py -t -d {old_file} {new_file}"
+        cmd = f"{old_name}/wkp/spack/bin/spack-python diffSpecs.py -t -d {old_file} {new_file}"
         # Combine sourcing the script and running the command
         try:
             diff = subprocess.run(

lib/scripts/diffPackages.py

@@ -0,0 +1,119 @@
+import argparse
+import difflib
+import os
+import subprocess
+import sys
+import itertools
+
+import benchpark.paths
+
+EXIT_CODE = 0
+
+parser = argparse.ArgumentParser(
+    description="Script to compare packages in benchpark against upstream spack packages.",
+    usage="benchpark-python diffPackages.py [OPTIONS]",
+)
+parser.add_argument(
+    "--spack-tag",
+    default="develop",
+    help="Specify the spack version in the format 'vX.Y.Z', e.g., 'v0.23.1'.",
+)
+parser.add_argument("--print-diff", action="store_true", help="Print file diff")
+parser.add_argument(
+    "--packages",
+    nargs="+",  # Allows one or more package names
+    help="Specify one or more packages to compare. If not provided, all packages will be compared.",
+)
+args = parser.parse_args()
+
+print(f"Comparing benchpark packages to packages in spack {args.spack_tag}")
+
+if "spack" not in os.listdir(os.getcwd()):
+    subprocess.run(["git", "clone", "https://github.com/spack/spack.git"])
+subprocess.run(["git", "checkout", args.spack_tag], cwd="spack")
+
+sys.path.append("spack/lib/spack")
+import llnl.util.tty.color as color  # noqa: E402
+
+
+def main():
+    global EXIT_CODE
+    spack_dir = "spack/var/spack/repos/builtin/packages/"
+    benchpark_dir = str(benchpark.paths.benchpark_root) + "/repo/"
+
+    # Get the list of packages to process
+    if args.packages:
+        # Use only the specified packages
+        packages_to_compare = []
+        for item in args.packages:
+            packages_to_compare.extend(item.strip().split())
+    else:
+        # Process all packages if --packages is not provided
+        packages_to_compare = sorted(os.listdir(benchpark_dir))
+
+    for package in packages_to_compare:
+        if package not in ["repo.yaml"]:
+            spack_package_path = spack_dir + package + "/package.py"
+            benchpark_package_path = benchpark_dir + package + "/package.py"
+
+            if not os.path.exists(spack_package_path):
+                color.cprint("@*b" + package + "@.")
+                color.cprint(
+                    "    " + package + "/package.py @*rdoes not@. exist in @*ospack@."
+                )
+                continue
+            elif not os.path.exists(benchpark_package_path):
+                # color.cprint("    "+package+" package.py @*rdoes not@. exist in @*obenchpark@.")
+                continue
+
+            color.cprint("@*b" + package + "@.")
+
+            # Read the files
+            with open(spack_package_path, "r") as file1, open(
+                benchpark_package_path, "r"
+            ) as file2:
+                spack_lines = [
+                    line for line in file1 if not line.lstrip().startswith("#")
+                ]
+                benchpark_lines = [
+                    line for line in file2 if not line.lstrip().startswith("#")
+                ]
+
+            # Compare the files
+            diff = difflib.unified_diff(
+                spack_lines,
+                benchpark_lines,
+                fromfile="spack " + package,
+                tofile="benchpark " + package,
+                lineterm="",
+            )
+
+            diff, dc, dc2 = itertools.tee(diff, 3)
+
+            diff_list = list(diff)
+            # Check if there is no diff
+            if not diff_list:
+                color.cprint(
+                    f"    @*gNo differences found. Please delete 'benchpark/repo/{package}/package.py' (use spack upstream)@."
+                )
+                EXIT_CODE = 1
+
+            # Use difflib.ndiff to compare the lines
+            dc3 = difflib.ndiff(spack_lines, benchpark_lines)
+            # Count the differing lines (ignoring duplicates)
+            differing_lines_count = sum(
+                1 for line in dc3 if line.startswith("- ") or line.startswith("+ ")
+            )
+            print("    ", differing_lines_count // 2, "different lines")
+
+            if args.print_diff:
+                # Print the differences
+                print("\n".join(dc))
+
+    print(f"EXIT_CODE: {EXIT_CODE}")
+    return EXIT_CODE
+
+
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(EXIT_CODE)

lib/scripts/altdiff.py → lib/scripts/diffSpecs.py

@@ -152,13 +152,13 @@ def main():
 
     parser = argparse.ArgumentParser(
         description="""Diff two specs.
-(spack-python altdiff.py spec1 spec2)
+(spack-python diffSpecs.py spec1 spec2)
 
 Example usage:
 
     $ spack spec --yaml dray+mpi > dray-mpi.yaml
     $ spack spec --yaml dray~mpi > dray-nompi.yaml
-    $ spack-python lib/scripts/altdiff.py --truncate ./dray-mpi.yaml ./dray-nompi.yaml
+    $ spack-python lib/scripts/diffSpecs.py --truncate ./dray-mpi.yaml ./dray-nompi.yaml
 """,
         formatter_class=argparse.RawTextHelpFormatter,
     )