add script to add entries to leaderboard locally

.gitignore

@@ -2,9 +2,11 @@
 # Ignore everything
 *
 
-# Except README.md, leaderboard.json, .gitignore and .github
+# Except README.md, leaderboard.json, scripts, .gitignore and .github
 !README.md
 !leaderboard.json
+!scripts/
+!scripts/*
 !.gitignore
 !.github/
 !.github/workflows/

scripts/add_to_leaderboard.py

@@ -0,0 +1,18 @@
+import argparse
+import subprocess
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Add entries to the leaderboard.")
+    parser.add_argument("experiment_prefixes", nargs="+", type=str, help="experiment prefixes")
+    args = parser.parse_args()
+    for experiment_prefix in args.experiment_prefixes:
+        print(f"\nWorking on experiment prefix: {experiment_prefix}")
+        experiment_names = [experiment_prefix + "_test_normal", experiment_prefix + "_test_challenge"]
+        for experiment_name in experiment_names:
+            subprocess.run(f"appworld evaluate {experiment_name}_test_normal test_normal", check=True, shell=True)
+            subprocess.run(f"appworld evaluate {experiment_name}_test_challenge test_challenge", check=True, shell=True)
+            subprocess.run(f"appworld make {experiment_name}_test_normal {experiment_name}_challenge --save", check=True, shell=True)
+
+if __name__ == "__main__":
+    main()