diff --git a/.gitignore b/.gitignore index 832135b..3ffd78e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,11 @@ # Ignore everything * -# Except README.md, leaderboard.json, .gitignore and .github +# Except README.md, leaderboard.json, scripts, .gitignore and .github !README.md !leaderboard.json +!scripts/ +!scripts/* !.gitignore !.github/ !.github/workflows/ diff --git a/scripts/add_to_leaderboard.py b/scripts/add_to_leaderboard.py new file mode 100644 index 0000000..eaa2cb5 --- /dev/null +++ b/scripts/add_to_leaderboard.py @@ -0,0 +1,18 @@ +import argparse +import subprocess + + +def main(): + parser = argparse.ArgumentParser(description="Add entries to the leaderboard.") + parser.add_argument("experiment_prefixes", nargs="+", type=str, help="experiment prefixes") + args = parser.parse_args() + for experiment_prefix in args.experiment_prefixes: + print(f"\nWorking on experiment prefix: {experiment_prefix}") + experiment_names = [experiment_prefix + "_test_normal", experiment_prefix + "_test_challenge"] + for experiment_name in experiment_names: + subprocess.run(f"appworld evaluate {experiment_name}_test_normal test_normal", check=True, shell=True) + subprocess.run(f"appworld evaluate {experiment_name}_test_challenge test_challenge", check=True, shell=True) + subprocess.run(f"appworld make {experiment_name}_test_normal {experiment_name}_challenge --save", check=True, shell=True) + +if __name__ == "__main__": + main()