feat: working

.gitignore

@@ -17,6 +17,8 @@ scripts/master_papers.csv
 scripts/t.py
 /RP_eval_results_*.json
 scripts/master_papers.csv
+data/mmlu
+data/benchmarking/*
 scratch
 scratch/*
 scratch/acl-anthology

data/mmlu_configs.json

@@ -0,0 +1,60 @@
+{
+    "configs": [
+        "high_school_european_history",
+        "business_ethics",
+        "clinical_knowledge",
+        "medical_genetics",
+        "high_school_us_history",
+        "high_school_physics",
+        "high_school_world_history",
+        "virology",
+        "high_school_microeconomics",
+        "econometrics",
+        "college_computer_science",
+        "high_school_biology",
+        "abstract_algebra",
+        "professional_accounting",
+        "philosophy",
+        "professional_medicine",
+        "nutrition",
+        "global_facts",
+        "machine_learning",
+        "security_studies",
+        "public_relations",
+        "professional_psychology",
+        "prehistory",
+        "anatomy",
+        "college_medicine",
+        "high_school_government_and_politics",
+        "college_chemistry",
+        "logical_fallacies",
+        "high_school_geography",
+        "elementary_mathematics",
+        "human_aging",
+        "college_mathematics",
+        "high_school_psychology",
+        "formal_logic",
+        "high_school_statistics",
+        "international_law",
+        "high_school_mathematics",
+        "high_school_computer_science",
+        "conceptual_physics",
+        "miscellaneous",
+        "high_school_chemistry",
+        "marketing",
+        "professional_law",
+        "management",
+        "college_physics",
+        "jurisprudence",
+        "world_religions",
+        "sociology",
+        "us_foreign_policy",
+        "high_school_macroeconomics",
+        "computer_security",
+        "moral_scenarios",
+        "moral_disputes",
+        "electrical_engineering",
+        "astronomy",
+        "college_biology"
+    ]
+}

requirements.txt

@@ -36,4 +36,5 @@ urllib3<2
 acl-anthology-py
 seaborn
 requests
+tenacity
 -e .