fix gpt4 assets for MultiNativQA (#366)

qcri · Sep 25, 2024 · 19fb1d2 · 19fb1d2
1 parent cb8c25e
commit 19fb1d2
Show file tree

Hide file tree

Showing 10 changed files with 38 additions and 2 deletions.
diff --git a/assets/ar/QA/MultiNativQA_GPT4_ZeroShot.py b/assets/ar/QA/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "arabic_qa"},
     }
 
 

diff --git a/assets/asm/QA/MultiNativQA_GPT4_ZeroShot.py b/assets/asm/QA/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "assamese_in"},
     }
 
 

diff --git a/assets/bn/QA/MultiNativQA_GPT4_ZeroShot.py b/assets/bn/QA/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "bangla_bd"},
     }
 
 

diff --git a/assets/bn_in/QA/MultiNativQA_GPT4_ZeroShot.py b/assets/bn_in/QA/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "bangla_in"},
     }
 
 

diff --git a/assets/en/QA/bd/MultiNativQA_GPT4_ZeroShot.py b/assets/en/QA/bd/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "english_bd"},
     }
 
 

diff --git a/assets/en/QA/qa/MultiNativQA_GPT4_ZeroShot.py b/assets/en/QA/qa/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "english_qa"},
     }
 
 

diff --git a/assets/hi/QA/MultiNativQA_GPT4_ZeroShot.py b/assets/hi/QA/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "hindi_in"},
     }
 
 

diff --git a/assets/ne/QA/MultiNativQA_GPT4_ZeroShot.py b/assets/ne/QA/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "nepali_np"},
     }
 
 

diff --git a/assets/tr/QA/MultiNativQA_GPT4_ZeroShot.py b/assets/tr/QA/MultiNativQA_GPT4_ZeroShot.py
@@ -1,3 +1,6 @@
+import json
+import re
+
 from llmebench.datasets import MultiNativQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MultiNativQATask
@@ -17,6 +20,7 @@ def config():
         "dataset": MultiNativQADataset,
         "task": MultiNativQATask,
         "model": OpenAIModel,
+        "general_args": {"test_split": "turkish_tr"},
     }
 
 

diff --git a/llmebench/datasets/MultiNativQA.py b/llmebench/datasets/MultiNativQA.py
@@ -62,8 +62,8 @@ def metadata():
                     "test": "nepali_np/NativQA_ne_NA_np_test.tsv",
                 },
                 "turkish_tr": {
-                    "dev": "english_qa/NativQA_tr_NA_tr_dev.tsv",
-                    "test": "english_qa/NativQA_tr_NA_tr_test.tsv",
+                    "dev": "turkish_tr/NativQA_tr_NA_tr_dev.tsv",
+                    "test": "turkish_tr/NativQA_tr_NA_tr_test.tsv",
                 },
                 "default": [
                     "assamese_in",