Change languages available in dlt.lang (#35)

* Change languages available in dlt.lang

* Add Khmer as a language for m2m100

* scripts/generate_lang.py

* Update available languages in the docs

* Update tests for dlt.lang

* Apply black

* Bump version

* Fix tests (replace space by _)

Author: Xing Han Lu

dl_translate/_pairs.py

@@ -49,6 +49,7 @@
     ("Javanese", "jv"),
     ("Georgian", "ka"),
     ("Kazakh", "kk"),
+    ("Khmer", "km"),
     ("Central Khmer", "km"),
     ("Kannada", "kn"),
     ("Korean", "ko"),

dl_translate/lang/__init__.py

@@ -1,2 +1,2 @@
-from .mbart50 import *
-from . import m2m100
+from .m2m100 import *
+from . import m2m100, mbart50

dl_translate/lang/m2m100.py

@@ -48,6 +48,7 @@
 JAVANESE = "Javanese"
 GEORGIAN = "Georgian"
 KAZAKH = "Kazakh"
+KHMER = "Khmer"
 CENTRAL_KHMER = "Central Khmer"
 KANNADA = "Kannada"
 KOREAN = "Korean"

docs/available_languages.md

@@ -109,6 +109,7 @@ This page gives all the languages available for each model family.
 - Javanese (jv)
 - Georgian (ka)
 - Kazakh (kk)
+- Khmer (km)
 - Central Khmer (km)
 - Kannada (kn)
 - Korean (ko)

scripts/langs_coverage/m2m100.json

@@ -48,6 +48,7 @@
     "Javanese": "jv",
     "Georgian": "ka",
     "Kazakh": "kk",
+    "Khmer": "km",
     "Central Khmer": "km",
     "Kannada": "kn",
     "Korean": "ko",

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="dl-translate",
-    version="0.2.1",
+    version="0.2.2",
     author="Xing Han Lu",
     author_email="github@xinghanlu.com",
     description="A deep learning-based translation library built on Huggingface transformers",

tests/quick/test_lang.py

@@ -1,12 +1,17 @@
 import dl_translate as dlt
-from dl_translate._pairs import _PAIRS_MBART50
+from dl_translate._pairs import _PAIRS_MBART50, _PAIRS_M2M100
 
 
 def test_lang():
-    for l, _ in _PAIRS_MBART50:
-        assert getattr(dlt.lang, l.upper()) == l
+    for l, _ in _PAIRS_M2M100:
+        assert getattr(dlt.lang, l.upper().replace(" ", "_")) == l
+
+
+def test_lang_m2m100():
+    for l, _ in _PAIRS_M2M100:
+        assert getattr(dlt.lang.m2m100, l.upper().replace(" ", "_")) == l
 
 
 def test_lang_mbart50():
     for l, _ in _PAIRS_MBART50:
-        assert getattr(dlt.lang.mbart50, l.upper()) == l
+        assert getattr(dlt.lang.mbart50, l.upper().replace(" ", "_")) == l