From 03aa3a1f8e42eca2a10be4ccb73836e3fbedbfdb Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Fri, 28 Jan 2022 23:46:07 +0900 Subject: [PATCH 01/44] Workaround the installation issue #27 Not sure about the exact root cause, but the error messages suggested that there's issues with setuptools/pip. I can confirm we can fix the issue by changing the build-time setuptools requirement to =0.21.0", "numpy Date: Sat, 29 Jan 2022 00:34:41 +0900 Subject: [PATCH 02/44] parepare for v0.1.6 --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fce3799..c7ec18e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,9 +1,11 @@ Change log ========== -v0.1.6 <2021-xx-xx> +v0.1.6 <2022-01-29> ------------------- +* `#27`_: pyopenjtalk cannot be installed in google colab + v0.1.5 <2021-09-18> ------------------- @@ -73,3 +75,4 @@ Initial release with OpenJTalk's text processsing functionality .. _#22: https://github.com/r9y9/pyopenjtalk/pull/22 .. _#24: https://github.com/r9y9/pyopenjtalk/pull/24 .. _#25: https://github.com/r9y9/pyopenjtalk/pull/25 +.. _#27: https://github.com/r9y9/pyopenjtalk/issues/27 From e12f67d06a6a51fd5b4b18ca342e8c72a63b234e Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sat, 29 Jan 2022 00:41:07 +0900 Subject: [PATCH 03/44] Start a new dev cycle --- docs/changelog.rst | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c7ec18e..69da912 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,10 @@ Change log ========== +v0.1.7 <2022-xx-xx> +------------------- + + v0.1.6 <2022-01-29> ------------------- diff --git a/setup.py b/setup.py index 0f75369..b1ddce6 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ platform_is_windows = sys.platform == "win32" -version = "0.1.6" +version = "0.1.7" min_cython_ver = "0.21.0" try: From 2e18beaadf9341154a349b312b3edd3c60f211d2 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 6 Feb 2022 00:13:23 +0900 Subject: [PATCH 04/44] Update hts_engine_API --- lib/hts_engine_API | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hts_engine_API b/lib/hts_engine_API index b7e1c8b..214e26d 160000 --- a/lib/hts_engine_API +++ b/lib/hts_engine_API @@ -1 +1 @@ -Subproject commit b7e1c8b51787e19ea4376176afd9707c3c9d599a +Subproject commit 214e26dfb7f728ff9db39c14a59db709abcc121d From 35347fca93816e489c4b1f8e31a51aaf0732fb80 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 6 Feb 2022 00:13:33 +0900 Subject: [PATCH 05/44] Update open_jtalk https://github.com/r9y9/open_jtalk/releases/tag/v1.11.2 --- lib/open_jtalk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/open_jtalk b/lib/open_jtalk index 9572293..ce06fc5 160000 --- a/lib/open_jtalk +++ b/lib/open_jtalk @@ -1 +1 @@ -Subproject commit 957229334996d2c9d9fcb73cdb3f4d9c15bcdd57 +Subproject commit ce06fc565eecbf57a698dacde935f2060a60d627 From 39fdf677371c9b7309fecfea1af47c4e245443e2 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 6 Feb 2022 00:14:48 +0900 Subject: [PATCH 06/44] Update version to v0.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1ddce6..6f4af65 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ platform_is_windows = sys.platform == "win32" -version = "0.1.7" +version = "0.2.0" min_cython_ver = "0.21.0" try: From 83f3bb25a70c38ebf3aa62cbfa4a6f96c07c4b25 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 6 Feb 2022 00:25:45 +0900 Subject: [PATCH 07/44] [ci skip] update changelog --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 69da912..012d042 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,9 +1,10 @@ Change log ========== -v0.1.7 <2022-xx-xx> +v0.2.0 <2022-02-06> ------------------- +* `#29`_: Update binary dependencies (hts_engine_API/open_jtalk) v0.1.6 <2022-01-29> ------------------- @@ -80,3 +81,4 @@ Initial release with OpenJTalk's text processsing functionality .. _#24: https://github.com/r9y9/pyopenjtalk/pull/24 .. _#25: https://github.com/r9y9/pyopenjtalk/pull/25 .. _#27: https://github.com/r9y9/pyopenjtalk/issues/27 +.. _#29: https://github.com/r9y9/pyopenjtalk/pull/29 From 6486ed77a78420cb2f044bb7a63667576fe708d2 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 6 Feb 2022 00:46:59 +0900 Subject: [PATCH 08/44] Start new dev cycle --- docs/changelog.rst | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 012d042..83c3b17 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,10 @@ Change log ========== +v0.2.1 <2022-xx-xx> +------------------- + + v0.2.0 <2022-02-06> ------------------- diff --git a/setup.py b/setup.py index 6f4af65..22646c0 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ platform_is_windows = sys.platform == "win32" -version = "0.2.0" +version = "0.2.1" min_cython_ver = "0.21.0" try: From 70e4a3746c24d4008b073d2631b010e8d8ae2224 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 7 Aug 2022 21:58:45 +0900 Subject: [PATCH 09/44] Fixees for Python 3.10 --- .github/workflows/ci.yaml | 17 ++++++++++++++--- pyproject.toml | 8 ++++---- setup.py | 5 ++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f4b97e3..8b22dcd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,8 +15,19 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest] - python-version: [3.7, 3.8, 3.9] + include: + - os: ubuntu-latest + python-version: 3.7 + - os: ubuntu-latest + python-version: 3.8 + - os: ubuntu-latest + python-version: 3.9 + - os: ubuntu-latest + python-version: '3.10' + - os: macos-latest + python-version: 3.9 + - os: windows-latest + python-version: 3.9 steps: - uses: actions/checkout@v2 @@ -36,4 +47,4 @@ jobs: - name: Test with pytest run: | pip install pytest - pytest \ No newline at end of file + pytest diff --git a/pyproject.toml b/pyproject.toml index 7d72fcf..212e445 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,12 +2,12 @@ requires = [ "wheel", "setuptools=0.21.0", - "numpy=0.28.0", + "numpy>=1.20.0", ] [tool.pysen] -version = "0.9" +version = "0.10.2" [tool.pysen.lint] enable_black = true @@ -18,4 +18,4 @@ mypy_preset = "strict" line_length = 88 py_version = "py37" [[tool.pysen.lint.mypy_targets]] - paths = ["."] \ No newline at end of file + paths = ["."] diff --git a/setup.py b/setup.py index 22646c0..cca74f8 100644 --- a/setup.py +++ b/setup.py @@ -276,7 +276,7 @@ def run(self): ext_modules=ext_modules, cmdclass=cmdclass, install_requires=[ - "numpy >= 1.8.0", + "numpy >= 1.20.0", "cython >= " + min_cython_ver, "six", "tqdm", @@ -296,9 +296,11 @@ def run(self): "types-setuptools", "mypy<=0.910", "black>=19.19b0,<=20.8", + "click<8.1.0", "flake8>=3.7,<4", "flake8-bugbear", "isort>=4.3,<5.2.0", + "types-decorator", ], "test": ["pytest", "scipy"], }, @@ -313,6 +315,7 @@ def run(self): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "License :: OSI Approved :: MIT License", "Topic :: Scientific/Engineering", "Topic :: Software Development", From b8ead589f75ad06bc0182ce8939cae1396cd3211 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 7 Aug 2022 22:13:18 +0900 Subject: [PATCH 10/44] bump version for dev --- docs/changelog.rst | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 83c3b17..500abd4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,9 +1,12 @@ Change log ========== -v0.2.1 <2022-xx-xx> +v0.3.0 <2022-xx-xx> ------------------- +Newer numpy (>v1.20.0) is required to avoid ABI compatibility issues. + +* `#35`_: Fixes for Python 3.10. v0.2.0 <2022-02-06> ------------------- @@ -86,3 +89,4 @@ Initial release with OpenJTalk's text processsing functionality .. _#25: https://github.com/r9y9/pyopenjtalk/pull/25 .. _#27: https://github.com/r9y9/pyopenjtalk/issues/27 .. _#29: https://github.com/r9y9/pyopenjtalk/pull/29 +.. _#35: https://github.com/r9y9/pyopenjtalk/pull/35 diff --git a/setup.py b/setup.py index cca74f8..c68db9c 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ platform_is_windows = sys.platform == "win32" -version = "0.2.1" +version = "0.3.0" min_cython_ver = "0.21.0" try: From 09319789f67f2b0d53aaa191035728aa0675a334 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sun, 7 Aug 2022 22:54:16 +0900 Subject: [PATCH 11/44] Add a workaround for Numpy's ABI issue --- README.md | 2 +- docs/changelog.rst | 2 +- docs/index.rst | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0d490fc..c69149e 100644 --- a/README.md +++ b/README.md @@ -114,4 +114,4 @@ Out[3]: 'コンニチワ' ## Acknowledgements -HTS Working Group for their dedicated efforts to develop and maintain Open JTalk. \ No newline at end of file +HTS Working Group for their dedicated efforts to develop and maintain Open JTalk. diff --git a/docs/changelog.rst b/docs/changelog.rst index 500abd4..be60a9a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Change log v0.3.0 <2022-xx-xx> ------------------- -Newer numpy (>v1.20.0) is required to avoid ABI compatibility issues. +Newer numpy (>v1.20.0) is required to avoid ABI compatibility issues. Please check the updated installation guide. * `#35`_: Fixes for Python 3.10. diff --git a/docs/index.rst b/docs/index.rst index 441c879..f9040d0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,24 @@ The latest release is availabe on pypi. You can install it by: pip install pyopenjtalk + +Workaround for ``ValueError: numpy.ndarray size changed, may indicate binary incompatibility`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This type of errors comes from the Numpys' ABI breaking changes. If you see ``ValueError: numpy.ndarray size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject`` or similar, please make sure to install numpy first, and then install pyopenjtalk by: + +.. code:: + + pip install pyopenjtalk --no-build-isolation + +or: + +.. code:: + + pip install git+https://github.com/r9y9/pyopenjtalk --no-build-isolation + +The option ``--no-build-isolation`` tells pip not to create a build environment, so the pre-installed numpy is used to build the packge. Hense there should be no Numpy's ABI issues. + .. toctree:: :maxdepth: 1 :caption: Notebooks From 45c6eded4e4e4232843e70a59f7fed882d4a16d9 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Sat, 17 Sep 2022 00:09:19 +0900 Subject: [PATCH 12/44] add codes --- pyopenjtalk/__init__.py | 66 +++++++++++++++++++--- pyopenjtalk/openjtalk.pyx | 113 +++++++++++++++++++++++++------------- pyopenjtalk/utils.py | 21 +++++++ setup.py | 1 + 4 files changed, 153 insertions(+), 48 deletions(-) create mode 100644 pyopenjtalk/utils.py diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index a266104..22b8be2 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -19,6 +19,7 @@ from .htsengine import HTSEngine from .openjtalk import OpenJTalk +from .utils import merge_njd_marine_features # Dictionary directory # defaults to the package directory where the dictionary will be automatically downloaded @@ -39,6 +40,8 @@ # Global instance of HTSEngine # mei_normal.voice is used as default _global_htsengine = None +# Global instance of Marine +_global_marine = None # https://github.com/tqdm/tqdm#hooks-and-callbacks @@ -98,18 +101,49 @@ def g2p(*args, **kwargs): return _global_jtalk.g2p(*args, **kwargs) -def extract_fullcontext(text): +def estimate_accent(njd_features): + """Accent estimation using marine + + This function requires for installation of marine (https://github.com/6gsn/marine) + + Args: + njd_result (list): features generated by OpenJTalk. + + Returns: + list: features for NJDNode with estimation result by marine. + """ + global _global_marine + if _global_marine is None: + try: + from marine.predict import Predictor + except BaseException: + raise ImportError( + 'Please install marine by `pip install marine`' + ) + _global_marine = Predictor() + from marine.utils.openjtalk_util import convert_njd_feature_to_marine_feature + + marine_feature = convert_njd_feature_to_marine_feature(njd_features) + marine_results = _global_marine.predict([marine_feature], require_open_jtalk_format=True) + njd_features = merge_njd_marine_features(njd_features, marine_results) + return njd_features + + +def extract_fullcontext(text, do_marine=False): """Extract full-context labels from text Args: text (str): Input text + do_marine (bool): Whether run marine Returns: list: List of full-context labels """ - # note: drop first return - _, labels = run_frontend(text) - return labels + + njd_features = run_frontend(text) + if do_marine: + njd_features = estimate_accent(njd_features) + return make_label(njd_features) def synthesize(labels, speed=1.0, half_tone=0.0): @@ -151,19 +185,33 @@ def tts(text, speed=1.0, half_tone=0.0): return synthesize(extract_fullcontext(text), speed, half_tone) -def run_frontend(text, verbose=0): +def run_frontend(text): """Run OpenJTalk's text processing frontend Args: text (str): Unicode Japanese text. - verbose (int): Verbosity. Default is 0. Returns: - tuple: Pair of 1) NJD_print and 2) JPCommon_make_label. - The latter is the full-context labels in HTS-style format. + list: features for NJDNode. + """ + global _global_jtalk + if _global_jtalk is None: + _lazy_init() + _global_jtalk = OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) + return _global_jtalk.run_frontend(text) + + +def make_label(njd_features): + """Make full-context label using features + + Args: + njd_features (list): features for NJDNode. + + Returns: + list: full-context labels. """ global _global_jtalk if _global_jtalk is None: _lazy_init() _global_jtalk = OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) - return _global_jtalk.run_frontend(text, verbose) + return _global_jtalk.make_label(njd_features) diff --git a/pyopenjtalk/openjtalk.pyx b/pyopenjtalk/openjtalk.pyx index 46e6004..f28509f 100644 --- a/pyopenjtalk/openjtalk.pyx +++ b/pyopenjtalk/openjtalk.pyx @@ -8,6 +8,7 @@ cimport numpy as np np.import_array() cimport cython +from libc.stdlib cimport calloc from openjtalk.mecab cimport Mecab, Mecab_initialize, Mecab_load, Mecab_analysis from openjtalk.mecab cimport Mecab_get_feature, Mecab_get_size, Mecab_refresh, Mecab_clear @@ -64,32 +65,57 @@ cdef njd_node_get_chain_flag(_njd.NJDNode* node): return _njd.NJDNode_get_chain_flag(node) -cdef njd_node_print(_njd.NJDNode* node): - return "{},{},{},{},{},{},{},{},{},{},{}/{},{},{}".format( - njd_node_get_string(node), - njd_node_get_pos(node), - njd_node_get_pos_group1(node), - njd_node_get_pos_group2(node), - njd_node_get_pos_group3(node), - njd_node_get_ctype(node), - njd_node_get_cform(node), - njd_node_get_orig(node), - njd_node_get_read(node), - njd_node_get_pron(node), - njd_node_get_acc(node), - njd_node_get_mora_size(node), - njd_node_get_chain_rule(node), - njd_node_get_chain_flag(node) - ) - - -cdef njd_print(_njd.NJD* njd): +cdef node2feature(_njd.NJDNode* node): + return { + "string": njd_node_get_string(node), + "pos": njd_node_get_pos(node), + "pos_group1": njd_node_get_pos_group1(node), + "pos_group2": njd_node_get_pos_group2(node), + "pos_group3": njd_node_get_pos_group3(node), + "ctype": njd_node_get_ctype(node), + "cform": njd_node_get_cform(node), + "orig": njd_node_get_orig(node), + "read": njd_node_get_read(node), + "pron": njd_node_get_pron(node), + "acc": njd_node_get_acc(node), + "mora_size": njd_node_get_mora_size(node), + "chain_rule": njd_node_get_chain_rule(node), + "chain_flag": njd_node_get_chain_flag(node), + } + + +cdef njd2feature(_njd.NJD* njd): cdef _njd.NJDNode* node = njd.head - njd_results = [] + features = [] while node is not NULL: - njd_results.append(njd_node_print(node)) + features.append(node2feature(node)) node = node.next - return njd_results + return features + + +cdef feature2njd(_njd.NJD* njd, features): + cdef _njd.NJDNode* node + + for feature_node in features: + node = <_njd.NJDNode *> calloc(1, sizeof(_njd.NJDNode)) + _njd.NJDNode_initialize(node) + # set values + _njd.NJDNode_set_string(node, feature_node["string"].encode("utf-8")) + _njd.NJDNode_set_pos(node, feature_node["pos"].encode("utf-8")) + _njd.NJDNode_set_pos_group1(node, feature_node["pos_group1"].encode("utf-8")) + _njd.NJDNode_set_pos_group2(node, feature_node["pos_group2"].encode("utf-8")) + _njd.NJDNode_set_pos_group3(node, feature_node["pos_group3"].encode("utf-8")) + _njd.NJDNode_set_ctype(node, feature_node["ctype"].encode("utf-8")) + _njd.NJDNode_set_cform(node, feature_node["cform"].encode("utf-8")) + _njd.NJDNode_set_orig(node, feature_node["orig"].encode("utf-8")) + _njd.NJDNode_set_read(node, feature_node["read"].encode("utf-8")) + _njd.NJDNode_set_pron(node, feature_node["pron"].encode("utf-8")) + _njd.NJDNode_set_acc(node, feature_node["acc"]) + _njd.NJDNode_set_mora_size(node, feature_node["mora_size"]) + _njd.NJDNode_set_chain_rule(node, feature_node["chain_rule"].encode("utf-8")) + _njd.NJDNode_set_chain_flag(node, feature_node["chain_flag"]) + _njd.NJD_push_node(njd, node) + cdef class OpenJTalk(object): """OpenJTalk @@ -125,12 +151,13 @@ cdef class OpenJTalk(object): return Mecab_load(self.mecab, dn_mecab) - def run_frontend(self, text, verbose=0): + def run_frontend(self, text): """Run OpenJTalk's text processing frontend """ - if isinstance(text, str): - text = text.encode("utf-8") cdef char buff[8192] + + if isinstance(text, str): + text = text.encode("utf-8") text2mecab(buff, text) Mecab_analysis(self.mecab, buff) mecab2njd(self.njd, Mecab_get_feature(self.mecab), Mecab_get_size(self.mecab)) @@ -140,7 +167,20 @@ cdef class OpenJTalk(object): _njd.njd_set_accent_type(self.njd) _njd.njd_set_unvoiced_vowel(self.njd) _njd.njd_set_long_vowel(self.njd) + features = njd2feature(self.njd) + + # Note that this will release memory for njd feature + NJD_refresh(self.njd) + Mecab_refresh(self.mecab) + + return features + + def make_label(self, features): + """Make full-context label + """ + feature2njd(self.njd, features) njd2jpcommon(self.jpcommon, self.njd) + JPCommon_make_label(self.jpcommon) cdef int label_size = JPCommon_get_label_size(self.jpcommon) @@ -153,23 +193,19 @@ cdef class OpenJTalk(object): # http://cython.readthedocs.io/en/latest/src/tutorial/strings.html labels.append(label_feature[i]) - njd_results = njd_print(self.njd) - - if verbose > 0: - NJD_print(self.njd) - # Note that this will release memory for label feature JPCommon_refresh(self.jpcommon) NJD_refresh(self.njd) - Mecab_refresh(self.mecab) - return njd_results, labels + return labels def g2p(self, text, kana=False, join=True): """Grapheme-to-phoeneme (G2P) conversion """ - njd_results, labels = self.run_frontend(text) + njd_features = self.run_frontend(text) + if not kana: + labels = self.make_label(njd_features) prons = list(map(lambda s: s.split("-")[1].split("+")[0], labels[1:-1])) if join: prons = " ".join(prons) @@ -177,12 +213,11 @@ cdef class OpenJTalk(object): # kana prons = [] - for n in njd_results: - row = n.split(",") - if row[1] == "記号": - p = row[0] + for n in njd_features: + if n["pos"] == "記号": + p = n["string"] else: - p = row[9] + p = n["pron"] # remove special chars for c in "’": p = p.replace(c,"") diff --git a/pyopenjtalk/utils.py b/pyopenjtalk/utils.py new file mode 100644 index 0000000..906ce4b --- /dev/null +++ b/pyopenjtalk/utils.py @@ -0,0 +1,21 @@ +def merge_njd_marine_features(njd_features, marine_results): + features = [] + + marine_accs = marine_results["accent_status"] + marine_chain_flags = marine_results["accent_phrase_boundary"] + + assert len(njd_features) == len(marine_accs) == len( + marine_chain_flags + ), "Invalid sequence sizes in njd_results, marine_results" + + for node_index, njd_feature in enumerate(njd_features): + _feature = {} + for feature_key in njd_feature.keys(): + if feature_key == "acc": + _feature["acc"] = int(marine_accs[node_index]) + elif feature_key == "chain_flag": + _feature[feature_key] = int(marine_chain_flags[node_index]) + else: + _feature[feature_key] = njd_feature[feature_key] + features.append(_feature) + return features diff --git a/setup.py b/setup.py index c68db9c..62ec246 100644 --- a/setup.py +++ b/setup.py @@ -303,6 +303,7 @@ def run(self): "types-decorator", ], "test": ["pytest", "scipy"], + "marine": ["marine>=0.0.5"], }, classifiers=[ "Operating System :: POSIX", From cf7c78fd58a149d0d4a55acc4dbf5b78b8a64c27 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Sat, 17 Sep 2022 00:25:32 +0900 Subject: [PATCH 13/44] update tests --- pyopenjtalk/__init__.py | 10 +++++----- pyopenjtalk/utils.py | 4 ++-- tests/test_openjtalk.py | 22 ++++++++++++---------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index 22b8be2..5c86628 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -103,7 +103,7 @@ def g2p(*args, **kwargs): def estimate_accent(njd_features): """Accent estimation using marine - + This function requires for installation of marine (https://github.com/6gsn/marine) Args: @@ -117,14 +117,14 @@ def estimate_accent(njd_features): try: from marine.predict import Predictor except BaseException: - raise ImportError( - 'Please install marine by `pip install marine`' - ) + raise ImportError("Please install marine by `pip install -e '.[marine]'`") _global_marine = Predictor() from marine.utils.openjtalk_util import convert_njd_feature_to_marine_feature marine_feature = convert_njd_feature_to_marine_feature(njd_features) - marine_results = _global_marine.predict([marine_feature], require_open_jtalk_format=True) + marine_results = _global_marine.predict( + [marine_feature], require_open_jtalk_format=True + ) njd_features = merge_njd_marine_features(njd_features, marine_results) return njd_features diff --git a/pyopenjtalk/utils.py b/pyopenjtalk/utils.py index 906ce4b..7aeb1bf 100644 --- a/pyopenjtalk/utils.py +++ b/pyopenjtalk/utils.py @@ -4,8 +4,8 @@ def merge_njd_marine_features(njd_features, marine_results): marine_accs = marine_results["accent_status"] marine_chain_flags = marine_results["accent_phrase_boundary"] - assert len(njd_features) == len(marine_accs) == len( - marine_chain_flags + assert ( + len(njd_features) == len(marine_accs) == len(marine_chain_flags) ), "Invalid sequence sizes in njd_results, marine_results" for node_index, njd_feature in enumerate(njd_features): diff --git a/tests/test_openjtalk.py b/tests/test_openjtalk.py index 56b4a80..abff87e 100644 --- a/tests/test_openjtalk.py +++ b/tests/test_openjtalk.py @@ -1,10 +1,9 @@ import pyopenjtalk -def _print_results(njd_results, labels): - for n in njd_results: - row = n.split(",") - s, p = row[0], row[9] +def _print_results(njd_features, labels): + for f in njd_features: + s, p = f["string"], f["pron"] print(s, p) for label in labels: @@ -12,12 +11,14 @@ def _print_results(njd_results, labels): def test_hello(): - njd_results, labels = pyopenjtalk.run_frontend("こんにちは") - _print_results(njd_results, labels) + njd_features = pyopenjtalk.run_frontend("こんにちは") + labels = pyopenjtalk.make_label(njd_features) + _print_results(njd_features, labels) def test_fullcontext(): - _, labels = pyopenjtalk.run_frontend("こんにちは") + features = pyopenjtalk.run_frontend("こんにちは") + labels = pyopenjtalk.make_label(features) labels2 = pyopenjtalk.extract_fullcontext("こんにちは") for a, b in zip(labels, labels2): assert a == b @@ -30,10 +31,11 @@ def test_jtalk(): "どんまい！", "パソコンのとりあえず知っておきたい使い方", ]: - njd_results, labels = pyopenjtalk.run_frontend(text) - _print_results(njd_results, labels) + njd_features = pyopenjtalk.run_frontend(text) + labels = pyopenjtalk.make_label(njd_features) + _print_results(njd_features, labels) - surface = "".join(map(lambda s: s.split(",")[0], njd_results)) + surface = "".join(map(lambda f: f["string"], njd_features)) assert surface == text From 93d1a9001b9ceb5fe10463e799c684ffaa231045 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Sat, 17 Sep 2022 00:42:19 +0900 Subject: [PATCH 14/44] add test for frontend --- pyopenjtalk/__init__.py | 10 +++++----- tests/test_openjtalk.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index 5c86628..c44d96a 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -104,13 +104,13 @@ def g2p(*args, **kwargs): def estimate_accent(njd_features): """Accent estimation using marine - This function requires for installation of marine (https://github.com/6gsn/marine) + This function requires marine (https://github.com/6gsn/marine) Args: njd_result (list): features generated by OpenJTalk. Returns: - list: features for NJDNode with estimation result by marine. + list: features for NJDNode with estimation results by marine. """ global _global_marine if _global_marine is None: @@ -129,19 +129,19 @@ def estimate_accent(njd_features): return njd_features -def extract_fullcontext(text, do_marine=False): +def extract_fullcontext(text, run_marine=False): """Extract full-context labels from text Args: text (str): Input text - do_marine (bool): Whether run marine + run_marine (bool): Whether run marine for estimating accent using DNN-based method Returns: list: List of full-context labels """ njd_features = run_frontend(text) - if do_marine: + if run_marine: njd_features = estimate_accent(njd_features) return make_label(njd_features) diff --git a/tests/test_openjtalk.py b/tests/test_openjtalk.py index abff87e..0f43363 100644 --- a/tests/test_openjtalk.py +++ b/tests/test_openjtalk.py @@ -16,6 +16,29 @@ def test_hello(): _print_results(njd_features, labels) +def test_njd_features(): + njd_features = pyopenjtalk.run_frontend("こんにちは") + expected_feature = [ + { + "string": "こんにちは", + "pos": "感動詞", + "pos_group1": "*", + "pos_group2": "*", + "pos_group3": "*", + "ctype": "*", + "cform": "*", + "orig": "こんにちは", + "read": "コンニチハ", + "pron": "コンニチワ", + "acc": 0, + "mora_size": 5, + "chain_rule": "-1", + "chain_flag": -1, + } + ] + assert njd_features == expected_feature + + def test_fullcontext(): features = pyopenjtalk.run_frontend("こんにちは") labels = pyopenjtalk.make_label(features) From 8f8a9f137971f49e4089dfeeddfd17edb3732705 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 01:12:00 +0900 Subject: [PATCH 15/44] add run_marine option to tts() --- pyopenjtalk/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index c44d96a..5863369 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -134,7 +134,9 @@ def extract_fullcontext(text, run_marine=False): Args: text (str): Input text - run_marine (bool): Whether run marine for estimating accent using DNN-based method + run_marine (bool): Whether estimate accent using marine. + Default is False. If you want activate this option, you need to install marine + by `pip install pyopenjtalk[marine]` Returns: list: List of full-context labels @@ -170,19 +172,24 @@ def synthesize(labels, speed=1.0, half_tone=0.0): return _global_htsengine.synthesize(labels), sr -def tts(text, speed=1.0, half_tone=0.0): +def tts(text, speed=1.0, half_tone=0.0, run_marine=False): """Text-to-speech Args: text (str): Input text speed (float): speech speed rate. Default is 1.0. half_tone (float): additional half-tone. Default is 0. + run_marine (bool): Whether estimate accent using marine. + Default is False. If you want activate this option, you need to install marine + by `pip install pyopenjtalk[marine]` Returns: np.ndarray: speech waveform (dtype: np.float64) int: sampling frequency (defualt: 48000) """ - return synthesize(extract_fullcontext(text), speed, half_tone) + return synthesize( + extract_fullcontext(text, run_marine=run_marine), speed, half_tone + ) def run_frontend(text): From a59b7f052c3f28d8f9e59d8cd6a4d05def90ad4a Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 01:12:20 +0900 Subject: [PATCH 16/44] update the README for run_marin_option --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index c69149e..c456d4f 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,24 @@ In [3]: pyopenjtalk.g2p("こんにちは", kana=True) Out[3]: 'コンニチワ' ``` +### About `run_marine` option + +After v0.2.1, the `run_marine` option has been available for estimating the Japanese accent with the DNN-based method (see [marine](https://github.com/6gsn/marine)). If you want to use the feature, please install pyopenjtalk as below; + +```shell +pip install pyopenjtalk[marine] +``` + +And then, you can use the option as the following examples; + +```python +In [1]: import pyopenjtalk + +In [2]: x, sr = pyopenjtalk.tts("おめでとうございます") # for TTS + +In [3]: label = pyopenjtalk.extract_fullcontext("こんにちは") # for text processing frontend only +``` + ## LICENSE From b825fd2ebeb900b65674193df66859a2c6e55fc5 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 01:27:58 +0900 Subject: [PATCH 17/44] update error message for importing marine --- pyopenjtalk/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index 5863369..7d73ebe 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -117,7 +117,7 @@ def estimate_accent(njd_features): try: from marine.predict import Predictor except BaseException: - raise ImportError("Please install marine by `pip install -e '.[marine]'`") + raise ImportError("Please install marine by `pip install pyopenjtalk[marine]`") _global_marine = Predictor() from marine.utils.openjtalk_util import convert_njd_feature_to_marine_feature From a4a4e8121e0f5b148ce49263fc7e08df2f95f864 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 01:28:35 +0900 Subject: [PATCH 18/44] fix for lint --- pyopenjtalk/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index 7d73ebe..23890c7 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -117,7 +117,9 @@ def estimate_accent(njd_features): try: from marine.predict import Predictor except BaseException: - raise ImportError("Please install marine by `pip install pyopenjtalk[marine]`") + raise ImportError( + "Please install marine by `pip install pyopenjtalk[marine]`" + ) _global_marine = Predictor() from marine.utils.openjtalk_util import convert_njd_feature_to_marine_feature From f1b87ca42f84e6fa41a4c461fba459c16564b200 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 18:31:40 +0900 Subject: [PATCH 19/44] add marine's license --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c456d4f..e1c8321 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ In [3]: label = pyopenjtalk.extract_fullcontext("こんにちは") # for text pr - pyopenjtalk: MIT license ([LICENSE.md](LICENSE.md)) - Open JTalk: Modified BSD license ([COPYING](https://github.com/r9y9/open_jtalk/blob/1.10/src/COPYING)) - htsvoice in this repository: Please check [pyopenjtalk/htsvoice/README.md](pyopenjtalk/htsvoice/README.md). +- marine: marine: Apache 2.0 license ([LICENSE](https://github.com/6gsn/marine/blob/main/LICENSE)) ## Acknowledgements From aae4fce17bf993ecef6cfb64b28630d0703392b8 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 18:34:21 +0900 Subject: [PATCH 20/44] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e1c8321..8d61287 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Out[3]: 'コンニチワ' ### About `run_marine` option -After v0.2.1, the `run_marine` option has been available for estimating the Japanese accent with the DNN-based method (see [marine](https://github.com/6gsn/marine)). If you want to use the feature, please install pyopenjtalk as below; +After v0.3.0, the `run_marine` option has been available for estimating the Japanese accent with the DNN-based method (see [marine](https://github.com/6gsn/marine)). If you want to use the feature, please install pyopenjtalk as below; ```shell pip install pyopenjtalk[marine] @@ -118,9 +118,9 @@ And then, you can use the option as the following examples; ```python In [1]: import pyopenjtalk -In [2]: x, sr = pyopenjtalk.tts("おめでとうございます") # for TTS +In [2]: x, sr = pyopenjtalk.tts("おめでとうございます", run_marine=True) # for TTS -In [3]: label = pyopenjtalk.extract_fullcontext("こんにちは") # for text processing frontend only +In [3]: label = pyopenjtalk.extract_fullcontext("こんにちは", run_marine=True) # for text processing frontend only ``` From 34528ad594a1d2975f7c3922130924e944051057 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 18:37:43 +0900 Subject: [PATCH 21/44] add new API in docs/pyopenjtalk.rst --- docs/pyopenjtalk.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pyopenjtalk.rst b/docs/pyopenjtalk.rst index 370d2f5..5e03e7b 100644 --- a/docs/pyopenjtalk.rst +++ b/docs/pyopenjtalk.rst @@ -25,3 +25,5 @@ Misc ---- .. autofunction:: run_frontend +.. autofunction:: make_label +.. autofunction:: estimate_accent From 2c10e3f734209de6867328b51e5c9420b33ceb31 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 18:48:38 +0900 Subject: [PATCH 22/44] add chage log --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index be60a9a..662d44b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ v0.3.0 <2022-xx-xx> Newer numpy (>v1.20.0) is required to avoid ABI compatibility issues. Please check the updated installation guide. +* `#40`_: Introduce marine for Japanese accent estimation. Introduce marine for Japanese accent estimation. Note that there could be a breakpoint regarding `run_frontend` because this PR changed the behavior of the API. * `#35`_: Fixes for Python 3.10. v0.2.0 <2022-02-06> From 23e7363a62014dd694992d31cb75d18fd16a7fe1 Mon Sep 17 00:00:00 2001 From: "park.byeongseon" Date: Mon, 19 Sep 2022 18:51:21 +0900 Subject: [PATCH 23/44] fix typo --- README.md | 2 +- docs/changelog.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d61287..76b7369 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ In [3]: label = pyopenjtalk.extract_fullcontext("こんにちは", run_marine=Tr - pyopenjtalk: MIT license ([LICENSE.md](LICENSE.md)) - Open JTalk: Modified BSD license ([COPYING](https://github.com/r9y9/open_jtalk/blob/1.10/src/COPYING)) - htsvoice in this repository: Please check [pyopenjtalk/htsvoice/README.md](pyopenjtalk/htsvoice/README.md). -- marine: marine: Apache 2.0 license ([LICENSE](https://github.com/6gsn/marine/blob/main/LICENSE)) +- marine: Apache 2.0 license ([LICENSE](https://github.com/6gsn/marine/blob/main/LICENSE)) ## Acknowledgements diff --git a/docs/changelog.rst b/docs/changelog.rst index 662d44b..44faab0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ v0.3.0 <2022-xx-xx> Newer numpy (>v1.20.0) is required to avoid ABI compatibility issues. Please check the updated installation guide. -* `#40`_: Introduce marine for Japanese accent estimation. Introduce marine for Japanese accent estimation. Note that there could be a breakpoint regarding `run_frontend` because this PR changed the behavior of the API. +* `#40`_: Introduce marine for Japanese accent estimation. Note that there could be a breakpoint regarding `run_frontend` because this PR changed the behavior of the API. * `#35`_: Fixes for Python 3.10. v0.2.0 <2022-02-06> From 7361ec6eba0090f2c5c5561e461c245ee47a44d8 Mon Sep 17 00:00:00 2001 From: "bgsn.pk" Date: Mon, 19 Sep 2022 23:07:29 +0900 Subject: [PATCH 24/44] Update pyopenjtalk/__init__.py Co-authored-by: Ryuichi Yamamoto --- pyopenjtalk/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index 23890c7..1f08183 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -181,7 +181,7 @@ def tts(text, speed=1.0, half_tone=0.0, run_marine=False): text (str): Input text speed (float): speech speed rate. Default is 1.0. half_tone (float): additional half-tone. Default is 0. - run_marine (bool): Whether estimate accent using marine. + run_marine (bool): Whether to estimate accent using marine. Default is False. If you want activate this option, you need to install marine by `pip install pyopenjtalk[marine]` From 11c2b974fc244ebf7e5bf07222667719c36c7bb0 Mon Sep 17 00:00:00 2001 From: "bgsn.pk" Date: Mon, 19 Sep 2022 23:07:50 +0900 Subject: [PATCH 25/44] Update pyopenjtalk/__init__.py Co-authored-by: Ryuichi Yamamoto --- pyopenjtalk/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index 1f08183..fec7150 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -136,7 +136,7 @@ def extract_fullcontext(text, run_marine=False): Args: text (str): Input text - run_marine (bool): Whether estimate accent using marine. + run_marine (bool): Whether to estimate accent using marine. Default is False. If you want activate this option, you need to install marine by `pip install pyopenjtalk[marine]` From ba42f0b9d62142cfd14b8fd9ec5736bb36ad473c Mon Sep 17 00:00:00 2001 From: "bgsn.pk" Date: Mon, 19 Sep 2022 23:07:58 +0900 Subject: [PATCH 26/44] Update pyopenjtalk/__init__.py Co-authored-by: Ryuichi Yamamoto --- pyopenjtalk/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index fec7150..299ac29 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -137,7 +137,7 @@ def extract_fullcontext(text, run_marine=False): Args: text (str): Input text run_marine (bool): Whether to estimate accent using marine. - Default is False. If you want activate this option, you need to install marine + Default is False. If you want to activate this option, you need to install marine by `pip install pyopenjtalk[marine]` Returns: From 60d85ff0dffe97de8dc0c7b263de90caf6025bb2 Mon Sep 17 00:00:00 2001 From: 6gsn Date: Mon, 19 Sep 2022 23:10:11 +0900 Subject: [PATCH 27/44] add link --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 44faab0..e75eee2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -91,3 +91,4 @@ Initial release with OpenJTalk's text processsing functionality .. _#27: https://github.com/r9y9/pyopenjtalk/issues/27 .. _#29: https://github.com/r9y9/pyopenjtalk/pull/29 .. _#35: https://github.com/r9y9/pyopenjtalk/pull/35 +.. _#40: https://github.com/r9y9/pyopenjtalk/pull/40 From d6ff4377fae6bb6feb325df892736b914dd00c91 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Tue, 20 Sep 2022 01:46:15 +0900 Subject: [PATCH 28/44] prep for release --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e75eee2..cfb72f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,7 +1,7 @@ Change log ========== -v0.3.0 <2022-xx-xx> +v0.3.0 <2022-09-20> ------------------- Newer numpy (>v1.20.0) is required to avoid ABI compatibility issues. Please check the updated installation guide. From be70add6fc664ba9d6ddac086b35cfeccebbc9bb Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sat, 8 Apr 2023 01:18:02 +0900 Subject: [PATCH 29/44] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 62ec246..6cfc107 100644 --- a/setup.py +++ b/setup.py @@ -301,6 +301,7 @@ def run(self): "flake8-bugbear", "isort>=4.3,<5.2.0", "types-decorator", + "importlib-metadata<5.0", ], "test": ["pytest", "scipy"], "marine": ["marine>=0.0.5"], From d1781bfa02deacded2a7d0102bf436c4d3890c27 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Sat, 8 Apr 2023 01:20:46 +0900 Subject: [PATCH 30/44] Fix lint --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6cfc107..336144a 100644 --- a/setup.py +++ b/setup.py @@ -301,7 +301,7 @@ def run(self): "flake8-bugbear", "isort>=4.3,<5.2.0", "types-decorator", - "importlib-metadata<5.0", + "importlib-metadata<5.0", ], "test": ["pytest", "scipy"], "marine": ["marine>=0.0.5"], From ecd587a9e4515f89f3486d7382ab1d633b215861 Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Wed, 5 Apr 2023 19:35:55 +0900 Subject: [PATCH 31/44] Use oldest-supported-numpy --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 212e445..6c338a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "wheel", "setuptools=0.28.0", - "numpy>=1.20.0", + "oldest-supported-numpy", ] [tool.pysen] From aec2cfff5d910f8cd59e2579538cc0b1a6592e6a Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Mon, 19 Jun 2023 17:12:49 +0900 Subject: [PATCH 32/44] ADD: build-backend Remove wheel in requires --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6c338a5..c0bd23d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] requires = [ - "wheel", "setuptools=0.28.0", "oldest-supported-numpy", ] +build-backend = "setuptools.build_meta" [tool.pysen] version = "0.10.2" From e040337e74cdfa6879508377e74fc695a80d4472 Mon Sep 17 00:00:00 2001 From: Sharvil Nanavati Date: Wed, 19 Jul 2023 20:40:51 +0000 Subject: [PATCH 33/44] Limit maximum Cython version to avoid build break Issue: #55 --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0bd23d..24413ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ "setuptools=0.28.0", + "cython>=0.28.0,<3.0.0", "oldest-supported-numpy", ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 336144a..443d9a3 100644 --- a/setup.py +++ b/setup.py @@ -277,7 +277,7 @@ def run(self): cmdclass=cmdclass, install_requires=[ "numpy >= 1.20.0", - "cython >= " + min_cython_ver, + "cython >= " + min_cython_ver + ",<3.0.0", "six", "tqdm", ], From 4400e2cb29c5c3c1b3b6b0cc65df14dd277cd4b7 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Thu, 20 Jul 2023 13:27:27 +0900 Subject: [PATCH 34/44] prep for v0.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 443d9a3..1f6dba6 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ platform_is_windows = sys.platform == "win32" -version = "0.3.0" +version = "0.3.1" min_cython_ver = "0.21.0" try: From c826bfe5e3eed836e8b147e6bf03931730753906 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Thu, 20 Jul 2023 13:27:59 +0900 Subject: [PATCH 35/44] start new dev cycle --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f6dba6..bb8ac16 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ platform_is_windows = sys.platform == "win32" -version = "0.3.1" +version = "0.3.2-dev" min_cython_ver = "0.21.0" try: From 471a6a6b5a51b495ed59220e2844ac9d3ad7efe6 Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Wed, 19 Jul 2023 22:09:58 +0900 Subject: [PATCH 36/44] FIX: Fix compile error when Cython 3.0 --- pyopenjtalk/htsengine.pyx | 4 ++-- pyopenjtalk/openjtalk.pyx | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pyopenjtalk/htsengine.pyx b/pyopenjtalk/htsengine.pyx index a69400c..9936a6e 100644 --- a/pyopenjtalk/htsengine.pyx +++ b/pyopenjtalk/htsengine.pyx @@ -10,8 +10,8 @@ np.import_array() cimport cython from libc.stdlib cimport malloc, free -from htsengine cimport HTS_Engine -from htsengine cimport ( +from .htsengine cimport HTS_Engine +from .htsengine cimport ( HTS_Engine_initialize, HTS_Engine_load, HTS_Engine_clear, HTS_Engine_refresh, HTS_Engine_get_sampling_frequency, HTS_Engine_get_fperiod, HTS_Engine_set_speed, HTS_Engine_add_half_tone, diff --git a/pyopenjtalk/openjtalk.pyx b/pyopenjtalk/openjtalk.pyx index f28509f..96505b8 100644 --- a/pyopenjtalk/openjtalk.pyx +++ b/pyopenjtalk/openjtalk.pyx @@ -10,17 +10,17 @@ np.import_array() cimport cython from libc.stdlib cimport calloc -from openjtalk.mecab cimport Mecab, Mecab_initialize, Mecab_load, Mecab_analysis -from openjtalk.mecab cimport Mecab_get_feature, Mecab_get_size, Mecab_refresh, Mecab_clear -from openjtalk.njd cimport NJD, NJD_initialize, NJD_refresh, NJD_print, NJD_clear -from openjtalk cimport njd as _njd -from openjtalk.jpcommon cimport JPCommon, JPCommon_initialize,JPCommon_make_label -from openjtalk.jpcommon cimport JPCommon_get_label_size, JPCommon_get_label_feature -from openjtalk.jpcommon cimport JPCommon_refresh, JPCommon_clear -from openjtalk cimport njd2jpcommon -from openjtalk.text2mecab cimport text2mecab -from openjtalk.mecab2njd cimport mecab2njd -from openjtalk.njd2jpcommon cimport njd2jpcommon +from .openjtalk.mecab cimport Mecab, Mecab_initialize, Mecab_load, Mecab_analysis +from .openjtalk.mecab cimport Mecab_get_feature, Mecab_get_size, Mecab_refresh, Mecab_clear +from .openjtalk.njd cimport NJD, NJD_initialize, NJD_refresh, NJD_print, NJD_clear +from .openjtalk cimport njd as _njd +from .openjtalk.jpcommon cimport JPCommon, JPCommon_initialize,JPCommon_make_label +from .openjtalk.jpcommon cimport JPCommon_get_label_size, JPCommon_get_label_feature +from .openjtalk.jpcommon cimport JPCommon_refresh, JPCommon_clear +from .openjtalk cimport njd2jpcommon +from .openjtalk.text2mecab cimport text2mecab +from .openjtalk.mecab2njd cimport mecab2njd +from .openjtalk.njd2jpcommon cimport njd2jpcommon cdef njd_node_get_string(_njd.NJDNode* node): return ((_njd.NJDNode_get_string(node))).decode("utf-8") From bf0780d9731155e5c7bc9affd69beebd9bad739f Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Wed, 26 Jul 2023 01:45:59 +0900 Subject: [PATCH 37/44] ENH: Fix requires numpy version --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 24413ed..999e500 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,8 @@ requires = [ "setuptools=0.28.0,<3.0.0", - "oldest-supported-numpy", + "numpy>=1.25.0; python_version>='3.9'", + "oldest-supported-numpy; python_version<'3.9'", ] build-backend = "setuptools.build_meta" From 2e59f814efc276721ee2ce49bf761246d368280e Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Fri, 21 Jul 2023 07:50:16 +0900 Subject: [PATCH 38/44] ENH: add build-system requires cmake --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 999e500..d5b34cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = [ "setuptools=0.28.0,<3.0.0", + "cmake", "numpy>=1.25.0; python_version>='3.9'", "oldest-supported-numpy; python_version<'3.9'", ] From 7b892c891e0f39bc49458d965b904381b9f406e4 Mon Sep 17 00:00:00 2001 From: sabonerune <102559104+sabonerune@users.noreply.github.com> Date: Thu, 3 Aug 2023 17:54:45 +0900 Subject: [PATCH 39/44] ENH: Remove Cython from install_requires. Change lint to dev. Add Cython to dev. --- .github/workflows/ci.yaml | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b22dcd..f383277 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,7 +40,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[lint]" + pip install -e ".[dev]" - name: Lint with pysen run: | pysen run lint diff --git a/setup.py b/setup.py index bb8ac16..66593ca 100644 --- a/setup.py +++ b/setup.py @@ -277,7 +277,6 @@ def run(self): cmdclass=cmdclass, install_requires=[ "numpy >= 1.20.0", - "cython >= " + min_cython_ver + ",<3.0.0", "six", "tqdm", ], @@ -291,7 +290,8 @@ def run(self): "ipython", "jupyter", ], - "lint": [ + "dev": [ + "cython >= " + min_cython_ver + ",<3.0.0", "pysen", "types-setuptools", "mypy<=0.910", From 807e40665f5bdf957215712ff11155fbf852b736 Mon Sep 17 00:00:00 2001 From: YCHuang2112 <96855801+YCHuang2112@users.noreply.github.com> Date: Sun, 20 Aug 2023 14:23:46 +0800 Subject: [PATCH 40/44] Checking cmake is in the system path for windows --- setup.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/setup.py b/setup.py index 66593ca..06e74b8 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,36 @@ def build_extensions(self): raise RuntimeError("Cython is required to generate C++ code") + + +def check_cmake_in_path(): + try: + result = subprocess.run(['cmake', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + # CMake is in the system path + return True, result.stdout.strip() + else: + # CMake is not in the system path + return False, None + except FileNotFoundError: + # CMake command not found + return False, None + +if os.name == 'nt': # Check if the OS is Windows + # Check if CMake is in the system path + cmake_found, cmake_version = check_cmake_in_path() + + if cmake_found: + print(f"CMake is in the system path. Version: {cmake_version}") + else: + raise SystemError("CMake is not found in the system path. Make sure CMake is installed and in the system path.") + + + + + + + # Workaround for `distutils.spawn` problem on Windows python < 3.9 # See details: [bpo-39763: distutils.spawn now uses subprocess (GH-18743)] # (https://github.com/python/cpython/commit/1ec63b62035e73111e204a0e03b83503e1c58f2e) From b2bb286a354a711a7469c50351ae213b1ed8d4cf Mon Sep 17 00:00:00 2001 From: YCHuang2112 <96855801+YCHuang2112@users.noreply.github.com> Date: Mon, 21 Aug 2023 07:13:04 +0800 Subject: [PATCH 41/44] update lint error --- .vscode/settings.json | 4 ++++ setup.py | 8 +------- 2 files changed, 5 insertions(+), 7 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e137fad --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/setup.py b/setup.py index 06e74b8..147fa69 100644 --- a/setup.py +++ b/setup.py @@ -71,8 +71,6 @@ def build_extensions(self): raise RuntimeError("Cython is required to generate C++ code") - - def check_cmake_in_path(): try: result = subprocess.run(['cmake', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -86,6 +84,7 @@ def check_cmake_in_path(): # CMake command not found return False, None + if os.name == 'nt': # Check if the OS is Windows # Check if CMake is in the system path cmake_found, cmake_version = check_cmake_in_path() @@ -96,11 +95,6 @@ def check_cmake_in_path(): raise SystemError("CMake is not found in the system path. Make sure CMake is installed and in the system path.") - - - - - # Workaround for `distutils.spawn` problem on Windows python < 3.9 # See details: [bpo-39763: distutils.spawn now uses subprocess (GH-18743)] # (https://github.com/python/cpython/commit/1ec63b62035e73111e204a0e03b83503e1c58f2e) From c5e10f4a540181fb76698d37ef9c2bb025a78348 Mon Sep 17 00:00:00 2001 From: YCHuang2112 <96855801+YCHuang2112@users.noreply.github.com> Date: Mon, 21 Aug 2023 10:55:46 +0800 Subject: [PATCH 42/44] fix lint error --- setup.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 147fa69..d430eac 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,12 @@ def build_extensions(self): def check_cmake_in_path(): try: - result = subprocess.run(['cmake', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run( + ["cmake", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) if result.returncode == 0: # CMake is in the system path return True, result.stdout.strip() @@ -85,14 +90,22 @@ def check_cmake_in_path(): return False, None -if os.name == 'nt': # Check if the OS is Windows +if os.name == "nt": # Check if the OS is Windows # Check if CMake is in the system path cmake_found, cmake_version = check_cmake_in_path() if cmake_found: - print(f"CMake is in the system path. Version: {cmake_version}") + print( + f"CMake is in the system path. Version: \ + {cmake_version}" + ) else: - raise SystemError("CMake is not found in the system path. Make sure CMake is installed and in the system path.") + raise SystemError( + "CMake is not found in the \ + system path. Make sure CMake \ + is installed and in the system \ + path." + ) # Workaround for `distutils.spawn` problem on Windows python < 3.9 From 96f3cf3628bd900acac42d1ecf65bacd9a808cea Mon Sep 17 00:00:00 2001 From: YCHuang2112 <96855801+YCHuang2112@users.noreply.github.com> Date: Mon, 21 Aug 2023 13:00:08 +0800 Subject: [PATCH 43/44] remove vscode setting file --- .vscode/settings.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index e137fad..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true -} \ No newline at end of file From e6bc65fe73439ed646ea114136b5f0ff85c8d1e0 Mon Sep 17 00:00:00 2001 From: Ryuichi Yamamoto Date: Tue, 22 Aug 2023 10:16:37 +0900 Subject: [PATCH 44/44] prepare for v0.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d430eac..9563649 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ platform_is_windows = sys.platform == "win32" -version = "0.3.2-dev" +version = "0.3.2" min_cython_ver = "0.21.0" try: