From d267b2b8ff5dfe9073d2df9013a1691ff8c74978 Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 14:41:12 +0900 Subject: [PATCH 01/11] :art: lessen dependencies --- pqkmeans/evaluation.py | 5 ++++- requirements.txt | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pqkmeans/evaluation.py b/pqkmeans/evaluation.py index e91d7a7..c3b84f6 100644 --- a/pqkmeans/evaluation.py +++ b/pqkmeans/evaluation.py @@ -3,7 +3,10 @@ import os import six.moves.urllib import tarfile -import texmex_python +try: + import texmex_python +except ImportError: + pass def get_gmm_random_dataset(k, dimension=100, test_size=5000, train_size=500): diff --git a/requirements.txt b/requirements.txt index 29656d8..53751e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -numpy +numpy<1.21 scikit-learn -pipe +pipe<2.0 scipy six -texmex_python==1.0.0 From 8a3cbfe1c90b85bc398d23576b7385949bae7db7 Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 14:44:54 +0900 Subject: [PATCH 02/11] :fire: remove py3.5 tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edd2b0c..1fc0cfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 3 matrix: platform: [ubuntu-latest] - python-version: [3.5, 3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 From aa95a04613a7ac824c985127c2fa7ea33b15c577 Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 15:16:40 +0900 Subject: [PATCH 03/11] texmex-python as extra package --- .github/workflows/test.yml | 2 +- README.md | 2 ++ pqkmeans/evaluation.py | 8 ++++---- setup.py | 3 +++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1fc0cfd..38badc1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 3 matrix: platform: [ubuntu-latest] - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index fbc51a4..b922c3d 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ python setup.py install ``` +# evaluation needs extra texmex package +pip install pqkmeans[texmex] # with artificial data python bin/run_experiment.py --dataset artificial --algorithm bkmeans pqkmeans --k 100 # with texmex dataset (http://corpus-texmex.irisa.fr/) diff --git a/pqkmeans/evaluation.py b/pqkmeans/evaluation.py index c3b84f6..1c573e9 100644 --- a/pqkmeans/evaluation.py +++ b/pqkmeans/evaluation.py @@ -3,10 +3,6 @@ import os import six.moves.urllib import tarfile -try: - import texmex_python -except ImportError: - pass def get_gmm_random_dataset(k, dimension=100, test_size=5000, train_size=500): @@ -42,6 +38,10 @@ def get_sift1m_dataset(cache_directory="."): def get_texmex_dataset(url, filename, member_names, cache_directory="."): + try: + import texmex_python + except ImportError: + raise ImportError("Missing optional dependency 'texmex_python'. You must install it to use this dataset.") path = os.path.join(cache_directory, filename) if not os.path.exists(path): print("downloading {}".format(url)) diff --git a/setup.py b/setup.py index 953ed25..97b7ca4 100644 --- a/setup.py +++ b/setup.py @@ -92,4 +92,7 @@ def build_extension(self, ext): cmdclass=dict(build_ext=CMakeBuild), test_suite='test', zip_safe=False, + extras_require={ + "texmex": ["texmex-python>=1.0.0"], + }, ) From ebaceca49dd59b34017c2f0090f560b792e07fe2 Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 15:26:39 +0900 Subject: [PATCH 04/11] cannot installed with python3.10 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38badc1..6146d03 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 3 matrix: platform: [ubuntu-latest] - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9"] steps: - uses: actions/checkout@v2 From ac2960d06b73e890d824729a52230bd616bd92ef Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 16:26:17 +0900 Subject: [PATCH 05/11] :star: fix to be able to use recent numpy --- pqkmeans/encoder/pq_encoder.py | 6 +++--- requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pqkmeans/encoder/pq_encoder.py b/pqkmeans/encoder/pq_encoder.py index 513b9fb..9ac922a 100644 --- a/pqkmeans/encoder/pq_encoder.py +++ b/pqkmeans/encoder/pq_encoder.py @@ -22,9 +22,9 @@ def fit(self, x_train): self.Ds = int(D / self.M) assert self.trained_encoder is None, "fit must be called only once" - codewords = numpy.zeros((self.M, self.Ks, self.Ds), dtype=numpy.float) + codewords = numpy.zeros((self.M, self.Ks, self.Ds), dtype=float) for m in range(self.M): - x_train_sub = x_train[:, m * self.Ds: (m + 1) * self.Ds].astype(numpy.float) + x_train_sub = x_train[:, m * self.Ds: (m + 1) * self.Ds].astype(float) codewords[m], _ = kmeans2(x_train_sub, self.Ks, iter=self.iteration, minit='points') self.trained_encoder = TrainedPQEncoder(codewords, self.code_dtype) @@ -66,7 +66,7 @@ def decode_multi(self, codes): assert M == self.M assert codes.dtype == self.code_dtype - decoded = numpy.empty((N, self.Ds * self.M), dtype=numpy.float) + decoded = numpy.empty((N, self.Ds * self.M), dtype=float) for m in range(self.M): decoded[:, m * self.Ds: (m + 1) * self.Ds] = self.codewords[m][codes[:, m], :] return decoded diff --git a/requirements.txt b/requirements.txt index 53751e4..8c90c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy<1.21 +numpy scikit-learn pipe<2.0 scipy From 08bd4e1dc38525a9f8206432aeb1c6ba570ca518 Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 16:26:54 +0900 Subject: [PATCH 06/11] :star: test with recent python --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6146d03..87808d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 3 matrix: platform: [ubuntu-latest] - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 From 501db9f104c5bb2c3ae8e68538ffc4531ec46c57 Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 16:34:55 +0900 Subject: [PATCH 07/11] 3.11 is not yet supported --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87808d5..b1de936 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 3 matrix: platform: [ubuntu-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10"] # 3.11 is not yet supported steps: - uses: actions/checkout@v2 From f6e97fb2a439b399fbbce60dd700a546c6f0c28f Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 16:38:10 +0900 Subject: [PATCH 08/11] :star: update pybind --- ext/pybind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/pybind b/ext/pybind index a23996f..0e2c3e5 160000 --- a/ext/pybind +++ b/ext/pybind @@ -1 +1 @@ -Subproject commit a23996fce38ff6ccfbcdc09f1e63f2c4be5ea2ef +Subproject commit 0e2c3e5db41b6b2af4038734c84ab855ccaaa5f0 From a1d16b6ecf2fceb70d19a237cda9f2d8964ae06f Mon Sep 17 00:00:00 2001 From: Keisuke Ogaki Date: Sat, 14 Oct 2023 16:43:29 +0900 Subject: [PATCH 09/11] :white_check_mark: test with python 3.11 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1de936..87808d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 3 matrix: platform: [ubuntu-latest] - python-version: ["3.7", "3.8", "3.9", "3.10"] # 3.11 is not yet supported + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 From b6905972d4ba46ee68fdc61d3202d5505481292f Mon Sep 17 00:00:00 2001 From: Keisuke OGAKI Date: Mon, 16 Oct 2023 01:23:17 +0000 Subject: [PATCH 10/11] fails-fast = false --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87808d5..bfe6e45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,7 @@ jobs: tests: runs-on: ${{ matrix.platform }} strategy: + fail-fast: false # would like to see which version fails max-parallel: 3 matrix: platform: [ubuntu-latest] From 06f44871066037f323c1323610508b0d1fe23740 Mon Sep 17 00:00:00 2001 From: Keisuke OGAKI Date: Mon, 16 Oct 2023 01:26:41 +0000 Subject: [PATCH 11/11] numpy compatibility --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53751e4..8c90c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy<1.21 +numpy scikit-learn pipe<2.0 scipy