From 25e21e888af1bfc377872d943fef05e47b4ef7d2 Mon Sep 17 00:00:00 2001 From: JoeyHwong Date: Thu, 2 Sep 2021 21:16:53 +0800 Subject: [PATCH 1/4] Fix reference error - fix setup.py error - fix service import error Signed-off-by: JoeyHwong Signed-off-by: joeyhwong Signed-off-by: JoeyHwong --- lib/MANIFEST.in | 2 +- lib/sedna/service/server/base.py | 1 - lib/setup.py | 21 +++++++++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/MANIFEST.in b/lib/MANIFEST.in index 41fd45496..c3df2fe30 100644 --- a/lib/MANIFEST.in +++ b/lib/MANIFEST.in @@ -1,2 +1,2 @@ -include OWNERS requirements.txt sedna/VERSION README.md +include OWNERS requirements.txt sedna/VERSION sedna/README.md recursive-include examples *.txt *.py diff --git a/lib/sedna/service/server/base.py b/lib/sedna/service/server/base.py index d5302e316..4ab39ac47 100644 --- a/lib/sedna/service/server/base.py +++ b/lib/sedna/service/server/base.py @@ -15,7 +15,6 @@ import contextlib import time import threading -import asyncio import uvicorn from fastapi.middleware.cors import CORSMiddleware diff --git a/lib/setup.py b/lib/setup.py index 2cf986363..492286332 100644 --- a/lib/setup.py +++ b/lib/setup.py @@ -19,7 +19,8 @@ assert sys.version_info >= (3, 6), "Sorry, Python < 3.6 is not supported." -with open("README.md", "r") as fh: +with open(os.path.join(os.path.dirname(__file__), "sedna", "README.md"), + "r", encoding="utf-8") as fh: long_desc = fh.read() with open(os.path.join(os.path.dirname(__file__), 'sedna', 'VERSION'), @@ -30,6 +31,18 @@ install_requires = [line.strip() for line in fh.readlines() if line.strip()] +with open("OWNERS", "r", encoding="utf-8") as fh: + check, approvers = False, set() + for line in fh: + if not line.strip(): + continue + if check: + approvers.add(line.strip().split()[-1]) + check = (line.startswith("approvers:") or + (line.startswith(" -") and check)) + +maintainer = ",".join(approvers) or "sedna" + setup( name='sedna', version=__version__, @@ -38,9 +51,9 @@ on Sedna project", packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]), - author="", - author_email="", - maintainer="", + author=maintainer, + author_email="pujie2@huawei.com", + maintainer=maintainer, maintainer_email="", include_package_data=True, python_requires=">=3.6", From 8aa2c9261dd9950c42f75462d2a57e0843ccc9ef Mon Sep 17 00:00:00 2001 From: JoeyHwong Date: Mon, 13 Sep 2021 17:00:16 +0800 Subject: [PATCH 2/4] fix extras require in setup Signed-off-by: JoeyHwong --- lib/MANIFEST.in | 2 +- lib/requirements.dev.txt | 4 ++ lib/sedna/__version__.py | 4 +- lib/setup.py | 117 +++++++++++++++++++++++++++++---------- 4 files changed, 96 insertions(+), 31 deletions(-) create mode 100644 lib/requirements.dev.txt diff --git a/lib/MANIFEST.in b/lib/MANIFEST.in index c3df2fe30..64d59c633 100644 --- a/lib/MANIFEST.in +++ b/lib/MANIFEST.in @@ -1,2 +1,2 @@ -include OWNERS requirements.txt sedna/VERSION sedna/README.md +include OWNERS requirements.txt requirements.dev.txt sedna/VERSION sedna/README.md recursive-include examples *.txt *.py diff --git a/lib/requirements.dev.txt b/lib/requirements.dev.txt new file mode 100644 index 000000000..aa353baed --- /dev/null +++ b/lib/requirements.dev.txt @@ -0,0 +1,4 @@ +# federated_learning +plato-learn~=0.26 # Apache-2.0 +# lifelong_learning +scikit-learn~=0.24.1 # BSD diff --git a/lib/sedna/__version__.py b/lib/sedna/__version__.py index 9ea7aeb11..f1bcb07d6 100644 --- a/lib/sedna/__version__.py +++ b/lib/sedna/__version__.py @@ -14,4 +14,6 @@ """sedna version information.""" -__version__ = '0.3.1' +with open("./VERSION", "r", encoding="utf-8") as fin: + tmp = [line.strip() for line in fin if line.strip()] + __version__ = "-".join(tmp) if tmp else "dev" diff --git a/lib/setup.py b/lib/setup.py index 492286332..7c70e8c70 100644 --- a/lib/setup.py +++ b/lib/setup.py @@ -19,45 +19,105 @@ assert sys.version_info >= (3, 6), "Sorry, Python < 3.6 is not supported." -with open(os.path.join(os.path.dirname(__file__), "sedna", "README.md"), - "r", encoding="utf-8") as fh: - long_desc = fh.read() -with open(os.path.join(os.path.dirname(__file__), 'sedna', 'VERSION'), - "r", encoding="utf-8") as fh: - __version__ = fh.read().strip() +class InstallPrepare: + """ + Parsing dependencies + """ -with open("requirements.txt", "r", encoding="utf-8") as fh: - install_requires = [line.strip() for line in - fh.readlines() if line.strip()] + def __init__(self): + self.project = os.path.join(os.path.dirname(__file__), "sedna") + self._long_desc = os.path.join(self.project, "README.md") + self._version = os.path.join(self.project, "VERSION") + self._owner = os.path.join(self.project, "..", "OWNERS") + self._requirements = os.path.join(self.project, "..", + "requirements.txt") + self._dev_requirements = os.path.join(self.project, "..", + "requirements.dev.txt") -with open("OWNERS", "r", encoding="utf-8") as fh: - check, approvers = False, set() - for line in fh: - if not line.strip(): - continue - if check: - approvers.add(line.strip().split()[-1]) - check = (line.startswith("approvers:") or - (line.startswith(" -") and check)) + @property + def long_desc(self): + if not os.path.isfile(self._long_desc): + return "" + with open(self._long_desc, "r", encoding="utf-8") as fh: + long_desc = fh.read() + return long_desc -maintainer = ",".join(approvers) or "sedna" + @property + def version(self): + default_version = "999.dev" + if not os.path.isfile(self._version): + return default_version + with open(self._version, "r", encoding="utf-8") as fh: + __version__ = fh.read().strip() + return __version__ or default_version + + @property + def owners(self): + default_owner = "sedna" + if not os.path.isfile(self._owner): + return default_owner + with open(self._owner, "r", encoding="utf-8") as fh: + check, approvers = False, set() + for line in fh: + if not line.strip(): + continue + if check: + approvers.add(line.strip().split()[-1]) + check = (line.startswith("approvers:") or + (line.startswith(" -") and check)) + return ",".join(approvers) or default_owner + + @property + def basic_dependencies(self): + return self._read_requirements(self._requirements) + + def feature_dependencies(self, feature): + _c = os.path.join(self.project, 'core', feature, "requirements.txt") + if os.path.isfile(_c): + return self._read_requirements(_c) + return self._read_requirements(self._dev_requirements, feature) + + @staticmethod + def _read_requirements(file_path, section="all"): + print(f"Start to install requirements of {section} " + f"in sedna from {file_path}") + if not os.path.isfile(file_path): + return [] + with open(file_path, "r", encoding="utf-8") as f: + install_requires = [p.strip() for p in f.readlines() if p.strip()] + if section == "all": + return list(filter(lambda x: not x.startswith("#"), + install_requires)) + section_start = False + section_requires = [] + for p in install_requires: + if section_start: + if p.startswith("#"): + return section_requires + section_requires.append(p) + elif p.startswith(f"# {section}"): + section_start = True + return section_requires + + +_infos = InstallPrepare() setup( name='sedna', - version=__version__, + version=_infos.version, description="The sedna package is designed to help developers \ better use open source frameworks such as tensorflow \ on Sedna project", packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]), - author=maintainer, + author=_infos.owners, author_email="pujie2@huawei.com", - maintainer=maintainer, + maintainer=_infos.owners, maintainer_email="", include_package_data=True, python_requires=">=3.6", - long_description=long_desc, + long_description=_infos.long_desc, long_description_content_type="text/markdown", license="Apache License 2.0", url="https://github.com/kubeedge/sedna", @@ -66,12 +126,11 @@ "License :: OSI Approved :: Apache Software License", "Operating System :: POSIX :: Linux", ], - install_requires=install_requires, + install_requires=_infos.basic_dependencies, extras_require={ - "tf": ["tensorflow>=1.0.0,<2.0"], - "tf_gpu": ["tensorflow-gpu>=1.0.0,<2.0"], - "pytorch": ["torch==0.4.0", "torchvision==0.2.1"], - "ms": ["mindspore==1.1.1"], - "sklearn": ["pandas>=0.25.0", "scikit-learn==0.24.1"] + "fl": _infos.feature_dependencies("federated_learning"), + "il": _infos.feature_dependencies("incremental_learning"), + "ji": _infos.feature_dependencies("joint_inference"), + "ll": _infos.feature_dependencies("lifelong_learning") }, ) From 5a7be01e11fd776907057b3d87213a8743ad0c9f Mon Sep 17 00:00:00 2001 From: JoeyHwong Date: Wed, 13 Oct 2021 10:35:46 +0800 Subject: [PATCH 3/4] fix PR commnet Signed-off-by: JoeyHwong --- lib/requirements.txt | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/requirements.txt b/lib/requirements.txt index 35ca78fa2..f595ce212 100644 --- a/lib/requirements.txt +++ b/lib/requirements.txt @@ -1,15 +1,14 @@ -numpy>=1.13.3 # BSD -colorlog~=4.7.2 # MIT -websockets~=9.1 # BSD -requests==2.24.0 # Apache-2.0 -PyYAML~=5.4.1 # MIT -setuptools~=54.2.0 -fastapi~=0.63.0 # MIT -starlette~=0.13.6 # BSD -pydantic~=1.8.1 # MIT -tenacity~=8.0.1 # Apache-2.0 -joblib~=1.0.1 # BSD -pandas~=1.1.5 # BSD -six~=1.15.0 # MIT -minio~=7.0.3 # Apache-2.0 -uvicorn~=0.14.0 # BSD +numpy>=1.13.3 # BSD +colorlog~=4.7.2 # MIT +websockets~=9.1 # BSD +requests==2.24.0 # Apache-2.0 +PyYAML~=5.4.1 # MIT +setuptools~=54.2.0 +fastapi~=0.63.0 # MIT +pydantic>=1.8.1 # MIT +tenacity~=8.0.1 # Apache-2.0 +joblib~=1.0.1 # BSD +pandas~=1.1.5 # BSD +six~=1.15.0 # MIT +minio~=7.0.3 # Apache-2.0 +uvicorn>=0.14.0 # BSD From 14b22ece3127bde8ee49340c2db5801fa24e0190 Mon Sep 17 00:00:00 2001 From: JoeyHwong Date: Wed, 13 Oct 2021 11:02:00 +0800 Subject: [PATCH 4/4] fix incompatible in requirement Signed-off-by: JoeyHwong --- lib/requirements.txt | 2 +- lib/sedna/README.md | 4 ++++ lib/sedna/__version__.py | 6 +++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/requirements.txt b/lib/requirements.txt index 8ca443ac8..c3ec5572c 100644 --- a/lib/requirements.txt +++ b/lib/requirements.txt @@ -1,7 +1,7 @@ numpy>=1.13.3 # BSD colorlog~=4.7.2 # MIT websockets~=9.1 # BSD -requests==2.24.0 # Apache-2.0 +requests>=2.24.0 # Apache-2.0 PyYAML~=5.4.1 # MIT setuptools~=54.2.0 fastapi~=0.68.1 # MIT diff --git a/lib/sedna/README.md b/lib/sedna/README.md index e860d8e41..7174059f5 100644 --- a/lib/sedna/README.md +++ b/lib/sedna/README.md @@ -29,6 +29,10 @@ pip install dist/sedna*.whl Install via Setuptools ```bash +# Install dependence +pip install -r requirements.txt + +# Install sedna python setup.py install --user ``` diff --git a/lib/sedna/__version__.py b/lib/sedna/__version__.py index f1bcb07d6..5e0fae371 100644 --- a/lib/sedna/__version__.py +++ b/lib/sedna/__version__.py @@ -13,7 +13,11 @@ # limitations under the License. """sedna version information.""" +import os -with open("./VERSION", "r", encoding="utf-8") as fin: + +_VERSION = os.path.join(os.path.dirname(__file__), "VERSION") + +with open(_VERSION, "r", encoding="utf-8") as fin: tmp = [line.strip() for line in fin if line.strip()] __version__ = "-".join(tmp) if tmp else "dev"