From 88b2f572c9ff2a20da68f81e1f588564b1846879 Mon Sep 17 00:00:00 2001 From: LynnL4 Date: Wed, 10 May 2023 09:47:41 +0800 Subject: [PATCH] release 2.0.0rc1 --- .github/workflows/build-docs.yml | 3 +- edgelab/version.py | 2 +- tools/env_config.py | 438 ------------------------------- tools/ubuntu_utils.py | 253 ------------------ 4 files changed, 2 insertions(+), 694 deletions(-) delete mode 100644 tools/env_config.py delete mode 100644 tools/ubuntu_utils.py diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index cf72df24..c639c86d 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - lab2.0 paths: - 'docs/**' - 'package.json' @@ -23,7 +22,7 @@ jobs: - name: checkout repository uses: actions/checkout@v2 with: - ref: 'lab2.0' # should be main later + ref: 'main' # should be main later - name: setup node uses: actions/setup-node@v3 diff --git a/edgelab/version.py b/edgelab/version.py index 2495c9b0..2745548e 100644 --- a/edgelab/version.py +++ b/edgelab/version.py @@ -1,4 +1,4 @@ -__version__ = '0.0.1rc0' +__version__ = '2.0.0rc1' short_version = __version__ diff --git a/tools/env_config.py b/tools/env_config.py deleted file mode 100644 index a50f534d..00000000 --- a/tools/env_config.py +++ /dev/null @@ -1,438 +0,0 @@ -import os -import re -import time -import logging -import argparse -import subprocess -import os.path as osp - -from subprocess import Popen, PIPE -from .ubuntu_utils import cmd_result, ensure_base_env - -download_links = { - 'miniconda': - 'https://repo.anaconda.com/miniconda/Miniconda3-py38_4.12.0-Linux-x86_64.sh', - 'anaconda': - 'https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh' -} - - -def log_init(): - loger = logging.getLogger('ENV_CONFIG') - loger.setLevel(logging.INFO) - - hd = logging.StreamHandler() - hd.setLevel(logging.INFO) - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - hd.setFormatter(formatter) - loger.addHandler(hd) - return loger - - -def command(cmd, retry_num=3): - for i in range(retry_num): - if os.system(cmd) != 0: - time.sleep(1) - loger.warning(f'COMMAND:"{cmd}"execution failed') - loger.info('retrying') if i != (retry_num - 1) else None - else: - return True - return False - - -def test_delay(name): - p = Popen(f'ping {name} -c 3', shell=True, encoding='utf8', stdout=PIPE) - loger.info(f'testing to "{name}" delay') - data = p.stdout.read() - p.wait(5) - if len(data): - delay = re.findall('time=(.*?) ms', data) - return sum([float(i) for i in delay]) / len(delay).__round__(2) if len( - delay) else 1000 - - -def qure_ip(): - p = subprocess.Popen('curl http://myip.ipip.net', - shell=True, - encoding='utf8', - stdout=subprocess.PIPE) - data = p.stdout.read() - loger.info(data) - if '中国' in data: - if any(i in data for i in ['台湾', '香港', '澳门']): return False - else: return True - return False - - -def test_network(): - delay = 1000 - domain = 'pypi.tuna.tsinghua.edu.cn' - if qure_ip(): - mirror = { - 'pypi.mirrors.ustc.edu.cn': - 'https://pypi.mirrors.ustc.edu.cn/simple', - 'pypi.douban.com': 'http://pypi.douban.com/simple/', - 'mirrors.aliyun.com': 'https://mirrors.aliyun.com/pypi/simple/', - 'pypi.tuna.tsinghua.edu.cn': - 'https://pypi.tuna.tsinghua.edu.cn/simple' - } - for i in mirror.keys(): - test = test_delay(i) - if delay > test: - delay = test - domain = i - loger.info(mirror[domain]) - return mirror[domain], domain - - return False - - -def qure_gpu(): - p = subprocess.Popen(args='lspci |grep -i nvidia', - stdout=subprocess.PIPE, - encoding='utf8', - shell=True) - data = p.stdout.read() - if 'NVIDIA' in data.upper(): - return True - return False - - -def download_file(link, path): - os.chdir(path) - if osp.exists(osp.join(path, link.split('/')[-1])): - loger.info( - f"{link.split('/')[-1]} already exists under the {path} path") - return - if command(f'wget {link}'): - loger.info(f"{link.split('/')[-1]} Downloaded") - else: - loger.warning(f"{link.split('/')[-1]} download failed") - - -def anaconda_install(conda='miniconda'): - os.chdir(proce_path) - if command(f'{conda_bin} -V', 1): - loger.info( - 'Your conda has been installed, skip the installation this time') - return - - file_name = download_links[conda].split('/')[-1] - if not osp.exists(file_name): - download_file(download_links[conda], proce_path) - - time.sleep(1) - command(f'chmod +x {file_name}') - command(f'./{file_name} -b') - - # r = subprocess.Popen(args=f'./{file_name}', - # stdin=PIPE, - # stderr=PIPE, - # stdout=None, - # shell=True, - # encoding='utf8') - # os.popen('ls').writelines(['\n','q','yes\n']) - # try: - # r.communicate('\nq\n\nyes\n\nyes\n\n') - # # r.wait(15) - # except: - # r.kill() - # loger.info('anaconda installation failed!') - - -def cuda_install(): - os.chdir(proce_path) - if not osp.exists('./cuda_11.2.1_460.32.03_linux.run'): - command( - 'wget https://developer.download.nvidia.com/compute/cuda/11.2.1/local_installers/cuda_11.2.1_460.32.03_linux.run' - ) - - command( - 'wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin' - ) - command( - 'sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600' - ) - command( - 'wget https://developer.download.nvidia.com/compute/cuda/11.2.1/local_installers/cuda-repo-ubuntu2004-11-2-local_11.2.1-460.32.03-1_amd64.deb' - ) - command( - 'sudo dpkg -i cuda-repo-ubuntu2004-11-2-local_11.2.1-460.32.03-1_amd64.deb' - ) - command( - 'sudo apt-key add /var/cuda-repo-ubuntu2004-11-2-local/7fa2af80.pub') - command('sudo apt-get update && sudo apt-get -y install cuda') - - -def conda_create_env(name, version=3.8): - command(f"{conda_bin} init") - if command(f"{pip} -V", 1): - loger.info(f'The virtual environment {name} already exists') - return - - p = subprocess.Popen(f'{conda_bin} create -n {name} python=={version}', - stdin=subprocess.PIPE, - shell=True, - encoding='utf8') - p.communicate('y\n') - - p = subprocess.Popen(f'{conda_bin} info -e', - stdout=subprocess.PIPE, - shell=True, - encoding='utf8') - if name in p.stdout.read(): - loger.info(f'The virtual environment {name} has been created') - else: - loger.warning(f'Failed to create virtual environment {name}') - - -def conda_acti_env(name): - command(f'{conda_bin} info -e') - time.sleep(2) - if command(f'{conda_bin} activate {name}'): - loger.info(f'The virtual environment {name} is activated') - else: - loger.warning(f'Virtual environment {name} activation failed') - - -def torch_install(): - p = subprocess.Popen(args=f'{pip} list | grep torch', - shell=True, - encoding='utf8', - stdout=subprocess.PIPE, - stdin=subprocess.PIPE) - data = p.stdout.read() - if len(data) and 'torch ' in data: - loger.warning('Torch is has installed!') - return - - if GPU: - p = subprocess.Popen(args='nvidia-smi | grep Driver', - stdout=subprocess.PIPE, - encoding='utf8', - shell=True) - data = p.stdout.read() - if len(data) and 'Driver' in data: - version = re.findall('CUDA Version: (.*?) ', data)[0] - loger.info(f'CUDA VERSION:{version}') - else: - loger.warning('cuda version not found！') - command( - f'{pip} install torch==1.10.0+cu113 torchvision==0.11.1+cu113 torchaudio==0.10.0+cu113 --extra-index-url https://download.pytorch.org/whl/cu113' - + pip_mirror) - else: - command( - f'{pip} install torch==1.10.0 torchvision==0.11.1 torchaudio==0.10.0 ' - + pip_mirror) - - -def mmlab_install(): - req_path = osp.join(project_path, 'requirements') - if command(f'{pip} install -r {req_path}/requirements.txt' + pip_mirror): - loger.info('mmlab env install succeeded!') - if GPU: - if command( - f'{pip} install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html' - ): - loger.info('mmcv-full install succeeded!') - else: - command( - f'{pip} install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.10.0/index.html' - + pip_mirror) - loger.info('mmcv-full install succeeded!') - - -def install_protobuf() -> int: - print('-' * 10 + 'install protobuf' + '-' * 10) - - os.chdir(proce_path) - if not osp.exists('protobuf-cpp-3.20.0.tar.gz'): - command( - 'wget https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz' # noqa: E501 - ) - command('tar xvf protobuf-cpp-3.20.0.tar.gz') - - os.chdir(osp.join(proce_path, 'protobuf-3.20.0')) - - install_dir = osp.join(proce_path, 'pbinstall') - if osp.exists(install_dir): - command('rm -rf {}'.format(install_dir)) - else: - os.makedirs(install_dir, exist_ok=True) - - command('make clean') - command('./configure --prefix={}'.format(install_dir)) - command('make -j {} && make install'.format(g_jobs)) - protoc = osp.join(install_dir, 'bin', 'protoc') - - print('protoc \t:{}'.format(cmd_result('{} --version'.format(protoc)))) - - command(""" echo 'export PATH={}:$PATH' >> ~/mmdeploy.env """.format( - osp.join(install_dir, 'bin'))) - command( - """ echo 'export LD_LIBRARY_PATH={}:$LD_LIBRARY_PATH' >> ~/mmdeploy.env """ # noqa: E501 - .format(osp.join(install_dir, 'lib'))) - - return 0 - - -def install_pyncnn(): - print('-' * 10 + 'build and install pyncnn' + '-' * 10) - time.sleep(2) - - # generate unzip and build dir - os.chdir(proce_path) - - # git clone - if not osp.exists('ncnn'): - command( - 'git clone --depth 1 --branch 20220729 https://github.com/tencent/ncnn && cd ncnn' # noqa: E501 - ) - - ncnn_dir = osp.join(proce_path, 'ncnn') - os.chdir(ncnn_dir) - - # update submodule pybind11, gslang not required - command('git submodule init && git submodule update python/pybind11') - # build - if not osp.exists('build'): - command('mkdir build') - - os.chdir(osp.join(ncnn_dir, 'build')) - command('rm -rf CMakeCache.txt') - pb_install = osp.join(proce_path, 'pbinstall') - pb_bin = osp.join(pb_install, 'bin', 'protoc') - pb_lib = osp.join(pb_install, 'lib', 'libprotobuf.so') - pb_include = osp.join(pb_install, 'include') - - cmd = 'cmake .. ' - cmd += ' -DNCNN_PYTHON=ON ' - cmd += ' -DProtobuf_LIBRARIES={} '.format(pb_lib) - cmd += ' -DProtobuf_PROTOC_EXECUTABLE={} '.format(pb_bin) - cmd += ' -DProtobuf_INCLUDE_DIR={} '.format(pb_include) - cmd += ' && make -j {} '.format(g_jobs) - cmd += ' && make install ' - command(cmd) - - # install - os.chdir(project_path) - command(f'{pip} install ncnn' + pip_mirror) - - path_ls = [] - path_ls.append(osp.join(ncnn_dir, 'build', 'tools', 'onnx')) - path_ls.append(osp.join(ncnn_dir, 'build', 'tools', 'quantize')) - path_ls.append(osp.join(ncnn_dir, 'build', 'tools', 'caffe')) - path_ls.append(osp.join(ncnn_dir, 'build', 'install', 'bin')) - path_ls.append(osp.join(ncnn_dir, 'build', 'install', 'include')) - path_ls.append(osp.join(ncnn_dir, 'build', 'install', 'lib')) - PATH = os.environ['PATH'] - for p in path_ls: - if p in PATH: continue - else: - command(f'echo export PATH={p}:\$PATH >> ~/.bashrc') if osp.exists( - p) else None - - -def proto_ncnn_install(): - """refer:https://github.com/open-mmlab/mmdeploy/blob/master/tools/scripts/build_ubuntu_x64_ncnn.py - Returns: - _type_: _description_ - """ - if success != 0: - return -1 - - if install_protobuf() != 0: - return -1 - - install_pyncnn() - - -def check_env(): - check_list = {} - ncnn_dir = osp.join(proce_path, 'ncnn') - ncnn = osp.join(ncnn_dir, 'build', 'install', 'bin', 'onnx2ncnn') - check_list['anaconda'] = 'OK' if command(f"{conda_bin} -V", 1) else 'faile' - check_list["virtual env"] = 'OK' if command(f"{pip} -V", 1) else 'faile' - check_list['python ncnn'] = 'OK' if command( - f"{python_bin} -c 'import ncnn'", 1) else 'faile' - - check_list['ncnn'] = 'OK' if osp.exists(ncnn) else 'faile' - - check_list['torch'] = 'OK' if command(f"{python_bin} -c 'import torch'", - 1) else 'faile' - check_list['torchvision'] = 'OK' if command( - f"{python_bin} -c 'import torchvision'", 1) else 'faile' - check_list['torchaudio'] = 'OK' if command( - f"{python_bin} -c 'import torchaudio'", 1) else 'faile' - - check_list['mmcv'] = 'OK' if command(f"{python_bin} -c 'import mmcv'", - 1) else 'faile' - check_list['mmdet'] = 'OK' if command(f"{python_bin} -c 'import mmdet'", - 1) else 'faile' - check_list['mmcls'] = 'OK' if command(f"{python_bin} -c 'import mmcls'", - 1) else 'faile' - check_list['mmpose'] = 'OK' if command(f"{python_bin} -c 'import mmpose'", - 1) else 'faile' - - for key, value in check_list.items(): - print(f"{key:30s}:{value:10s}") - - -def pare_args(): - args = argparse.ArgumentParser() - args.add_argument('--envname', - default='edgelab', - help='conda vertual enverimen name') - args.add_argument('--conda', - default='anaconda', - help='conda vertual enverimen name') - return args.parse_args() - - -def prepare(): - global args, project_path, pip, python_bin, conda_bin, home, proce_path, loger, pip_mirror, GPU, g_jobs, success - args = pare_args() - g_jobs = os.cpu_count() if os.cpu_count else 8 - project_path = osp.dirname(osp.dirname(osp.abspath(__file__))) - pip = f'~/{args.conda}3/envs/{args.envname}/bin/pip' - home = os.environ['HOME'] - python_bin = f'~/{args.conda}3/envs/{args.envname}/bin/python' - - conda_bin = f'{home}/{args.conda}3/bin/conda' - proce_path = f'{home}/software' - try: - PATH = os.environ['PYTHONPATH'] - except: - PATH = '' - finally: - if proce_path not in PATH: - command(f'echo export PYTHONPATH={home}:\$PYTHONPATH >> ~/.bashrc') - - success = ensure_base_env(project_path, proce_path) - os.makedirs(proce_path, exist_ok=True) - loger = log_init() - mirror = test_network() - pip_mirror = f' -i {mirror[0]} --trusted-host {mirror[1]}' if mirror else '' - GPU = qure_gpu() - - -def main(): - prepare() - - anaconda_install(conda=args.conda) - conda_create_env(args.envname) - - # cuda_install() - torch_install() - # mmlab - mmlab_install() - # export - proto_ncnn_install() - - check_env() - # command('source ~/.bashrc') - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/tools/ubuntu_utils.py b/tools/ubuntu_utils.py deleted file mode 100644 index d69b344c..00000000 --- a/tools/ubuntu_utils.py +++ /dev/null @@ -1,253 +0,0 @@ -import os -import re -import time - - -def pytorch_version(): - version = None - try: - import torch - raw = torch.__version__ - pattern = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+') - version = pattern.findall(raw)[0] - except Exception: - pass - return version - - -def cmd_result(txt: str): - cmd = os.popen(txt) - return cmd.read().rstrip().lstrip() - - -def get_job(argv) -> int: - # get nprocs, if user not specified, use max(1, nproc-2) - job = 2 - if len(argv) <= 1: - print('your can use `python3 {} N` to set make -j [N]'.format(argv[0])) - nproc = cmd_result('nproc') - if nproc is not None and len(nproc) > 0: - job = max(int(nproc) - 2, 1) - else: - job = 1 - else: - job = int(argv[1]) - return job - - -def version_major(txt: str) -> int: - return int(txt.split('.')[0]) - - -def version_minor(txt: str) -> int: - return int(txt.split('.')[1]) - - -def cu_version_name(version: str) -> str: - versions = version.split('.') - return 'cu' + versions[0] + versions[1] - - -def simple_check_install(bin: str, sudo: str) -> str: - result = cmd_result('which {}'.format(bin)) - if result is None or len(result) < 1: - print('{} not found, try install {} ..'.format(bin, bin), end='') - os.system('{} apt install {} -y'.format(sudo, bin)) - result = cmd_result('which {}'.format(bin)) - if result is None or len(result) < 1: - print('Check {} failed.'.format(bin)) - return None - print('success') - return result - - -def ensure_base_env(work_dir, dep_dir): - description = """ - check python, root, pytorch version, auto install these binary: - - * make - * g++ - * git - * wget - * unzip - * opencv - * mmcv (not compulsory) - """ - - print('-' * 10 + 'ensure base env' + '-' * 10) - print(description) - - os.system('python3 -m ensurepip') - os.system('python3 -m pip install wheel') - - sudo = 'sudo' - if 'root' in cmd_result('whoami'): - sudo = '' - - #dep - os.system( - f"{sudo} apt-get install cmake make curl wget git inetutils-ping libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev python3-dev pciutils -y" - ) - - # check ubuntu - ubuntu = cmd_result( - """ lsb_release -a 2>/dev/null | grep "Release" | tail -n 1 | awk '{print $NF}' """ # noqa: E501 - ) - - # check cmake version - cmake = cmd_result('which cmake') - if cmake is None or len(cmake) < 1: - print('cmake not found, try install cmake ..', end='') - os.system('python3 -m pip install cmake') - - cmake = cmd_result('which cmake') - if cmake is None or len(cmake) < 1: - env = 'export PATH=${PATH}:~/.local/bin' - os.system(env) - os.system(""" echo '{}' >> ~/mmdeploy.env """.format(env)) - - cmake = cmd_result('which cmake') - if cmake is None or len(cmake) < 1: - print('Check cmake failed.') - return -1 - print('success') - - # check make - make = cmd_result('which make') - if make is None or len(make) < 1: - print('make not found, try install make ..', end='') - os.system('{} apt update --fix-missing'.format(sudo)) - - os.system( - '{} DEBIAN_FRONTEND="noninteractive" apt install make'.format( - sudo)) - make = cmd_result('which make') - if make is None or len(make) < 1: - print('Check make failed.') - return -1 - print('success') - - # check g++ version - gplus = cmd_result('which g++') - if gplus is None or len(gplus) < 1: - # install g++ - print('g++ not found, try install g++ ..', end='') - os.system( - '{} DEBIAN_FRONTEND="noninteractive" apt install software-properties-common -y' # noqa: E501 - .format(sudo)) # noqa: E501 - os.system('{} apt update'.format(sudo)) - if ubuntu is None or len(ubuntu) < 1 or version_major(ubuntu) <= 18: - os.system( - '{} add-apt-repository ppa:ubuntu-toolchain-r/test -y'.format( - sudo)) - os.system('{} apt install gcc g++ -y'.format(sudo)) - - gplus = cmd_result('which g++') - if gplus is None or len(gplus) < 1: - print('Check g++ failed.') - return -1 - print('success') - - # wget - wget = simple_check_install('wget', sudo) - - # check torch and mmcv, we try to install mmcv, it is not compulsory - mmcv_version = None - torch_version = None - # try: - # import torch - # torch_version = torch.__version__ - - # try: - # import mmcv - # mmcv_version = mmcv.__version__ - # except Exception: - # # install mmcv - # print('mmcv not found, try install mmcv ..', end='') - # os.system('python3 -m pip install -U openmim') - # os.system('mim install mmcv-full==1.5.1') - # except Exception: - # pass - - # git - git = simple_check_install('git', sudo) - - # unzip - unzip = simple_check_install('unzip', sudo) - - # opencv - ocv = cmd_result('which opencv_version') - if ocv is None or len(ocv) < 1: - print('ocv not found, try install ocv ..', end='') - os.system('{} apt update'.format(sudo)) - - pattern = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+') - upstream = cmd_result('{} apt list libopencv-dev -a'.format(sudo)) - - add_ppa = True - if upstream is not None and len(upstream) > 0: - versions = pattern.findall(upstream) - if versions is not None and len(versions) > 0: - version = versions[0] - major = int(version.split('.')[0]) - if major >= 3: - # Directly install upstream OCV, do not need add ppa - add_ppa = False - - if add_ppa: - os.system( - '{} add-apt-repository ppa:ignaciovizzo/opencv3-nonfree -y'. - format(sudo)) - - os.system( - '{} DEBIAN_FRONTEND="noninteractive" apt install libopencv-dev -y' - .format(sudo)) - - ocv = cmd_result('which opencv_version') - if ocv is None or len(ocv) < 1: - print('Check ocv failed.') - return -1 - print('success') - - # print all - print('ubuntu \t\t:{}'.format(ubuntu)) - - # check python - print('python bin\t:{}'.format(cmd_result('which python3'))) - print('python version\t:{}'.format( - cmd_result("python3 --version | awk '{print $2}'"))) - - print('cmake bin\t:{}'.format(cmake)) - print('cmake version\t:{}'.format( - cmd_result("cmake --version | head -n 1 | awk '{print $3}'"))) - - print('make bin\t:{}'.format(make)) - print('make version\t:{}'.format( - cmd_result(" make --version | head -n 1 | awk '{print $3}' "))) - - print('wget bin\t:{}'.format(wget)) - print('g++ bin\t:{}'.format(gplus)) - - print('mmcv version\t:{}'.format(mmcv_version)) - if mmcv_version is None: - print('\t please install mmcv later.') - time.sleep(2) - - print('torch version\t:{}'.format(torch_version)) - if torch_version is None: - print('\t please install pytorch later.') - time.sleep(2) - - print('ocv version\t:{}'.format(cmd_result('opencv_version'))) - - print('git bin\t\t:{}'.format(git)) - print('git version\t:{}'.format( - cmd_result("git --version | awk '{print $3}' "))) - print('unzip bin\t:{}'.format(unzip)) - # work dir - print('work dir \t:{}'.format(work_dir)) - # dep dir - print('dep dir \t:{}'.format(dep_dir)) - - print('\n') - return 0