From 98c25ad69030a83f4fac9dcc779dfe2db5e91629 Mon Sep 17 00:00:00 2001 From: komima <58747243+komima@users.noreply.github.com> Date: Wed, 18 Oct 2023 10:40:03 +0300 Subject: [PATCH] fix: bundle using file catalog --- CHANGELOG.md | 2 +- src/qgis_plugin_dev_tools/build/packaging.py | 223 ++++++++---------- .../utils/distributions.py | 61 ++--- test/test_build.py | 2 + whitelist.txt | 1 + 5 files changed, 130 insertions(+), 159 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c809c..816465f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased -- Fix: Bundle top-level scripts +- Fix: Bundle contents by parsing pep-compliant distribution file catalog instead of possibly missing tool-specific top-level.txt ## [0.6.2] - 2023-09-27 diff --git a/src/qgis_plugin_dev_tools/build/packaging.py b/src/qgis_plugin_dev_tools/build/packaging.py index 5934819..e187d98 100644 --- a/src/qgis_plugin_dev_tools/build/packaging.py +++ b/src/qgis_plugin_dev_tools/build/packaging.py @@ -28,10 +28,7 @@ insert_as_first_import, ) from qgis_plugin_dev_tools.config import DevToolsConfig -from qgis_plugin_dev_tools.utils.distributions import ( - get_distribution_top_level_package_names, - get_distribution_top_level_script_names, -) +from qgis_plugin_dev_tools.utils.distributions import get_distribution_top_level_names IGNORED_FILES = shutil.ignore_patterns("__pycache__", "*.pyc", "*.pyi") LOGGER = logging.getLogger(__name__) @@ -62,164 +59,134 @@ def copy_runtime_requirements( dev_tools_config: DevToolsConfig, build_directory_path: Path, ) -> None: + if len(dev_tools_config.runtime_distributions) == 0: + return + plugin_package_name = dev_tools_config.plugin_package_name + vendor_path = build_directory_path / plugin_package_name / "_vendor" + + vendor_path.mkdir(parents=True) + vendor_init_file = vendor_path / "__init__.py" + vendor_init_file.touch() + + if dev_tools_config.append_distributions_to_path: + vendor_init_file.write_text(VENDOR_PATH_APPEND_SCRIPT) + insert_as_first_import( + build_directory_path / plugin_package_name / "__init__.py", + f"{plugin_package_name}._vendor", + ) + + vendored_runtime_top_level_names: list[str] = [] - if len(dev_tools_config.runtime_distributions) > 0: - vendor_path = build_directory_path / plugin_package_name / "_vendor" - vendor_path.mkdir(parents=True) - vendor_init_file = vendor_path / "__init__.py" - vendor_init_file.touch() - if dev_tools_config.append_distributions_to_path: - vendor_init_file.write_text(VENDOR_PATH_APPEND_SCRIPT) - - runtime_package_names = [] - # copy dist infos (licenses etc.) and all provided top level packages - for dist in ( + for vendored_distribution in ( dev_tools_config.runtime_distributions + dev_tools_config.extra_runtime_distributions ): - dist_info_path = Path(dist._path) # type: ignore + # if a recursively found dependency is provided by system packages, + # assume it does not have to be bundled (this is possibly dangerous, + # if build is made on a different system package set than runtime) if ( - Path(sys.base_prefix) in dist_info_path.parent.parents - and dist in dev_tools_config.extra_runtime_distributions + vendored_distribution in dev_tools_config.extra_runtime_distributions + and Path( + vendored_distribution._path, # type: ignore + ).is_relative_to(Path(sys.base_prefix)) ): - # If build enviroment system packages include a dependency - # resolved via recursive flag, assume it does not have - # to be bundled (possibly dangerous, if build is made on - # a different system package set than runtime) LOGGER.warning( "skipping recursively found runtime requirement %s " "because it is included in system packages", - dist.metadata["Name"], + vendored_distribution.name, ) continue - LOGGER.debug( - "bundling runtime requirement %s", - dist.metadata["Name"], - ) - - dist_top_level_packages = get_distribution_top_level_package_names(dist) - dist_top_level_scripts = get_distribution_top_level_script_names(dist) - - # don't vendor self, but allow to vendor other packages provided + # don't vendor plugin package, but allow to vendor other packages provided # from plugin distribution. if "-e ." installs "my-plugin-name" distribution # containing my_plugin & my_util_package top level packages, bundling is only # needed for my_util_package - dist_top_level_packages = [ - name for name in dist_top_level_packages if name != plugin_package_name - ] + dist_top_level_names = get_distribution_top_level_names(vendored_distribution) + dist_top_level_names.discard(plugin_package_name) LOGGER.debug( - "bundling %s top level packages and %s top level scripts", - dist_top_level_packages, - dist_top_level_scripts, + "bundling runtime requirement %s with top level names %s", + vendored_distribution.name, + dist_top_level_names, ) - - runtime_package_names.extend(dist_top_level_packages) - runtime_package_names.extend(dist_top_level_scripts) - - LOGGER.debug( - "copying %s to build directory", - dist_info_path.resolve(), - ) - shutil.copytree( - src=dist_info_path, - dst=build_directory_path - / plugin_package_name - / "_vendor" - / dist_info_path.name, - ignore=IGNORED_FILES, + _copy_distribution_files( + vendored_distribution, + dist_top_level_names, + vendor_path, ) - for package_name in dist_top_level_packages + dist_top_level_scripts: - _copy_package( - build_directory_path, - dist, - dist_info_path, - package_name, - plugin_package_name, - ) + vendored_runtime_top_level_names.extend(dist_top_level_names) - if dev_tools_config.append_distributions_to_path: - plugin_init_file = (build_directory_path / plugin_package_name) / "__init__.py" - insert_as_first_import(plugin_init_file, f"{plugin_package_name}._vendor") - return + if not dev_tools_config.append_distributions_to_path: + for package_name in vendored_runtime_top_level_names: + LOGGER.debug("rewriting imports for %s", package_name) - for package_name in runtime_package_names: - LOGGER.debug("rewriting imports for %s", package_name) + py_files = list((build_directory_path / plugin_package_name).rglob("*.py")) + ui_files = list((build_directory_path / plugin_package_name).rglob("*.ui")) - py_files = list((build_directory_path / plugin_package_name).rglob("*.py")) - ui_files = list((build_directory_path / plugin_package_name).rglob("*.ui")) - - for source_file in py_files + ui_files: - rewrite_imports_in_source_file( - source_file, - rewritten_package_name=package_name, - container_package_name=f"{plugin_package_name}._vendor", - ) + for source_file in py_files + ui_files: + rewrite_imports_in_source_file( + source_file, + rewritten_package_name=package_name, + container_package_name=f"{plugin_package_name}._vendor", + ) -def _copy_package( - build_directory_path: Path, - dist: Distribution, - dist_info_path: Path, - original_top_level_name: str, - plugin_package_name: str, +def _copy_distribution_files( + distribution: Distribution, + top_level_names: set[str], + target_root_path: Path, ) -> None: + if (file_paths := distribution.files) is None: + LOGGER.warning("could not resolve %s contents to bundle", distribution.name) + return + + # bundle metadata directory first + distribution_metadata_path = Path(distribution._path) # type: ignore LOGGER.debug( - "bundling runtime requirement %s package %s", - dist.metadata["Name"], - original_top_level_name, + "copying %s to build directory", + distribution_metadata_path.resolve(), + ) + shutil.copytree( + src=distribution_metadata_path, + dst=target_root_path / distribution_metadata_path.name, + ignore=IGNORED_FILES, ) - # top-level package - dist_package_src = dist_info_path.parent / original_top_level_name - if dist_package_src.exists(): - LOGGER.debug( - "copying %s to build directory", - dist_package_src.resolve(), - ) + directories_to_bundle = { + top_directory_name + for file_path in file_paths + if len(file_path.parts) > 1 + and (top_directory_name := file_path.parts[0]) in top_level_names + } + files_to_bundle = { + file_path + for file_path in file_paths + if len(file_path.parts) == 1 and file_path.stem in top_level_names + } + + record_root_path = distribution_metadata_path.parent + + for directory_path in directories_to_bundle: + original_path = record_root_path / directory_path + new_path = target_root_path / directory_path + + LOGGER.debug("copying %s to build directory", original_path.resolve()) + shutil.copytree( - src=dist_package_src, - dst=( - build_directory_path - / plugin_package_name - / "_vendor" - / original_top_level_name - ), + src=original_path, + dst=new_path, ignore=IGNORED_FILES, ) - return - # top level script - dist_script_src = dist_info_path.parent / f"{original_top_level_name}.py" - if dist_script_src.exists(): - LOGGER.debug( - "copying %s to build directory", - dist_script_src.resolve(), - ) - shutil.copy( - src=dist_script_src, - dst=build_directory_path / plugin_package_name / "_vendor", - ) - return + for file_path in files_to_bundle: + original_path = record_root_path / file_path + new_path = target_root_path / file_path - # pyd file - pyd_files = list(dist_info_path.parent.glob(f"{original_top_level_name}*.pyd")) - if pyd_files: - for dist_binary_src in pyd_files: - shutil.copy( - src=dist_binary_src, - dst=build_directory_path / plugin_package_name / "_vendor", - ) - return + LOGGER.debug("copying %s to build directory", original_path.resolve()) - if not original_top_level_name.startswith("_"): - raise ValueError( - f"Sources for {original_top_level_name} " - f"from runtime requirement {dist.metadata['Name']} " - f"cannot be found in {dist_info_path.parent}" + shutil.copy( + src=original_path, + dst=new_path, ) - - LOGGER.warning("Could not find %s to bundle", original_top_level_name) diff --git a/src/qgis_plugin_dev_tools/utils/distributions.py b/src/qgis_plugin_dev_tools/utils/distributions.py index 38f4ca7..e6eefad 100644 --- a/src/qgis_plugin_dev_tools/utils/distributions.py +++ b/src/qgis_plugin_dev_tools/utils/distributions.py @@ -19,7 +19,7 @@ import importlib.util import logging from importlib.machinery import SourceFileLoader -from typing import Dict, List, Optional, cast +from typing import Optional, cast import importlib_metadata from importlib_metadata import Distribution, distribution @@ -28,41 +28,39 @@ LOGGER = logging.getLogger(__name__) -def get_distribution_top_level_package_names(dist: Distribution) -> List[str]: - if ( - top_level_contents := cast( - Optional[str], - dist.read_text("top_level.txt"), - ) - ) is None: - LOGGER.debug("%s has no top level packages", dist.name) - return [] - - return top_level_contents.split() - - -def get_distribution_top_level_script_names(dist: Distribution) -> List[str]: +def get_distribution_top_level_names(dist: Distribution) -> set[str]: if (file_paths := dist.files) is None: - LOGGER.warning("%s file catalog missing", dist.name) - return [] + LOGGER.warning("could not resolve %s top level names", dist.name) + return set() + + return { + top_level_directory_name + for path in file_paths + if ( + len(path.parts) > 1 + and not (top_level_directory_name := path.parts[0]).endswith( + (".dist-info", ".egg-info") + ) + and top_level_directory_name not in ("..", "__pycache__") + ) + } | { + path.stem + for path in file_paths + if len(path.parts) == 1 and path.suffix in (".py", ".pyd") + } - return [ - file_path.stem - for file_path in file_paths - if len(file_path.parts) == 1 and file_path.match("*.py") - ] +def get_distribution_requirements(dist: Distribution) -> dict[str, Distribution]: + requirement_distributions: dict[str, Distribution] = {} -def get_distribution_requirements(dist: Distribution) -> Dict[str, Distribution]: requirements = [ Requirement(requirement.split(" ")[0]) for requirement in dist.requires or [] if "extra ==" not in requirement ] - distributions = {} for requirement in requirements: try: - distributions[requirement.name] = distribution(requirement.name) + requirement_distributions[requirement.name] = distribution(requirement.name) except importlib_metadata.PackageNotFoundError: LOGGER.warning( "Getting distribution for %s failed. " @@ -76,8 +74,11 @@ def get_distribution_requirements(dist: Distribution) -> Dict[str, Distribution] LOGGER.error("Could not find package %s", requirement.name) continue - sub_requirements = {} - for requirement in distributions.values(): - sub_requirements.update(get_distribution_requirements(requirement)) - distributions.update(sub_requirements) - return distributions + nested_requirement_distributions: dict[str, Distribution] = {} + for requirement_distribution in requirement_distributions.values(): + nested_requirement_distributions.update( + get_distribution_requirements(requirement_distribution) + ) + requirement_distributions.update(nested_requirement_distributions) + + return requirement_distributions diff --git a/test/test_build.py b/test/test_build.py index 4a2a3bc..3842748 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -136,6 +136,7 @@ def test_make_zip(dev_tools_config: "DevToolsConfig", plugin_dir: Path, tmp_path "pluggy-1.0.0.dist-info", "py", "py-1.11.0.dist-info", + "pyparsing", "pyparsing-3.0.8.dist-info", "pytest", "pytest-6.2.5.dist-info", @@ -191,6 +192,7 @@ def test_make_zip_with_duplicate_dependencies( "pluggy-1.0.0.dist-info", "py", "py-1.11.0.dist-info", + "pyparsing", "pyparsing-3.0.8.dist-info", "pytest", "pytest-6.2.5.dist-info", diff --git a/whitelist.txt b/whitelist.txt index 65811e5..97bac86 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -33,3 +33,4 @@ pyd base64 optionstr ui +vendored