From 80fe6f781f8c5e1440210319c0d42b0fdfbf2630 Mon Sep 17 00:00:00 2001 From: deathaxe Date: Wed, 15 May 2024 18:19:12 +0200 Subject: [PATCH 1/4] Optimize managed library detection This commit creates a list of orphaned managed libraries by checking each libraries installer via InstalledLibrary object model instead of performing filesystem traversal twice (to subtract unmanaged ones). --- package_control/distinfo.py | 8 +++++--- package_control/library.py | 6 ++++++ package_control/package_manager.py | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/package_control/distinfo.py b/package_control/distinfo.py index dd60544b..b0861878 100644 --- a/package_control/distinfo.py +++ b/package_control/distinfo.py @@ -378,9 +378,11 @@ def read_installer(self): :returns: An unicode string of of which installer was used. """ - - with open(self.abs_path("INSTALLER"), "r", encoding="utf-8") as fobj: - return fobj.readline().strip() + try: + with open(self.abs_path("INSTALLER"), "r", encoding="utf-8") as fobj: + return fobj.readline().strip() + except FileNotFoundError: + return "" def write_installer(self): """ diff --git a/package_control/library.py b/package_control/library.py index 4d1b2bf3..e0adb8e7 100644 --- a/package_control/library.py +++ b/package_control/library.py @@ -141,6 +141,12 @@ def __init__(self, install_root, dist_info_dir, python_version): self.dist_name = dist_info_dir[: dist_info_dir.find("-")].lower() self.python_version = python_version + def is_managed(self): + """ + Library was installed and is therefore managed by Package Control. + """ + return self.dist_info.read_installer() == self.dist_info.generate_installer().strip() + def list_all(): """ diff --git a/package_control/package_manager.py b/package_control/package_manager.py index 193b9db2..f603e373 100644 --- a/package_control/package_manager.py +++ b/package_control/package_manager.py @@ -958,8 +958,8 @@ def find_orphaned_libraries(self, required_libraries=None): installed_libraries = self.list_libraries() if required_libraries is None: required_libraries = self.find_required_libraries() - unmanaged_libraries = library.list_unmanaged() - return installed_libraries - required_libraries - unmanaged_libraries + + return set(lib for lib in installed_libraries - required_libraries if lib.is_managed()) def _download_zip_file(self, name, url, sha256=None): try: From 97385faf17e4925ca6dcc635332023f90cf93d69 Mon Sep 17 00:00:00 2001 From: deathaxe Date: Wed, 15 May 2024 19:00:03 +0200 Subject: [PATCH 2/4] Fix unmanaged libraries being upgraded This commit makes sure not to upgrade unmanaged libraries. --- package_control/package_manager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/package_control/package_manager.py b/package_control/package_manager.py index f603e373..2f82b422 100644 --- a/package_control/package_manager.py +++ b/package_control/package_manager.py @@ -1158,6 +1158,13 @@ def install_library(self, lib): installed_version = pep440.PEP440Version(installed_version) is_upgrade = installed_library is not None + if is_upgrade and not installed_library.is_managed(): + if debug: + console_write( + 'The library "%s" for Python %s was not installed by Package Control; leaving alone', + (lib.name, lib.python_version) + ) + return True release = None available_version = None From 36885a4d75cdf61341a9fe12237857120b75182e Mon Sep 17 00:00:00 2001 From: deathaxe Date: Wed, 15 May 2024 19:00:46 +0200 Subject: [PATCH 3/4] Satisfy libraries in sorted order ... for better readability of debug logs. --- package_control/package_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package_control/package_manager.py b/package_control/package_manager.py index 2f82b422..8c3c44e0 100644 --- a/package_control/package_manager.py +++ b/package_control/package_manager.py @@ -1128,7 +1128,7 @@ def install_libraries(self, libraries, fail_early=True): """ error = False - for lib in libraries: + for lib in sorted(libraries): if not self.install_library(lib): if fail_early: return False @@ -1351,7 +1351,7 @@ def cleanup_libraries(self, required_libraries=None): orphaned_libraries = self.find_orphaned_libraries(required_libraries) error = False - for lib in orphaned_libraries: + for lib in sorted(orphaned_libraries): if not self.remove_library(lib): error = True From ebf4edd85ac69c698ee33a8d42e281db6e73a8b4 Mon Sep 17 00:00:00 2001 From: deathaxe Date: Fri, 17 May 2024 18:47:03 +0200 Subject: [PATCH 4/4] Fix adding INSTALLER file This commit ensures to add .dist-info/INSTALLER and related RECORD entry to mark installed WHEELS as managed. --- package_control/distinfo.py | 22 ++++++++++++++++ package_control/package_cleanup.py | 42 ++++++++++++++++++++++++++++++ package_control/package_manager.py | 3 +++ 3 files changed, 67 insertions(+) diff --git a/package_control/distinfo.py b/package_control/distinfo.py index b0861878..d9bacb76 100644 --- a/package_control/distinfo.py +++ b/package_control/distinfo.py @@ -222,6 +222,28 @@ def generate_installer(self): return "Package Control\n" + def add_installer_to_record(self): + R""" + Add INSTALLER entry to .dist-info/RECORD file. + + Note: hash has been pre-compiled using... + + ```py + digest = hashlib.sha256("Package Control\n".encode("utf-8")).digest() + sha = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("utf-8") + ``` + """ + installer = self.dir_name + "/INSTALLER," + record = self.abs_path("RECORD") + + # make sure not to add duplicate entries + with open(record, "r", encoding="utf-8") as fobj: + items = [item for item in fobj.readlines() if not item.startswith(installer)] + items.append(installer + "sha256=Hg_Q6w_I4zpFfb6C24LQdd4oTAMHJZDk9gtuV2yOgkw,16\n") + + with open(record, "w", encoding="utf-8", newline="\n") as fobj: + fobj.writelines(sorted(items)) + def generate_record(self, package_dirs, package_files): """ Generates the .dist-info/RECORD file contents diff --git a/package_control/package_cleanup.py b/package_control/package_cleanup.py index 05003213..581a678b 100644 --- a/package_control/package_cleanup.py +++ b/package_control/package_cleanup.py @@ -97,6 +97,7 @@ def run(self): removed_packages = None self.remove_legacy_libraries() + self.autofix_missing_installer() # Check metadata to verify packages were not improperly installed self.migrate_incompatible_packages(found_packages) @@ -446,6 +447,47 @@ def install_missing_packages(self, found_packages): with ActivityIndicator('Installing missing packages...') as progress: self.run_install_tasks(tasks, progress, unattended=True, package_kind='missing') + def autofix_missing_installer(self): + """ + Add missing INSTALLER file to available unmodified libraries + + Libraries without INSTALLER are treated unmanaged + and thus are not upgraded or removed automatically. + + Package Control prior to v4.0.7 didn't add INSTALLER for + installed whl files. This option lets users fix it, manually. + """ + + available_libraries = self.manager.list_available_libraries() + for lib in self.manager.list_libraries(): + installer = lib.dist_info.read_installer() + if installer: + continue + + available_library = available_libraries.get(lib.dist_name) + if not available_library: + continue + + if not any( + lib.python_version in available_release['python_versions'] + for available_release in available_library['releases'] + ): + continue + + if not lib.dist_info.verify_files(): + console_write( + 'Library "%s" for Python %s was modified, skipping!', + (lib.name, lib.python_version) + ) + continue + + lib.dist_info.write_installer() + lib.dist_info.add_installer_to_record() + console_write( + 'Library "%s" for Python %s fixed!', + (lib.name, lib.python_version) + ) + def remove_legacy_libraries(self): """ Rename .dist-info directory diff --git a/package_control/package_manager.py b/package_control/package_manager.py index 8c3c44e0..e52f8235 100644 --- a/package_control/package_manager.py +++ b/package_control/package_manager.py @@ -1275,6 +1275,9 @@ def install_library(self, lib): ) return False + temp_did.write_installer() + temp_did.add_installer_to_record() + try: temp_did.verify_python_version(lib.python_version) except EnvironmentError as e: