From 7b3e0718f6be48dc4b39cd8798cfcc5bc25c83d8 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:16:24 +0100 Subject: [PATCH 01/15] Implement a new plugin manager from scratch to replace Yapsy --- docs/internals.rst | 4 +- nikola/nikola.py | 162 ++++++------ nikola/plugin_categories.py | 47 ++-- nikola/plugin_manager.py | 232 ++++++++++++++++++ nikola/plugins/command/import_wordpress.py | 7 +- nikola/plugins/command/new_page.py | 2 +- nikola/plugins/command/new_post.py | 6 +- nikola/plugins/command/rst2html/__init__.py | 2 +- nikola/plugins/compile/pandoc.py | 4 +- nikola/plugins/compile/rest/chart.py | 2 +- nikola/plugins/compile/rest/post_list.py | 2 +- nikola/plugins/misc/scan_posts.plugin | 2 + nikola/plugins/shortcode/chart.plugin | 2 +- nikola/plugins/shortcode/emoji.plugin | 2 +- nikola/plugins/shortcode/gist.plugin | 2 +- nikola/plugins/shortcode/listing.plugin | 2 +- nikola/plugins/shortcode/post_list.plugin | 2 +- nikola/plugins/shortcode/thumbnail.plugin | 2 +- nikola/plugins/task/bundles.plugin | 2 +- nikola/plugins/task/gzip.plugin | 2 +- nikola/plugins/task/listings.py | 2 +- nikola/plugins/task/robots.plugin | 2 +- nikola/plugins/task/sitemap.plugin | 2 +- nikola/plugins/template/jinja.plugin | 2 +- nikola/plugins/template/mako.plugin | 2 +- requirements.txt | 1 - snapcraft.yaml | 1 - tests/data/plugin_manager/first.plugin | 13 + tests/data/plugin_manager/one.py | 47 ++++ tests/data/plugin_manager/second/__init__.py | 0 .../data/plugin_manager/second/second.plugin | 13 + .../plugin_manager/second/two/__init__.py | 41 ++++ tests/helper.py | 34 +-- tests/test_plugin_manager.py | 124 ++++++++++ tests/test_template_shortcodes.py | 2 +- 35 files changed, 606 insertions(+), 168 deletions(-) create mode 100644 nikola/plugin_manager.py create mode 100644 tests/data/plugin_manager/first.plugin create mode 100644 tests/data/plugin_manager/one.py create mode 100644 tests/data/plugin_manager/second/__init__.py create mode 100644 tests/data/plugin_manager/second/second.plugin create mode 100644 tests/data/plugin_manager/second/two/__init__.py create mode 100644 tests/test_plugin_manager.py diff --git a/docs/internals.rst b/docs/internals.rst index 0180f0d395..a3b7af2fca 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -15,7 +15,7 @@ So, this is a short document explaining what each piece of Nikola does and how it all fits together. Nikola is a Pile of Plugins - Most of Nikola is implemented as plugins using `Yapsy `_. + Most of Nikola is implemented as plugins (using a custom plugin manager inspired by Yapsy). You can ignore that they are plugins and just think of them as regular python modules and packages with a funny little ``.plugin`` file next to them. @@ -65,7 +65,7 @@ basename:name If you ever want to do your own tasks, you really should read the doit `documentation on tasks `_. - + Notably, by default doit redirects ``stdout`` and ``stderr``. To get a proper PDB debugging shell, you need to use doit's own `set_trace `_ function. diff --git a/nikola/nikola.py b/nikola/nikola.py index 056f606dd6..4accadcc44 100644 --- a/nikola/nikola.py +++ b/nikola/nikola.py @@ -35,6 +35,7 @@ import os import pathlib import sys +import typing import mimetypes from collections import defaultdict from copy import copy @@ -47,12 +48,12 @@ import PyRSS2Gen as rss from pkg_resources import resource_filename from blinker import signal -from yapsy.PluginManager import PluginManager from . import DEBUG, SHOW_TRACEBACKS, filters, utils, hierarchy_utils, shortcodes from . import metadata_extractors from .metadata_extractors import default_metadata_extractors_by from .post import Post # NOQA +from .plugin_manager import PluginCandidate, PluginInfo, PluginManager from .plugin_categories import ( Command, LateTask, @@ -377,25 +378,15 @@ def _enclosure(post, lang): return url, length, mime -def _plugin_load_callback(plugin_info): - """Set the plugin's module path. - - So we can find its template path later. - """ - try: - plugin_info.plugin_object.set_module_path(plugin_info.path) - except AttributeError: - # this is just for safety in case plugin_object somehow - # isn't set - pass - - class Nikola(object): """Class that handles site generation. Takes a site config as argument on creation. """ + plugin_manager: PluginManager + _template_system: TemplateSystem + def __init__(self, **config): """Initialize proper environment for running tasks.""" # Register our own path handlers @@ -1019,57 +1010,48 @@ def __init__(self, **config): # WebP files have no official MIME type yet, but we need to recognize them (Issue #3671) mimetypes.add_type('image/webp', '.webp') - def _filter_duplicate_plugins(self, plugin_list): + def _filter_duplicate_plugins(self, plugin_list: typing.Iterable[PluginCandidate]): """Find repeated plugins and discard the less local copy.""" - def plugin_position_in_places(plugin): + def plugin_position_in_places(plugin: PluginInfo): # plugin here is a tuple: # (path to the .plugin file, path to plugin module w/o .py, plugin metadata) for i, place in enumerate(self._plugin_places): - if plugin[0].startswith(place): + place: pathlib.Path + try: + # Path.is_relative_to backport + plugin.source_dir.relative_to(place) return i - utils.LOGGER.warn("Duplicate plugin found in unexpected location: {}".format(plugin[0])) + except ValueError: + pass + utils.LOGGER.warning("Duplicate plugin found in unexpected location: {}".format(plugin.source_dir)) return len(self._plugin_places) plugin_dict = defaultdict(list) - for data in plugin_list: - plugin_dict[data[2].name].append(data) + for plugin in plugin_list: + plugin_dict[plugin.name].append(plugin) result = [] - for _, plugins in plugin_dict.items(): + for name, plugins in plugin_dict.items(): if len(plugins) > 1: # Sort by locality plugins.sort(key=plugin_position_in_places) utils.LOGGER.debug("Plugin {} exists in multiple places, using {}".format( - plugins[-1][2].name, plugins[-1][0])) + name, plugins[-1].source_dir)) result.append(plugins[-1]) return result def init_plugins(self, commands_only=False, load_all=False): """Load plugins as needed.""" - self.plugin_manager = PluginManager(categories_filter={ - "Command": Command, - "Task": Task, - "LateTask": LateTask, - "TemplateSystem": TemplateSystem, - "PageCompiler": PageCompiler, - "TaskMultiplier": TaskMultiplier, - "CompilerExtension": CompilerExtension, - "MarkdownExtension": MarkdownExtension, - "RestExtension": RestExtension, - "MetadataExtractor": MetadataExtractor, - "ShortcodePlugin": ShortcodePlugin, - "SignalHandler": SignalHandler, - "ConfigPlugin": ConfigPlugin, - "CommentSystem": CommentSystem, - "PostScanner": PostScanner, - "Taxonomy": Taxonomy, - }) - self.plugin_manager.getPluginLocator().setPluginInfoExtension('plugin') extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS'] + self._loading_commands_only = commands_only self._plugin_places = [ resource_filename('nikola', 'plugins'), os.path.expanduser(os.path.join('~', '.nikola', 'plugins')), os.path.join(os.getcwd(), 'plugins'), ] + [path for path in extra_plugins_dirs if path] + self._plugin_places = [pathlib.Path(p) for p in self._plugin_places] + + + self.plugin_manager = PluginManager(plugin_places=self._plugin_places) compilers = defaultdict(set) # Also add aliases for combinations with TRANSLATIONS_PATTERN @@ -1088,41 +1070,37 @@ def init_plugins(self, commands_only=False, load_all=False): self.disabled_compilers = {} self.disabled_compiler_extensions = defaultdict(list) - self.plugin_manager.getPluginLocator().setPluginPlaces(self._plugin_places) - self.plugin_manager.locatePlugins() - bad_candidates = set([]) + candidates = self.plugin_manager.locate_plugins() + good_candidates = set() if not load_all: - for p in self.plugin_manager._candidates: + for p in candidates: if commands_only: - if p[-1].details.has_option('Nikola', 'PluginCategory'): - # FIXME TemplateSystem should not be needed - if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}: - bad_candidates.add(p) - else: - bad_candidates.add(p) + if p.category != 'Command': + continue elif self.configured: # Not commands-only, and configured # Remove blacklisted plugins - if p[-1].name in self.config['DISABLED_PLUGINS']: - bad_candidates.add(p) - utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name) - # Remove compilers we don't use - if p[-1].details.has_option('Nikola', 'PluginCategory') and p[-1].details.get('Nikola', 'PluginCategory') in ('Compiler', 'PageCompiler'): - bad_candidates.add(p) - self.disabled_compilers[p[-1].name] = p + if p.name in self.config['DISABLED_PLUGINS']: + utils.LOGGER.debug('Not loading disabled plugin {}', p.name) + continue + # Remove compilers - will be loaded later based on usage + if p.category == "PageCompiler": + self.disabled_compilers[p.name] = p + continue # Remove compiler extensions we don't need - if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers: - bad_candidates.add(p) - self.disabled_compiler_extensions[p[-1].details.get('Nikola', 'compiler')].append(p) - self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates) + if p.compiler and p.compiler in self.disabled_compilers: + self.disabled_compiler_extensions[p.compiler].append(p) + continue + good_candidates.add(p) - self.plugin_manager._candidates = self._filter_duplicate_plugins(self.plugin_manager._candidates) - self.plugin_manager.loadPlugins(callback_after=_plugin_load_callback) + good_candidates = self._filter_duplicate_plugins(good_candidates) + self.plugin_manager.load_plugins(good_candidates) # Search for compiler plugins which we disabled but shouldn't have self._activate_plugins_of_category("PostScanner") if not load_all: file_extensions = set() - for post_scanner in [p.plugin_object for p in self.plugin_manager.getPluginsOfCategory('PostScanner')]: + for post_scanner in [p.plugin_object for p in self.plugin_manager.get_plugins_of_category('PostScanner')]: + post_scanner: PostScanner exts = post_scanner.supported_extensions() if exts is not None: file_extensions.update(exts) @@ -1141,13 +1119,13 @@ def init_plugins(self, commands_only=False, load_all=False): for p in self.disabled_compiler_extensions.pop(k, []): to_add.append(p) for _, p in self.disabled_compilers.items(): - utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) + utils.LOGGER.debug('Not loading unneeded compiler %s', p.name) for _, plugins in self.disabled_compiler_extensions.items(): for p in plugins: - utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name) + utils.LOGGER.debug('Not loading compiler extension %s', p.name) if to_add: - self.plugin_manager._candidates = self._filter_duplicate_plugins(to_add) - self.plugin_manager.loadPlugins(callback_after=_plugin_load_callback) + extra_candidates = self._filter_duplicate_plugins(to_add) + self.plugin_manager.load_plugins(extra_candidates) # Jupyter theme configuration. If a website has ipynb enabled in post_pages # we should enable the Jupyter CSS (leaving that up to the theme itself). @@ -1160,7 +1138,8 @@ def init_plugins(self, commands_only=False, load_all=False): self._activate_plugins_of_category("Taxonomy") self.taxonomy_plugins = {} - for taxonomy in [p.plugin_object for p in self.plugin_manager.getPluginsOfCategory('Taxonomy')]: + for taxonomy in [p.plugin_object for p in self.plugin_manager.get_plugins_of_category('Taxonomy')]: + taxonomy: Taxonomy if not taxonomy.is_enabled(): continue if taxonomy.classification_name in self.taxonomy_plugins: @@ -1186,10 +1165,9 @@ def init_plugins(self, commands_only=False, load_all=False): # Activate all required compiler plugins self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension") - for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"): + for plugin_info in self.plugin_manager.get_plugins_of_category("PageCompiler"): if plugin_info.name in self.config["COMPILERS"].keys(): - self.plugin_manager.activatePluginByName(plugin_info.name) - plugin_info.plugin_object.set_site(self) + self._activate_plugin(plugin_info) # Activate shortcode plugins self._activate_plugins_of_category("ShortcodePlugin") @@ -1198,10 +1176,8 @@ def init_plugins(self, commands_only=False, load_all=False): self.compilers = {} self.inverse_compilers = {} - for plugin_info in self.plugin_manager.getPluginsOfCategory( - "PageCompiler"): - self.compilers[plugin_info.name] = \ - plugin_info.plugin_object + for plugin_info in self.plugin_manager.get_plugins_of_category("PageCompiler"): + self.compilers[plugin_info.name] = plugin_info.plugin_object # Load comment systems, config plugins and register templated shortcodes self._activate_plugins_of_category("CommentSystem") @@ -1344,13 +1320,26 @@ def _set_all_page_deps_from_config(self): self.ALL_PAGE_DEPS['index_read_more_link'] = self.config.get('INDEX_READ_MORE_LINK') self.ALL_PAGE_DEPS['feed_read_more_link'] = self.config.get('FEED_READ_MORE_LINK') - def _activate_plugins_of_category(self, category): + def _activate_plugin(self, plugin_info: PluginInfo) -> None: + plugin_info.plugin_object.set_site(self) + + if plugin_info.category == "TemplateSystem" or self._loading_commands_only: + return + + templates_directory_candidates = [ + plugin_info.source_dir / "templates" / self.template_system.name, + plugin_info.source_dir / plugin_info.module_name / "templates" / self.template_system.name + ] + for candidate in templates_directory_candidates: + if candidate.exists() and candidate.is_dir(): + self.template_system.inject_directory(str(candidate)) + + def _activate_plugins_of_category(self, category) -> typing.List[PluginInfo]: """Activate all the plugins of a given category and return them.""" # this code duplicated in tests/base.py plugins = [] - for plugin_info in self.plugin_manager.getPluginsOfCategory(category): - self.plugin_manager.activatePluginByName(plugin_info.name) - plugin_info.plugin_object.set_site(self) + for plugin_info in self.plugin_manager.get_plugins_of_category(category): + self._activate_plugin(plugin_info) plugins.append(plugin_info) return plugins @@ -1414,13 +1403,12 @@ def _get_template_system(self): if self._template_system is None: # Load template plugin template_sys_name = utils.get_template_engine(self.THEMES) - pi = self.plugin_manager.getPluginByName( - template_sys_name, "TemplateSystem") + pi = self.plugin_manager.get_plugin_by_name(template_sys_name, "TemplateSystem") if pi is None: sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name)) sys.exit(1) - self._template_system = pi.plugin_object + self._template_system = typing.cast(TemplateSystem, pi.plugin_object) lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES] self._template_system.set_directories(lookup_dirs, @@ -2089,7 +2077,7 @@ def flatten(task): yield ft task_dep = [] - for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category): + for pluginInfo in self.plugin_manager.get_plugins_of_category(plugin_category): for task in flatten(pluginInfo.plugin_object.gen_tasks()): if 'basename' not in task: raise ValueError("Task {0} does not have a basename".format(task)) @@ -2098,7 +2086,7 @@ def flatten(task): task['task_dep'] = [] task['task_dep'].extend(self.injected_deps[task['basename']]) yield task - for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"): + for multi in self.plugin_manager.get_plugins_of_category("TaskMultiplier"): flag = False for task in multi.plugin_object.process(task, name): flag = True @@ -2207,7 +2195,7 @@ def scan_posts(self, really=False, ignore_quit=False, quiet=False): self.timeline = [] self.pages = [] - for p in sorted(self.plugin_manager.getPluginsOfCategory('PostScanner'), key=operator.attrgetter('name')): + for p in sorted(self.plugin_manager.get_plugins_of_category('PostScanner'), key=operator.attrgetter('name')): try: timeline = p.plugin_object.scan() except Exception: diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py index 08316284d5..2abef7ed11 100644 --- a/nikola/plugin_categories.py +++ b/nikola/plugin_categories.py @@ -33,7 +33,6 @@ import doit from doit.cmd_base import Command as DoitCommand -from yapsy.IPlugin import IPlugin from .utils import LOGGER, first_line, get_logger, req_missing @@ -58,7 +57,7 @@ ) -class BasePlugin(IPlugin): +class BasePlugin: """Base plugin class.""" logger = None @@ -66,7 +65,6 @@ class BasePlugin(IPlugin): def set_site(self, site): """Set site, which is a Nikola instance.""" self.site = site - self.inject_templates() self.logger = get_logger(self.name) if not site.debug: self.logger.level = logging.INFO @@ -75,21 +73,6 @@ def set_module_path(self, module_path): """Set the plugin's module path.""" self.module_path = module_path - def inject_templates(self): - """Inject 'templates/' (if exists) very early in the theme chain.""" - try: - mod_dir = os.path.dirname(self.module_path) - tmpl_dir = os.path.join( - mod_dir, 'templates', self.site.template_system.name - ) - if os.path.isdir(tmpl_dir): - # Inject tmpl_dir low in the theme chain - self.site.template_system.inject_directory(tmpl_dir) - except AttributeError: - # This would likely mean we don't have module_path set. - # this should not usually happen, so log a warning - LOGGER.warning("Could not find template path for module {0}".format(self.name)) - def inject_dependency(self, target, dependency): """Add 'dependency' to the target task's task_deps.""" self.site.injected_deps[target].append(dependency) @@ -359,7 +342,7 @@ def get_compiler_extensions(self) -> list: """Activate all the compiler extension plugins for a given compiler and return them.""" plugins = [] for plugin_info in self.site.compiler_extensions: - if plugin_info.plugin_object.compiler_name == self.name: + if plugin_info.compiler == self.name or plugin_info.plugin_object.compiler_name == self.name: plugins.append(plugin_info) return plugins @@ -371,11 +354,8 @@ class CompilerExtension(BasePlugin): (a) create a new plugin class for them; or (b) use this class and filter them yourself. If you choose (b), you should the compiler name to the .plugin - file in the Nikola/Compiler section and filter all plugins of - this category, getting the compiler name with: - p.details.get('Nikola', 'Compiler') - Note that not all compiler plugins have this option and you might - need to catch configparser.NoOptionError exceptions. + file in the Nikola/compiler section and filter all plugins of + this category, getting the compiler name with `plugin_info.compiler`. """ name = "dummy_compiler_extension" @@ -904,3 +884,22 @@ def get_other_language_variants(self, classification: str, lang: str, classifica Provided is a set of classifications per language (`classifications_per_language`). """ return [] + +CATEGORIES = { + "Command": Command, + "Task": Task, + "LateTask": LateTask, + "TemplateSystem": TemplateSystem, + "PageCompiler": PageCompiler, + "TaskMultiplier": TaskMultiplier, + "CompilerExtension": CompilerExtension, + "MarkdownExtension": MarkdownExtension, + "RestExtension": RestExtension, + "MetadataExtractor": MetadataExtractor, + "ShortcodePlugin": ShortcodePlugin, + "SignalHandler": SignalHandler, + "ConfigPlugin": ConfigPlugin, + "CommentSystem": CommentSystem, + "PostScanner": PostScanner, + "Taxonomy": Taxonomy, +} diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py new file mode 100644 index 0000000000..a9a95bf6f0 --- /dev/null +++ b/nikola/plugin_manager.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2024 Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The Nikola plugin manager. Inspired by yapsy.""" + +import configparser +import importlib +import importlib.util +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Type, TYPE_CHECKING, Set + +from .plugin_categories import BasePlugin, CATEGORIES +from .utils import get_logger + +if TYPE_CHECKING: + import logging + +LEGACY_PLUGIN_NAMES: Dict[str, str] = { + "Compiler": "PageCompiler", + "Shortcode": "ShortcodePlugin", + "Template": "TemplateSystem", +} + +CATEGORY_NAMES: Set[str] = set(CATEGORIES.keys()) +CATEGORY_TYPES: Set[Type[BasePlugin]] = set(CATEGORIES.values()) + + +@dataclass(frozen=True) +class PluginCandidate: + name: str + description: Optional[str] + plugin_id: str + category: str + compiler: Optional[str] + source_dir: Path + module_name: str + + +@dataclass(frozen=True) +class PluginInfo: + name: str + description: Optional[str] + plugin_id: str + category: str + compiler: Optional[str] + source_dir: Path + module_name: str + module_object: object + plugin_object: BasePlugin + + +class PluginManager: + """The Nikola plugin manager.""" + + categories_filter: Dict[str, Type[BasePlugin]] + plugin_places: List[Path] + logger: "logging.Logger" + candidates: List[PluginCandidate] + plugins: List[PluginInfo] + _plugins_by_category: Dict[str, List[PluginInfo]] + + def __init__(self, plugin_places: List[Path]): + """Initialize the plugin manager.""" + self.plugin_places = plugin_places + self.candidates = [] + self.plugins = [] + self._plugins_by_category = {} + self.logger = get_logger("PluginManager") + + def locate_plugins(self, force=False) -> List[PluginCandidate]: + """Locate plugins in plugin_places.""" + if self.candidates and not force: + # Already located + return self.candidates + + self.candidates = [] + + plugin_files: List[Path] = [] + for place in self.plugin_places: + plugin_files += place.rglob("*.plugin") + + for plugin_file in plugin_files: + source_dir = plugin_file.parent + config = configparser.ConfigParser() + config.read(plugin_file) + name = config["Core"]["name"] + module_name = config["Core"]["module"] + plugin_id = f"Plugin {name} from {plugin_file}" + description = None + if "Documentation" in config: + description = config["Documentation"].get("Description") + if "Nikola" not in config: + self.logger.warning(f"{plugin_id} does not specify Nikola configuration - it will not be loaded") + continue + category = config["Nikola"].get("PluginCategory") + compiler = config["Nikola"].get("Compiler") + if not category: + self.logger.warning(f"{plugin_id} does not specify any category - it will not be loaded") + continue + if category in LEGACY_PLUGIN_NAMES: + category = LEGACY_PLUGIN_NAMES[category] + if category not in CATEGORY_NAMES: + self.logger.warning(f"{plugin_id} specifies invalid category '{category}'") + continue + self.logger.debug(f"Discovered {plugin_id}") + self.candidates.append( + PluginCandidate( + name=name, + description=description, + plugin_id=plugin_id, + category=category, + compiler=compiler, + source_dir=source_dir, + module_name=module_name, + ) + ) + return self.candidates + + def load_plugins(self, candidates: List[PluginCandidate]) -> None: + plugins_root = Path(__file__).parent.parent + + for candidate in candidates: + name = candidate.name + module_name = candidate.module_name + source_dir = candidate.source_dir + py_file_location = source_dir / f"{module_name}.py" + plugin_id = candidate.plugin_id + if not py_file_location.exists(): + py_file_location = source_dir / module_name / "__init__.py" + if not py_file_location.exists(): + self.logger.warning(f"{plugin_id} could not be loaded (no valid module detected)") + continue + + plugin_id += f" ({py_file_location})" + full_module_name = module_name + + try: + name_parts = list(py_file_location.relative_to(plugins_root).parts) + if name_parts[-1] == "__init__.py": + name_parts.pop(-1) + elif name_parts[-1].endswith(".py"): + name_parts[-1] = name_parts[-1][:-3] + full_module_name = ".".join(name_parts) + except ValueError: + pass + + if full_module_name.startswith("nikola.plugins") and full_module_name in sys.modules: + # Loaded by something else (a dependent plugin?) + module_object = sys.modules[full_module_name] + else: + try: + spec = importlib.util.spec_from_file_location(full_module_name, py_file_location) + module_object = importlib.util.module_from_spec(spec) + sys.modules[full_module_name] = module_object + spec.loader.exec_module(module_object) + except Exception as exc: + self.logger.exception(f"{plugin_id} threw an exception while loading") + continue + + plugin_classes = [ + c + for c in vars(module_object).values() + if isinstance(c, type) and issubclass(c, BasePlugin) and c not in CATEGORY_TYPES + ] + if len(plugin_classes) == 0: + self.logger.warning(f"{plugin_id} does not have any plugin classes") + continue + elif len(plugin_classes) > 1: + self.logger.warning(f"{plugin_id} has multiple plugin classes; this is not supported - skipping") + continue + try: + plugin_object = plugin_classes[0]() + except Exception as exc: + self.logger.exception(f"{plugin_id} threw an exception while creating an instance") + continue + self.logger.debug(f"Loaded {plugin_id}") + info = PluginInfo( + name=name, + description=candidate.description, + plugin_id=candidate.plugin_id, + category=candidate.category, + compiler=candidate.compiler, + source_dir=source_dir, + module_name=module_name, + module_object=module_object, + plugin_object=plugin_object, + ) + self.plugins.append(info) + + self._plugins_by_category = {category: [] for category in CATEGORY_NAMES} + for plugin_info in self.plugins: + self._plugins_by_category[plugin_info.category].append(plugin_info) + + def get_plugins_of_category(self, category: str) -> List[PluginInfo]: + return self._plugins_by_category.get(category, []) + + def get_plugin_by_name(self, name: str, category: str | None = None) -> PluginInfo | None: + for p in self.plugins: + if p.name == name and (category is None or p.category == category): + return p + + # Aliases for Yapsy compatibility + def getPluginsOfCategory(self, category: str) -> List[PluginInfo]: + return self._plugins_by_category.get(category, []) + + def getPluginByName(self, name: str, category: str | None = None) -> PluginInfo | None: + return self.get_plugin_by_name(name, category) diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index e419c8a1b7..27f58b0da6 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -66,7 +66,7 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False) """Install a Nikola plugin.""" LOGGER.info("Installing plugin '{0}'".format(plugin_name)) # Get hold of the 'plugin' plugin - plugin_installer_info = site.plugin_manager.getPluginByName('plugin', 'Command') + plugin_installer_info = site.plugin_manager.get_plugin_by_name('plugin', 'Command') if plugin_installer_info is None: LOGGER.error('Internal error: cannot find the "plugin" plugin which is supposed to come with Nikola!') return False @@ -236,10 +236,9 @@ def _get_compiler(self): self._find_wordpress_compiler() if self.wordpress_page_compiler is not None: return self.wordpress_page_compiler - plugin_info = self.site.plugin_manager.getPluginByName('markdown', 'PageCompiler') + plugin_info = self.site.plugin_manager.get_plugin_by_name('markdown', 'PageCompiler') if plugin_info is not None: if not plugin_info.is_activated: - self.site.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self.site) return plugin_info.plugin_object else: @@ -249,7 +248,7 @@ def _find_wordpress_compiler(self): """Find WordPress compiler plugin.""" if self.wordpress_page_compiler is not None: return - plugin_info = self.site.plugin_manager.getPluginByName('wordpress', 'PageCompiler') + plugin_info = self.site.plugin_manager.get_plugin_by_name('wordpress', 'PageCompiler') if plugin_info is not None: if not plugin_info.is_activated: self.site.plugin_manager.activatePluginByName(plugin_info.name) diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index cde5e5c6c6..780e5db2ff 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -109,5 +109,5 @@ def _execute(self, options, args): options['date-path'] = False # Even though stuff was split into `new_page`, it’s easier to do it # there not to duplicate the code. - p = self.site.plugin_manager.getPluginByName('new_post', 'Command').plugin_object + p = self.site.plugin_manager.get_plugin_by_name('new_post', 'Command').plugin_object return p.execute(options, args) diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index 45ca1a11f8..2c03fa5c73 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -216,9 +216,7 @@ class CommandNewPost(Command): def _execute(self, options, args): """Create a new post or page.""" global LOGGER - compiler_names = [p.name for p in - self.site.plugin_manager.getPluginsOfCategory( - "PageCompiler")] + compiler_names = [p.name for p in self.site.plugin_manager.get_plugins_of_category("PageCompiler")] if len(args) > 1: print(self.help()) @@ -298,7 +296,7 @@ def _execute(self, options, args): self.print_compilers() return - compiler_plugin = self.site.plugin_manager.getPluginByName( + compiler_plugin = self.site.plugin_manager.get_plugin_by_name( content_format, "PageCompiler").plugin_object # Guess where we should put this diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py index a133bd5ec8..d2854df94b 100644 --- a/nikola/plugins/command/rst2html/__init__.py +++ b/nikola/plugins/command/rst2html/__init__.py @@ -44,7 +44,7 @@ class CommandRst2Html(Command): def _execute(self, options, args): """Compile reStructuredText to standalone HTML files.""" - compiler = self.site.plugin_manager.getPluginByName('rest', 'PageCompiler').plugin_object + compiler = self.site.plugin_manager.get_plugin_by_name('rest', 'PageCompiler').plugin_object if len(args) != 1: print("This command takes only one argument (input file name).") return 2 diff --git a/nikola/plugins/compile/pandoc.py b/nikola/plugins/compile/pandoc.py index c79c4d88e7..f510256179 100644 --- a/nikola/plugins/compile/pandoc.py +++ b/nikola/plugins/compile/pandoc.py @@ -62,10 +62,10 @@ def _get_pandoc_options(self, source: str) -> List[str]: try: pandoc_options = list(config_options[ext]) except KeyError: - self.logger.warn('Setting PANDOC_OPTIONS to [], because extension {} is not defined in PANDOC_OPTIONS: {}.'.format(ext, config_options)) + self.logger.warning('Setting PANDOC_OPTIONS to [], because extension {} is not defined in PANDOC_OPTIONS: {}.'.format(ext, config_options)) pandoc_options = [] else: - self.logger.warn('Setting PANDOC_OPTIONS to [], because PANDOC_OPTIONS is expected to be of type Union[List[str], Dict[str, List[str]]] but this is not: {}'.format(config_options)) + self.logger.warning('Setting PANDOC_OPTIONS to [], because PANDOC_OPTIONS is expected to be of type Union[List[str], Dict[str, List[str]]] but this is not: {}'.format(config_options)) pandoc_options = [] return pandoc_options diff --git a/nikola/plugins/compile/rest/chart.py b/nikola/plugins/compile/rest/chart.py index d217c1bf2e..558fe7c904 100644 --- a/nikola/plugins/compile/rest/chart.py +++ b/nikola/plugins/compile/rest/chart.py @@ -153,7 +153,7 @@ class Chart(Directive): def run(self): """Run the directive.""" self.options['site'] = None - html = _site.plugin_manager.getPluginByName( + html = _site.plugin_manager.get_plugin_by_name( 'chart', 'ShortcodePlugin').plugin_object.handler( self.arguments[0], data='\n'.join(self.content), diff --git a/nikola/plugins/compile/rest/post_list.py b/nikola/plugins/compile/rest/post_list.py index f5b86baef2..9f66073643 100644 --- a/nikola/plugins/compile/rest/post_list.py +++ b/nikola/plugins/compile/rest/post_list.py @@ -88,7 +88,7 @@ def run(self): date = self.options.get('date') filename = self.state.document.settings._nikola_source_path - output, deps = self.site.plugin_manager.getPluginByName( + output, deps = self.site.plugin_manager.get_plugin_by_name( 'post_list', 'ShortcodePlugin').plugin_object.handler( start, stop, diff --git a/nikola/plugins/misc/scan_posts.plugin b/nikola/plugins/misc/scan_posts.plugin index f4af811276..0fb946c792 100644 --- a/nikola/plugins/misc/scan_posts.plugin +++ b/nikola/plugins/misc/scan_posts.plugin @@ -8,3 +8,5 @@ Version = 1.0 Website = https://getnikola.com/ Description = Scan posts and create timeline +[Nikola] +PluginCategory = PostScanner diff --git a/nikola/plugins/shortcode/chart.plugin b/nikola/plugins/shortcode/chart.plugin index edcbc1310a..be1fbc6efd 100644 --- a/nikola/plugins/shortcode/chart.plugin +++ b/nikola/plugins/shortcode/chart.plugin @@ -3,7 +3,7 @@ name = chart module = chart [Nikola] -PluginCategory = Shortcode +PluginCategory = ShortcodePlugin [Documentation] author = Roberto Alsina diff --git a/nikola/plugins/shortcode/emoji.plugin b/nikola/plugins/shortcode/emoji.plugin index c9a272cdcd..4c09f03157 100644 --- a/nikola/plugins/shortcode/emoji.plugin +++ b/nikola/plugins/shortcode/emoji.plugin @@ -3,7 +3,7 @@ name = emoji module = emoji [Nikola] -PluginCategory = Shortcode +PluginCategory = ShortcodePlugin [Documentation] author = Roberto Alsina diff --git a/nikola/plugins/shortcode/gist.plugin b/nikola/plugins/shortcode/gist.plugin index b610763835..ee62c27069 100644 --- a/nikola/plugins/shortcode/gist.plugin +++ b/nikola/plugins/shortcode/gist.plugin @@ -3,7 +3,7 @@ name = gist module = gist [Nikola] -PluginCategory = Shortcode +PluginCategory = ShortcodePlugin [Documentation] author = Roberto Alsina diff --git a/nikola/plugins/shortcode/listing.plugin b/nikola/plugins/shortcode/listing.plugin index 90fb6ebb28..70fa1cf416 100644 --- a/nikola/plugins/shortcode/listing.plugin +++ b/nikola/plugins/shortcode/listing.plugin @@ -3,7 +3,7 @@ name = listing_shortcode module = listing [Nikola] -PluginCategory = Shortcode +PluginCategory = ShortcodePlugin [Documentation] author = Roberto Alsina diff --git a/nikola/plugins/shortcode/post_list.plugin b/nikola/plugins/shortcode/post_list.plugin index 494a1d8755..9c39eb9e99 100644 --- a/nikola/plugins/shortcode/post_list.plugin +++ b/nikola/plugins/shortcode/post_list.plugin @@ -3,7 +3,7 @@ name = post_list module = post_list [Nikola] -PluginCategory = Shortcode +PluginCategory = ShortcodePlugin [Documentation] author = Udo Spallek diff --git a/nikola/plugins/shortcode/thumbnail.plugin b/nikola/plugins/shortcode/thumbnail.plugin index e55d34f6c3..bd36169002 100644 --- a/nikola/plugins/shortcode/thumbnail.plugin +++ b/nikola/plugins/shortcode/thumbnail.plugin @@ -3,7 +3,7 @@ name = thumbnail module = thumbnail [Nikola] -PluginCategory = Shortcode +PluginCategory = ShortcodePlugin [Documentation] author = Chris Warrick diff --git a/nikola/plugins/task/bundles.plugin b/nikola/plugins/task/bundles.plugin index 939065b7cb..4afc3ecd3f 100644 --- a/nikola/plugins/task/bundles.plugin +++ b/nikola/plugins/task/bundles.plugin @@ -9,5 +9,5 @@ website = https://getnikola.com/ description = Bundle assets [Nikola] -PluginCategory = Task +PluginCategory = LateTask diff --git a/nikola/plugins/task/gzip.plugin b/nikola/plugins/task/gzip.plugin index 70e966b83a..b1aab25dc8 100644 --- a/nikola/plugins/task/gzip.plugin +++ b/nikola/plugins/task/gzip.plugin @@ -9,5 +9,5 @@ website = https://getnikola.com/ description = Create gzipped copies of files [Nikola] -PluginCategory = Task +PluginCategory = TaskMultiplier diff --git a/nikola/plugins/task/listings.py b/nikola/plugins/task/listings.py index 4c1636e558..f4933e9379 100644 --- a/nikola/plugins/task/listings.py +++ b/nikola/plugins/task/listings.py @@ -113,7 +113,7 @@ def render_listing(in_name, out_name, input_folder, output_folder, folders=[], f needs_ipython_css = False if in_name and in_name.endswith('.ipynb'): # Special handling: render ipynbs in listings (Issue #1900) - ipynb_plugin = self.site.plugin_manager.getPluginByName("ipynb", "PageCompiler") + ipynb_plugin = self.site.plugin_manager.get_plugin_by_name("ipynb", "PageCompiler") if ipynb_plugin is None: msg = "To use .ipynb files as listings, you must set up the Jupyter compiler in COMPILERS and POSTS/PAGES." utils.LOGGER.error(msg) diff --git a/nikola/plugins/task/robots.plugin b/nikola/plugins/task/robots.plugin index 51f778183c..81c4c9a6b9 100644 --- a/nikola/plugins/task/robots.plugin +++ b/nikola/plugins/task/robots.plugin @@ -9,5 +9,5 @@ website = https://getnikola.com/ description = Generate /robots.txt exclusion file and promote sitemap. [Nikola] -PluginCategory = Task +PluginCategory = LateTask diff --git a/nikola/plugins/task/sitemap.plugin b/nikola/plugins/task/sitemap.plugin index c8aa832392..8367d8ecaa 100644 --- a/nikola/plugins/task/sitemap.plugin +++ b/nikola/plugins/task/sitemap.plugin @@ -9,5 +9,5 @@ website = https://getnikola.com/ description = Generate google sitemap. [Nikola] -PluginCategory = Task +PluginCategory = LateTask diff --git a/nikola/plugins/template/jinja.plugin b/nikola/plugins/template/jinja.plugin index 629b20e888..45dd621643 100644 --- a/nikola/plugins/template/jinja.plugin +++ b/nikola/plugins/template/jinja.plugin @@ -9,5 +9,5 @@ website = https://getnikola.com/ description = Support for Jinja2 templates. [Nikola] -PluginCategory = Template +PluginCategory = TemplateSystem diff --git a/nikola/plugins/template/mako.plugin b/nikola/plugins/template/mako.plugin index 2d353bf1d9..a46575255e 100644 --- a/nikola/plugins/template/mako.plugin +++ b/nikola/plugins/template/mako.plugin @@ -9,5 +9,5 @@ website = https://getnikola.com/ description = Support for Mako templates. [Nikola] -PluginCategory = Template +PluginCategory = TemplateSystem diff --git a/requirements.txt b/requirements.txt index e4ef1a36a2..5e7f0b01b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ mako>=1.0.0 Markdown>=3.0.0 unidecode>=0.04.16 lxml>=3.3.5 -Yapsy>=1.12.0 PyRSS2Gen>=1.1 blinker>=1.3 setuptools>=24.2.0 diff --git a/snapcraft.yaml b/snapcraft.yaml index 50962eaa65..bd7ce8b5f8 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -40,7 +40,6 @@ parts: - mako - unidecode - lxml - - Yapsy - PyRSS2Gen - blinker - setuptools diff --git a/tests/data/plugin_manager/first.plugin b/tests/data/plugin_manager/first.plugin new file mode 100644 index 0000000000..cae4442025 --- /dev/null +++ b/tests/data/plugin_manager/first.plugin @@ -0,0 +1,13 @@ +[Core] +name = first +module = one + +[Documentation] +author = Chris Warrick +version = 1.0 +website = https://getnikola.com/ +description = Do one thing + +[Nikola] +PluginCategory = Command +compiler = foo diff --git a/tests/data/plugin_manager/one.py b/tests/data/plugin_manager/one.py new file mode 100644 index 0000000000..817bd0770c --- /dev/null +++ b/tests/data/plugin_manager/one.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2024 Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The first command.""" + +from nikola.plugin_categories import Command + + +class CommandOne(Command): + """The first command.""" + + name = "one" + doc_purpose = "do one thing" + doc_description = "Do a thing." + one_site_set = False + + def set_site(self, site): + super().set_site(site) + print("Site for 1 was set") + self.one_site_set = True + + def _execute(self, options, args): + """Run the command.""" + print("Hello world!") diff --git a/tests/data/plugin_manager/second/__init__.py b/tests/data/plugin_manager/second/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/data/plugin_manager/second/second.plugin b/tests/data/plugin_manager/second/second.plugin new file mode 100644 index 0000000000..ad1e4b8caa --- /dev/null +++ b/tests/data/plugin_manager/second/second.plugin @@ -0,0 +1,13 @@ +[Core] +name = 2nd +module = two + +[Documentation] +author = Chris Warrick +version = 1.0 +website = https://getnikola.com/ +description = Do another thing + +[Nikola] +PluginCategory = ConfigPlugin + diff --git a/tests/data/plugin_manager/second/two/__init__.py b/tests/data/plugin_manager/second/two/__init__.py new file mode 100644 index 0000000000..06df33b8e2 --- /dev/null +++ b/tests/data/plugin_manager/second/two/__init__.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2024 Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The second plugin.""" + +from nikola.plugin_categories import ConfigPlugin + + +class TwoConfigPlugin(ConfigPlugin): + """The first command.""" + + name = "2nd" + two_site_set = False + + def set_site(self, site): + super().set_site(site) + print("Site for 2 was set") + self.two_site_set = True diff --git a/tests/helper.py b/tests/helper.py index a39ad88f70..495c3d3a54 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -6,12 +6,13 @@ """ import os +import pathlib from contextlib import contextmanager -from yapsy.PluginManager import PluginManager import nikola.utils import nikola.shortcodes +from nikola.plugin_manager import PluginManager from nikola.plugin_categories import ( Command, Task, @@ -55,24 +56,11 @@ def __init__(self): "TRANSLATIONS": {"en": ""}, } self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"] - self.plugin_manager = PluginManager( - categories_filter={ - "Command": Command, - "Task": Task, - "LateTask": LateTask, - "TemplateSystem": TemplateSystem, - "PageCompiler": PageCompiler, - "TaskMultiplier": TaskMultiplier, - "CompilerExtension": CompilerExtension, - "MarkdownExtension": MarkdownExtension, - "RestExtension": RestExtension, - } - ) + places = [pathlib.Path(nikola.utils.__file__).parent / "plugins"] + self.plugin_manager = PluginManager(plugin_places=places) self.shortcode_registry = {} - self.plugin_manager.setPluginInfoExtension("plugin") - places = [os.path.join(os.path.dirname(nikola.utils.__file__), "plugins")] - self.plugin_manager.setPluginPlaces(places) - self.plugin_manager.collectPlugins() + candidates = self.plugin_manager.locate_plugins() + self.plugin_manager.load_plugins(candidates) self.compiler_extensions = self._activate_plugins_of_category( "CompilerExtension" ) @@ -88,13 +76,9 @@ def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in nikola/nikola.py plugins = [] - for plugin_info in self.plugin_manager.getPluginsOfCategory(category): - if plugin_info.name in self.config.get("DISABLED_PLUGINS"): - self.plugin_manager.removePluginFromCategory(plugin_info, category) - else: - self.plugin_manager.activatePluginByName(plugin_info.name) - plugin_info.plugin_object.set_site(self) - plugins.append(plugin_info) + for plugin_info in self.plugin_manager.get_plugins_of_category(category): + plugin_info.plugin_object.set_site(self) + plugins.append(plugin_info) return plugins def render_template(self, name, _, context): diff --git a/tests/test_plugin_manager.py b/tests/test_plugin_manager.py new file mode 100644 index 0000000000..380e6a22c5 --- /dev/null +++ b/tests/test_plugin_manager.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2024 Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import nikola.plugin_manager + +from .helper import FakeSite +from nikola.plugin_manager import PluginManager +from pathlib import Path + + +def test_locate_plugins_finds_core_plugins(): + """Ensure that locate_plugins can find some core plugins.""" + places = [Path(nikola.plugin_manager.__file__).parent / "plugins"] + plugin_manager = PluginManager(places) + candidates = plugin_manager.locate_plugins() + plugin_names = [p.name for p in candidates] + assert plugin_manager.candidates == candidates + + assert "emoji" in plugin_names + assert "copy_assets" in plugin_names + assert "scan_posts" in plugin_names + + template_plugins = [p for p in candidates if p.category == "TemplateSystem"] + template_plugins.sort(key=lambda p: p.name) + assert len(template_plugins) == 2 + assert template_plugins[0].name == "jinja" + assert template_plugins[1].name == "mako" + + +def test_locate_plugins_finds_core_and_custom_plugins(): + """Ensure that locate_plugins can find some custom plugins.""" + places = [ + Path(nikola.plugin_manager.__file__).parent / "plugins", + Path(__file__).parent / "data" / "plugin_manager", + ] + plugin_manager = PluginManager(places) + candidates = plugin_manager.locate_plugins() + plugin_names = [p.name for p in candidates] + assert plugin_manager.candidates == candidates + + assert "emoji" in plugin_names + assert "copy_assets" in plugin_names + assert "scan_posts" in plugin_names + + assert "first" in plugin_names + assert "2nd" in plugin_names + + first_plugin = next(p for p in candidates if p.name == "first") + second_plugin = next(p for p in candidates if p.name == "2nd") + + assert first_plugin.category == "Command" + assert first_plugin.compiler == "foo" + assert first_plugin.source_dir == places[1] + + assert second_plugin.category == "ConfigPlugin" + assert second_plugin.compiler == None + assert second_plugin.source_dir == places[1] / "second" + + +def test_load_plugins(): + """Ensure that locate_plugins can load some core and custom plugins.""" + places = [ + Path(nikola.plugin_manager.__file__).parent / "plugins", + Path(__file__).parent / "data" / "plugin_manager", + ] + plugin_manager = PluginManager(places) + candidates = plugin_manager.locate_plugins() + plugins_to_load = [p for p in candidates if p.name in {"first", "2nd", "emoji"}] + + plugin_manager.load_plugins(plugins_to_load) + + assert len(plugin_manager.plugins) == 3 + assert plugin_manager._plugins_by_category["ShortcodePlugin"][0].name == "emoji" + assert plugin_manager._plugins_by_category["Command"][0].name == "first" + assert plugin_manager._plugins_by_category["ConfigPlugin"][0].name == "2nd" + + site = FakeSite() + for plugin in plugin_manager.plugins: + plugin.plugin_object.set_site(site) + + assert "emoji" in site.shortcode_registry + assert plugin_manager.get_plugin_by_name("first", "Command").plugin_object.one_site_set + assert plugin_manager.get_plugin_by_name("2nd").plugin_object.two_site_set + assert plugin_manager.get_plugin_by_name("2nd", "Command") is None + + +def test_load_plugins_twice(): + """Ensure that extra plugins can be added.""" + places = [ + Path(nikola.plugin_manager.__file__).parent / "plugins", + Path(__file__).parent / "data" / "plugin_manager", + ] + plugin_manager = PluginManager(places) + candidates = plugin_manager.locate_plugins() + plugins_to_load_first = [p for p in candidates if p.name in {"first", "emoji"}] + plugins_to_load_second = [p for p in candidates if p.name in {"2nd"}] + + plugin_manager.load_plugins(plugins_to_load_first) + assert len(plugin_manager.plugins) == 2 + plugin_manager.load_plugins(plugins_to_load_second) + assert len(plugin_manager.plugins) == 3 diff --git a/tests/test_template_shortcodes.py b/tests/test_template_shortcodes.py index c6c948da97..148f7e6091 100644 --- a/tests/test_template_shortcodes.py +++ b/tests/test_template_shortcodes.py @@ -67,7 +67,7 @@ class ShortcodeFakeSite(Nikola): def _get_template_system(self): if self._template_system is None: # Load template plugin - self._template_system = self.plugin_manager.getPluginByName( + self._template_system = self.plugin_manager.get_plugin_by_name( "jinja", "TemplateSystem" ).plugin_object self._template_system.set_directories(".", "cache") From 54bfb241458112f5a1c2baa12c2d3c19bc98c2e1 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:16:57 +0100 Subject: [PATCH 02/15] Enable CI on Python 3.12 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7476c20776..89196747fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] image: - ubuntu-latest include: @@ -84,7 +84,7 @@ jobs: strategy: matrix: python: - - '3.11' + - '3.12' runs-on: ubuntu-latest steps: - name: Check out code @@ -111,7 +111,7 @@ jobs: strategy: matrix: python: - - '3.11' + - '3.12' runs-on: ubuntu-latest steps: - name: Check out code From b68afce1b5602a1ce14c404758edea011862aa51 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:18:38 +0100 Subject: [PATCH 03/15] Changelog entry --- CHANGES.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 12e58f07d3..1f4f76f773 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,11 +4,16 @@ New in master Features -------- +* Implement a new plugin manager from scratch to replace Yapsy, + which does not work on Python 3.12 due to Python 3.12 carelessly + removing parts of the standard library (Issue #3719) * Support for Discourse as comment system (Issue #3689) Bugfixes -------- +* Fix loading of templates from plugins with ``__init__.py`` files + (Issue #3725) * Fix margins of paragraphs at the end of sections (Issue #3704) * Ignore ``.DS_Store`` files in listing indexes (Issue #3698) * Fix baguetteBox.js invoking in the base theme (Issue #3687) From 3afd07b1b22b3f11a5d3b86c7e688e43ee47b50d Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:28:01 +0100 Subject: [PATCH 04/15] Code style fixes --- nikola/nikola.py | 14 -------------- nikola/plugin_categories.py | 1 + nikola/plugin_manager.py | 13 +++++++++---- tests/helper.py | 14 +------------- tests/test_plugin_manager.py | 2 +- 5 files changed, 12 insertions(+), 32 deletions(-) diff --git a/nikola/nikola.py b/nikola/nikola.py index 4accadcc44..dd39e82d93 100644 --- a/nikola/nikola.py +++ b/nikola/nikola.py @@ -55,20 +55,7 @@ from .post import Post # NOQA from .plugin_manager import PluginCandidate, PluginInfo, PluginManager from .plugin_categories import ( - Command, - LateTask, - PageCompiler, - CompilerExtension, - MarkdownExtension, - RestExtension, - MetadataExtractor, - ShortcodePlugin, - Task, - TaskMultiplier, TemplateSystem, - SignalHandler, - ConfigPlugin, - CommentSystem, PostScanner, Taxonomy, ) @@ -1050,7 +1037,6 @@ def init_plugins(self, commands_only=False, load_all=False): ] + [path for path in extra_plugins_dirs if path] self._plugin_places = [pathlib.Path(p) for p in self._plugin_places] - self.plugin_manager = PluginManager(plugin_places=self._plugin_places) compilers = defaultdict(set) diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py index 2abef7ed11..980f9106a9 100644 --- a/nikola/plugin_categories.py +++ b/nikola/plugin_categories.py @@ -885,6 +885,7 @@ def get_other_language_variants(self, classification: str, lang: str, classifica """ return [] + CATEGORIES = { "Command": Command, "Task": Task, diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index a9a95bf6f0..f1ec91f23a 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -142,6 +142,7 @@ def locate_plugins(self, force=False) -> List[PluginCandidate]: return self.candidates def load_plugins(self, candidates: List[PluginCandidate]) -> None: + """Load selected candidate plugins.""" plugins_root = Path(__file__).parent.parent for candidate in candidates: @@ -178,7 +179,7 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: module_object = importlib.util.module_from_spec(spec) sys.modules[full_module_name] = module_object spec.loader.exec_module(module_object) - except Exception as exc: + except Exception: self.logger.exception(f"{plugin_id} threw an exception while loading") continue @@ -195,7 +196,7 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: continue try: plugin_object = plugin_classes[0]() - except Exception as exc: + except Exception: self.logger.exception(f"{plugin_id} threw an exception while creating an instance") continue self.logger.debug(f"Loaded {plugin_id}") @@ -217,16 +218,20 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: self._plugins_by_category[plugin_info.category].append(plugin_info) def get_plugins_of_category(self, category: str) -> List[PluginInfo]: + """Get loaded plugins of a given category.""" return self._plugins_by_category.get(category, []) - def get_plugin_by_name(self, name: str, category: str | None = None) -> PluginInfo | None: + def get_plugin_by_name(self, name: str, category: Optional[str] = None) -> Optional[PluginInfo]: + """Get a loaded plugin by name and optionally by category. Returns None if no such plugin is loaded.""" for p in self.plugins: if p.name == name and (category is None or p.category == category): return p # Aliases for Yapsy compatibility def getPluginsOfCategory(self, category: str) -> List[PluginInfo]: + """Get loaded plugins of a given category.""" return self._plugins_by_category.get(category, []) - def getPluginByName(self, name: str, category: str | None = None) -> PluginInfo | None: + def getPluginByName(self, name: str, category: Optional[str] = None) -> Optional[PluginInfo]: + """Get a loaded plugin by name and optionally by category. Returns None if no such plugin is loaded.""" return self.get_plugin_by_name(name, category) diff --git a/tests/helper.py b/tests/helper.py index 495c3d3a54..8fbebdb45b 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -9,21 +9,9 @@ import pathlib from contextlib import contextmanager - -import nikola.utils import nikola.shortcodes +import nikola.utils from nikola.plugin_manager import PluginManager -from nikola.plugin_categories import ( - Command, - Task, - LateTask, - TemplateSystem, - PageCompiler, - TaskMultiplier, - CompilerExtension, - MarkdownExtension, - RestExtension, -) __all__ = ["cd", "FakeSite"] diff --git a/tests/test_plugin_manager.py b/tests/test_plugin_manager.py index 380e6a22c5..a4039c5a27 100644 --- a/tests/test_plugin_manager.py +++ b/tests/test_plugin_manager.py @@ -76,7 +76,7 @@ def test_locate_plugins_finds_core_and_custom_plugins(): assert first_plugin.source_dir == places[1] assert second_plugin.category == "ConfigPlugin" - assert second_plugin.compiler == None + assert second_plugin.compiler is None assert second_plugin.source_dir == places[1] / "second" From 58817d3be9ba9035841add6a85d3d036b9e36213 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:31:39 +0100 Subject: [PATCH 05/15] Docstrings --- nikola/plugin_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index f1ec91f23a..a34b03f506 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -52,6 +52,7 @@ @dataclass(frozen=True) class PluginCandidate: + """A candidate plugin that was located but not yet loaded (imported).""" name: str description: Optional[str] plugin_id: str @@ -63,6 +64,7 @@ class PluginCandidate: @dataclass(frozen=True) class PluginInfo: + """A plugin that was loaded (imported).""" name: str description: Optional[str] plugin_id: str From f7f457d8afdf8ddc8c7870767d1fe8f50d072629 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:31:49 +0100 Subject: [PATCH 06/15] Fix world's dumbest deprecation --- nikola/plugins/task/sitemap.py | 2 +- nikola/post.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nikola/plugins/task/sitemap.py b/nikola/plugins/task/sitemap.py index bc9a5d6569..3c045eb0df 100644 --- a/nikola/plugins/task/sitemap.py +++ b/nikola/plugins/task/sitemap.py @@ -309,7 +309,7 @@ def get_lastmod(self, p): # RFC 3339 (web ISO 8601 profile) represented in UTC with Zulu # zone desgignator as recommeded for sitemaps. Second and # microsecond precision is stripped for compatibility. - lastmod = datetime.datetime.utcfromtimestamp(os.stat(p).st_mtime).replace(tzinfo=dateutil.tz.gettz('UTC'), second=0, microsecond=0).isoformat().replace('+00:00', 'Z') + lastmod = datetime.datetime.fromtimestamp(os.stat(p).st_mtime, dateutil.tz.tzutc()).replace(second=0, microsecond=0).isoformat().replace('+00:00', 'Z') return lastmod diff --git a/nikola/post.py b/nikola/post.py index 1dcd33d609..d4172d548b 100644 --- a/nikola/post.py +++ b/nikola/post.py @@ -350,8 +350,8 @@ def _set_date(self, default_metadata): if self.config['__invariant__']: default_metadata['date'] = datetime.datetime(2013, 12, 31, 23, 59, 59, tzinfo=self.config['__tzinfo__']) else: - default_metadata['date'] = datetime.datetime.utcfromtimestamp( - os.stat(self.source_path).st_ctime).replace(tzinfo=dateutil.tz.tzutc()).astimezone(self.config['__tzinfo__']) + default_metadata['date'] = datetime.datetime.fromtimestamp( + os.stat(self.source_path).st_ctime, dateutil.tz.tzutc()).astimezone(self.config['__tzinfo__']) # If time zone is set, build localized datetime. try: From c4fe3dde169b2cf6718e7ff5c7c02c6e3817221d Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:50:14 +0100 Subject: [PATCH 07/15] Avoid sys.modules reuse and replacement to fix tests (That suggests something is wrong with some plugins' initialization.) --- nikola/plugin_manager.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index a34b03f506..5420c8deb8 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -94,12 +94,8 @@ def __init__(self, plugin_places: List[Path]): self._plugins_by_category = {} self.logger = get_logger("PluginManager") - def locate_plugins(self, force=False) -> List[PluginCandidate]: + def locate_plugins(self) -> List[PluginCandidate]: """Locate plugins in plugin_places.""" - if self.candidates and not force: - # Already located - return self.candidates - self.candidates = [] plugin_files: List[Path] = [] @@ -172,18 +168,15 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: except ValueError: pass - if full_module_name.startswith("nikola.plugins") and full_module_name in sys.modules: - # Loaded by something else (a dependent plugin?) - module_object = sys.modules[full_module_name] - else: - try: - spec = importlib.util.spec_from_file_location(full_module_name, py_file_location) - module_object = importlib.util.module_from_spec(spec) + try: + spec = importlib.util.spec_from_file_location(full_module_name, py_file_location) + module_object = importlib.util.module_from_spec(spec) + if full_module_name not in sys.modules: sys.modules[full_module_name] = module_object - spec.loader.exec_module(module_object) - except Exception: - self.logger.exception(f"{plugin_id} threw an exception while loading") - continue + spec.loader.exec_module(module_object) + except Exception: + self.logger.exception(f"{plugin_id} threw an exception while loading") + continue plugin_classes = [ c From 9dedec30b724c157c578745771ee043f196f86e9 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:52:11 +0100 Subject: [PATCH 08/15] Make pydocstyle happy --- nikola/plugin_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index 5420c8deb8..47db9f6df4 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -53,6 +53,7 @@ @dataclass(frozen=True) class PluginCandidate: """A candidate plugin that was located but not yet loaded (imported).""" + name: str description: Optional[str] plugin_id: str @@ -65,6 +66,7 @@ class PluginCandidate: @dataclass(frozen=True) class PluginInfo: """A plugin that was loaded (imported).""" + name: str description: Optional[str] plugin_id: str From 7ab4aa4ba486d72a6d93e6b3f675355b2f427554 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:56:34 +0100 Subject: [PATCH 09/15] Improve testing matrix --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89196747fb..ae5f499fa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,9 +19,9 @@ jobs: image: - ubuntu-latest include: - - python: '3.11' + - python: '3.12' image: macos-latest - - python: '3.11' + - python: '3.12' image: windows-latest runs-on: '${{ matrix.image }}' steps: @@ -111,6 +111,7 @@ jobs: strategy: matrix: python: + - '3.11' - '3.12' runs-on: ubuntu-latest steps: From 44d923f13a9af286084e6991c8ad3d73f67f0dff Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Wed, 3 Jan 2024 00:59:52 +0100 Subject: [PATCH 10/15] List 3.12 in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f2f5615ad8..5db7f89474 100755 --- a/setup.py +++ b/setup.py @@ -133,6 +133,7 @@ def run(self): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Internet', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Text Processing :: Markup'], From 5f09c0881addf64e56591d29c4bd61cdb8bdc57c Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Fri, 5 Jan 2024 00:05:08 +0100 Subject: [PATCH 11/15] Document PluginCategory requirement --- CHANGES.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 1f4f76f773..63870f5cca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -21,6 +21,14 @@ Bugfixes for non-root SITE_URL, in particular when URL_TYPE is full_path. (Issue #3715) +For plugin developers +--------------------- + +Nikola now requires the ``.plugin`` file to contain a ``[Nikola]`` +section with a ``PluginCategory`` entry set to the name of the plugin +category class. This was already required by ``plugins.getnikola.com``, +but you may have custom plugins that don’t have this set. + New in v8.2.4 ============= From f4837e1efa948fee910a47fc0b5922a125dccde7 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Fri, 5 Jan 2024 20:53:22 +0100 Subject: [PATCH 12/15] Add deprecation warnings to getPluginsOfCategory/getPluginByName --- nikola/plugin_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index 47db9f6df4..f2e5f05f23 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -225,10 +225,13 @@ def get_plugin_by_name(self, name: str, category: Optional[str] = None) -> Optio return p # Aliases for Yapsy compatibility + # TODO: remove in v9 def getPluginsOfCategory(self, category: str) -> List[PluginInfo]: """Get loaded plugins of a given category.""" + self.logger.warning("Legacy getPluginsOfCategory method was used, it may be removed in the future. Please change it to get_plugins_of_category.") return self._plugins_by_category.get(category, []) def getPluginByName(self, name: str, category: Optional[str] = None) -> Optional[PluginInfo]: """Get a loaded plugin by name and optionally by category. Returns None if no such plugin is loaded.""" + self.logger.warning("Legacy getPluginByName method was used, it may be removed in the future. Please change it to get_plugin_by_name.") return self.get_plugin_by_name(name, category) From 15e86a45b376fcd19938ad1bbee013adc137ccc9 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Fri, 5 Jan 2024 20:54:34 +0100 Subject: [PATCH 13/15] Reject plugins if .plugin category does not match reality, improve warning messages --- nikola/plugin_manager.py | 20 ++++++--- tests/data/plugin_manager/broken.plugin | 12 ++++++ tests/data/plugin_manager/broken.py | 42 +++++++++++++++++++ tests/data/plugin_manager/second/__init__.py | 0 .../plugin_manager/second/two/__init__.py | 2 +- tests/test_plugin_manager.py | 18 ++++++++ 6 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 tests/data/plugin_manager/broken.plugin create mode 100644 tests/data/plugin_manager/broken.py delete mode 100644 tests/data/plugin_manager/second/__init__.py diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index f2e5f05f23..e56c6e652c 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -115,17 +115,18 @@ def locate_plugins(self) -> List[PluginCandidate]: if "Documentation" in config: description = config["Documentation"].get("Description") if "Nikola" not in config: - self.logger.warning(f"{plugin_id} does not specify Nikola configuration - it will not be loaded") + self.logger.warning(f"{plugin_id} does not specify Nikola configuration - it will not be loaded. " + "Please add a [Nikola] section to the .plugin file with a PluginCategory entry.") continue category = config["Nikola"].get("PluginCategory") compiler = config["Nikola"].get("Compiler") if not category: - self.logger.warning(f"{plugin_id} does not specify any category - it will not be loaded") + self.logger.warning(f"{plugin_id} does not specify any category (Nikola.PluginCategory in .plugin file) - it will not be loaded") continue if category in LEGACY_PLUGIN_NAMES: category = LEGACY_PLUGIN_NAMES[category] if category not in CATEGORY_NAMES: - self.logger.warning(f"{plugin_id} specifies invalid category '{category}'") + self.logger.warning(f"{plugin_id} specifies invalid category '{category}' in the .plugin file - it will not be loaded") continue self.logger.debug(f"Discovered {plugin_id}") self.candidates.append( @@ -186,13 +187,20 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: if isinstance(c, type) and issubclass(c, BasePlugin) and c not in CATEGORY_TYPES ] if len(plugin_classes) == 0: - self.logger.warning(f"{plugin_id} does not have any plugin classes") + self.logger.warning(f"{plugin_id} does not have any plugin classes - plugin will not be loaded") continue elif len(plugin_classes) > 1: - self.logger.warning(f"{plugin_id} has multiple plugin classes; this is not supported - skipping") + self.logger.warning(f"{plugin_id} has multiple plugin classes; this is not supported - plugin will not be loaded") continue + + plugin_class = plugin_classes[0] + + if not issubclass(plugin_class, CATEGORIES[candidate.category]): + self.logger.warning(f"{plugin_id} has category '{candidate.category}' in the .plugin file, but the implementation class {plugin_class} does not inherit from this category - plugin will not be loaded") + continue + try: - plugin_object = plugin_classes[0]() + plugin_object = plugin_class() except Exception: self.logger.exception(f"{plugin_id} threw an exception while creating an instance") continue diff --git a/tests/data/plugin_manager/broken.plugin b/tests/data/plugin_manager/broken.plugin new file mode 100644 index 0000000000..1a5f9e05ce --- /dev/null +++ b/tests/data/plugin_manager/broken.plugin @@ -0,0 +1,12 @@ +[Core] +name = broken +module = broken + +[Documentation] +author = Chris Warrick +version = 1.0 +website = https://getnikola.com/ +description = Broken (wrong category) + +[Nikola] +PluginCategory = Task diff --git a/tests/data/plugin_manager/broken.py b/tests/data/plugin_manager/broken.py new file mode 100644 index 0000000000..68af857cb5 --- /dev/null +++ b/tests/data/plugin_manager/broken.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2024 Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The second plugin.""" + +from nikola.plugin_categories import TemplateSystem + + +class BrokenPlugin(TemplateSystem): + """The TemplateSystem plugin whose .plugin file says it’s a Task plugin.""" + + name = "broken" + broken_site_set = False + + def set_site(self, site): + super().set_site(site) + print("Site for broken was set") + self.broken_site_set = True + raise Exception("Site for broken was set") diff --git a/tests/data/plugin_manager/second/__init__.py b/tests/data/plugin_manager/second/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/data/plugin_manager/second/two/__init__.py b/tests/data/plugin_manager/second/two/__init__.py index 06df33b8e2..14794c07ec 100644 --- a/tests/data/plugin_manager/second/two/__init__.py +++ b/tests/data/plugin_manager/second/two/__init__.py @@ -30,7 +30,7 @@ class TwoConfigPlugin(ConfigPlugin): - """The first command.""" + """The second plugin.""" name = "2nd" two_site_set = False diff --git a/tests/test_plugin_manager.py b/tests/test_plugin_manager.py index a4039c5a27..8deb7a60b6 100644 --- a/tests/test_plugin_manager.py +++ b/tests/test_plugin_manager.py @@ -122,3 +122,21 @@ def test_load_plugins_twice(): assert len(plugin_manager.plugins) == 2 plugin_manager.load_plugins(plugins_to_load_second) assert len(plugin_manager.plugins) == 3 + + +def test_load_plugins_skip_mismatching_category(caplog): + """If a plugin specifies a different category than it actually implements, refuse to load it.""" + places = [ + Path(__file__).parent / "data" / "plugin_manager", + ] + plugin_manager = PluginManager(places) + candidates = plugin_manager.locate_plugins() + plugins_to_load = [p for p in candidates if p.name in {"broken"}] + plugin_to_load = plugins_to_load[0] + assert len(plugins_to_load) == 1 + + plugin_manager.load_plugins(plugins_to_load) + + py_file = plugin_to_load.source_dir / "broken.py" + assert f"{plugin_to_load.plugin_id} ({py_file}) has category '{plugin_to_load.category}' in the .plugin file, but the implementation class does not inherit from this category - plugin will not be loaded" in caplog.text + assert len(plugin_manager.plugins) == 0 From 0c6db619afe565eafb2c3acdce0e6d7fa67a1656 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Fri, 5 Jan 2024 21:01:42 +0100 Subject: [PATCH 14/15] Add delay and warning message if plugins are invalid --- nikola/plugin_manager.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index e56c6e652c..c38be4d245 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -29,6 +29,7 @@ import configparser import importlib import importlib.util +import time import sys from dataclasses import dataclass from pathlib import Path @@ -87,6 +88,7 @@ class PluginManager: candidates: List[PluginCandidate] plugins: List[PluginInfo] _plugins_by_category: Dict[str, List[PluginInfo]] + has_warnings: bool = False def __init__(self, plugin_places: List[Path]): """Initialize the plugin manager.""" @@ -115,18 +117,21 @@ def locate_plugins(self) -> List[PluginCandidate]: if "Documentation" in config: description = config["Documentation"].get("Description") if "Nikola" not in config: - self.logger.warning(f"{plugin_id} does not specify Nikola configuration - it will not be loaded. " - "Please add a [Nikola] section to the .plugin file with a PluginCategory entry.") + self.logger.warning(f"{plugin_id} does not specify Nikola configuration - plugin will not be loaded") + self.logger.warning("Please add a [Nikola] section to the {plugin_file} file with a PluginCategory entry") + self.has_warnings = True continue category = config["Nikola"].get("PluginCategory") compiler = config["Nikola"].get("Compiler") if not category: - self.logger.warning(f"{plugin_id} does not specify any category (Nikola.PluginCategory in .plugin file) - it will not be loaded") + self.logger.warning(f"{plugin_id} does not specify any category (Nikola.PluginCategory is missing in .plugin file) - plugin will not be loaded") + self.has_warnings = True continue if category in LEGACY_PLUGIN_NAMES: category = LEGACY_PLUGIN_NAMES[category] if category not in CATEGORY_NAMES: - self.logger.warning(f"{plugin_id} specifies invalid category '{category}' in the .plugin file - it will not be loaded") + self.logger.warning(f"{plugin_id} specifies invalid category '{category}' in the .plugin file - plugin will not be loaded") + self.has_warnings = True continue self.logger.debug(f"Discovered {plugin_id}") self.candidates.append( @@ -156,6 +161,7 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: py_file_location = source_dir / module_name / "__init__.py" if not py_file_location.exists(): self.logger.warning(f"{plugin_id} could not be loaded (no valid module detected)") + self.has_warnings = True continue plugin_id += f" ({py_file_location})" @@ -179,6 +185,7 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: spec.loader.exec_module(module_object) except Exception: self.logger.exception(f"{plugin_id} threw an exception while loading") + self.has_warnings = True continue plugin_classes = [ @@ -188,21 +195,25 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: ] if len(plugin_classes) == 0: self.logger.warning(f"{plugin_id} does not have any plugin classes - plugin will not be loaded") + self.has_warnings = True continue elif len(plugin_classes) > 1: self.logger.warning(f"{plugin_id} has multiple plugin classes; this is not supported - plugin will not be loaded") + self.has_warnings = True continue plugin_class = plugin_classes[0] if not issubclass(plugin_class, CATEGORIES[candidate.category]): self.logger.warning(f"{plugin_id} has category '{candidate.category}' in the .plugin file, but the implementation class {plugin_class} does not inherit from this category - plugin will not be loaded") + self.has_warnings = True continue try: plugin_object = plugin_class() except Exception: - self.logger.exception(f"{plugin_id} threw an exception while creating an instance") + self.logger.exception(f"{plugin_id} threw an exception while creating the instance") + self.has_warnings = True continue self.logger.debug(f"Loaded {plugin_id}") info = PluginInfo( @@ -222,6 +233,13 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: for plugin_info in self.plugins: self._plugins_by_category[plugin_info.category].append(plugin_info) + if self.has_warnings: + self.logger.warning("Some plugins failed to load. Please review the above warning messages.") + # TODO remove following messages and delay in v8.3.1 + self.logger.warning("You may need to update some plugins (from plugins.getnikola.com) or to fix their .plugin files.") + self.logger.warning("Waiting 2 seconds before continuing.") + time.sleep(2) + def get_plugins_of_category(self, category: str) -> List[PluginInfo]: """Get loaded plugins of a given category.""" return self._plugins_by_category.get(category, []) From 01107d9b862065c3c4ed510b158635e8dbffefb3 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Sat, 6 Jan 2024 11:14:20 +0100 Subject: [PATCH 15/15] Add remove in v9 to second legacy method --- nikola/plugin_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index c38be4d245..5c1a29a103 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -257,6 +257,7 @@ def getPluginsOfCategory(self, category: str) -> List[PluginInfo]: self.logger.warning("Legacy getPluginsOfCategory method was used, it may be removed in the future. Please change it to get_plugins_of_category.") return self._plugins_by_category.get(category, []) + # TODO: remove in v9 def getPluginByName(self, name: str, category: Optional[str] = None) -> Optional[PluginInfo]: """Get a loaded plugin by name and optionally by category. Returns None if no such plugin is loaded.""" self.logger.warning("Legacy getPluginByName method was used, it may be removed in the future. Please change it to get_plugin_by_name.")