diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..53900a3 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,93 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Push tests +# run-name: ${{ github.actor }} push tests + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: +# - python: 2.7.18 +# plone: 4.3 +# - python: 3.7.14 +# plone: 5.2 +# - python: 3.10.11 +# plone: "6.0" + - python: 3.13.1 + plone: "6.1" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up pyenv and Python + uses: "gabrielfalcao/pyenv-action@v18" + with: + default: "${{ matrix.python }}" + - name: Setup Env + run: | + pip install --upgrade pip + pip install -r requirements-${{ matrix.plone }}.txt + - name: Cache eggs + uses: actions/cache@v4 + env: + cache-name: cache-eggs + with: + path: ~/buildout-cache/eggs + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ matrix.python }}-${{ matrix.plone }} + - name: buildout + run: | + sed -ie "s#test.cfg#test-${{matrix.plone}}.cfg#" gha.cfg + buildout -c gha.cfg annotate + buildout -c gha.cfg + - name: test + run: | + bin/test -t !robot + coverage: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: + - python: 3.13.1 + plone: "6.1" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up pyenv and Python + uses: "gabrielfalcao/pyenv-action@v18" + with: + default: "${{ matrix.python }}" + - name: Setup Env + run: | + pip install --upgrade pip + pip install -r requirements-${{matrix.plone}}.txt + pip install -U coveralls coverage + - name: Cache eggs + uses: actions/cache@v4 + env: + cache-name: cache-eggs + with: + path: ~/buildout-cache/eggs + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ matrix.python }}-${{ matrix.plone }} + - name: buildout + run: | + sed -ie "s#test.cfg#test-${{matrix.plone}}.cfg#" gha.cfg + buildout -c gha.cfg + - name: code-analysis + run: | + bin/code-analysis + - name: test coverage + run: | + coverage run bin/test -t !robot + - name: Publish to Coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coveralls --service=github diff --git a/.gitignore b/.gitignore index 9ce0a51..e7de298 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.pyc .installed.cfg .mr.developer.cfg +.plone.versioncheck.tracked.json .project .pydevproject .settings/ @@ -17,3 +18,5 @@ src/* !src/imio var/ .coverage +pyvenv.cfg +.idea diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..6f73421 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,5 @@ +[settings] +force_alphabetical_sort = True +force_single_line = True +lines_after_imports = 2 +line_length = 120 diff --git a/CHANGES.rst b/CHANGES.rst index d396fb9..b6c8d8c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,11 @@ Changelog ========= -1.39 (unreleased) ------------------ - -- Nothing changed yet. +2.0 (unreleased) +---------------- +- Add Plone 6.1 compatibility, drop Plone 4 / 5 compatibility + [laulaz] 1.38 (2024-10-02) ----------------- diff --git a/Makefile b/Makefile index ceda8f2..1dd1f20 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,94 @@ #!/usr/bin/make -# -all: run +# pyenv is a requirement, with 3.13+ python versions, and virtualenv installed in each version +# plone parameter must be passed to create environment 'make setup plone=6.1' or after a make cleanall +# The original Makefile can be found on https://github.com/IMIO/scripts-buildout -.PHONY: bootstrap -bootstrap: - virtualenv-2.7 --no-site-packages . - ./bin/pip install -r requirements.txt +SHELL=/bin/bash +plones=6.1 +b_o= +old_plone=$(shell [ -e .plone-version ] && cat .plone-version) + +ifeq (, $(shell which pyenv)) + $(error "pyenv command not found! Aborting") +endif + +ifndef plone +ifeq (,$(filter setup,$(MAKECMDGOALS))) + plone=$(old_plone) +endif +endif + +ifneq ($(wildcard bin/instance),) + b_o=-N +endif + +ifndef python +ifeq ($(plone),6.1) + python=3.13 +endif +endif + +all: buildout + +.PHONY: help +help: + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) + +.python-version: ## Setups pyenv version + @pyenv local `pyenv versions |grep " $(python)" |tail -1 |xargs` + @echo "Local pyenv version is `cat .python-version`" + @ if [[ `pyenv which virtualenv` != `pyenv prefix`* ]] ; then echo "You need to install virtualenv in `cat .python-version` pyenv python (pip install virtualenv)"; exit 1; fi + +bin/buildout: .python-version ## Setups environment + virtualenv . + ./bin/pip install --upgrade pip + ./bin/pip install -r requirements-$(plone).txt + @echo "$(plone)" > .plone-version + +.PHONY: setup +setup: oneof-plone backup cleanall bin/buildout restore ## Setups environment .PHONY: buildout -buildout: - if ! test -f bin/buildout;then make bootstrap;fi - bin/buildout -t 5 +buildout: oneof-plone bin/buildout ## Runs setup and buildout + rm -f .installed.cfg .mr.developer.cfg + bin/buildout -t 5 -c test-$(plone).cfg ${b_o} -.PHONY: run -run: - if ! test -f bin/instance1;then make buildout;fi - bin/instance1 fg +.PHONY: test +test: oneof-plone bin/buildout ## run bin/test without robot + # can be run by example with: make test opt='-t "settings"' + bin/test -t \!robot ${opt} .PHONY: cleanall -cleanall: - rm -fr bin develop-eggs downloads eggs lib parts .installed.cfg +cleanall: ## Cleans all installed buildout files + rm -fr bin include lib local share develop-eggs downloads eggs parts .installed.cfg .mr.developer.cfg .python-version pyvenv.cfg + +.PHONY: backup +backup: ## Backups db files + @if [ '$(old_plone)' != '' ] && [ -f var/filestorage/Data.fs ]; then mv var/filestorage/Data.fs var/filestorage/Data.fs.$(old_plone); mv var/blobstorage var/blobstorage.$(old_plone); fi + +.PHONY: restore +restore: ## Restores db files + @if [ '$(plone)' != '' ] && [ -f var/filestorage/Data.fs.$(plone) ]; then mv var/filestorage/Data.fs.$(plone) var/filestorage/Data.fs; mv var/blobstorage.$(plone) var/blobstorage; fi + +.PHONY: which-python +which-python: oneof-plone ## Displays versions information + @echo "current plone = $(old_plone)" + @echo "current python = `cat .python-version`" + @echo "plone var = $(plone)" + @echo "python var = $(python)" + +.PHONY: vcr +vcr: ## Shows requirements in checkversion-r.html + @bin/versioncheck -rbo checkversion-r-$(plone).html test-$(plone).cfg + +.PHONY: vcn +vcn: ## Shows newer packages in checkversion-n.html + @bin/versioncheck -npbo checkversion-n-$(plone).html test-$(plone).cfg + +.PHONY: guard-% +guard-%: + @ if [ "${${*}}" = "" ]; then echo "You must give a value for variable '$*' : like $*=xxx"; exit 1; fi + +.PHONY: oneof-% +oneof-%: + @ if ! echo "${${*}s}" | tr " " '\n' |grep -Fqx "${${*}}"; then echo "Invalid '$*' parameter ('${${*}}') : must be one of '${${*}s}'"; exit 1; fi diff --git a/base.cfg b/base.cfg index cc727dd..8f03d9c 100644 --- a/base.cfg +++ b/base.cfg @@ -1,38 +1,99 @@ [buildout] -show-picked-versions = true -allow-picked-versions = false +package-name = imio.migrator +package-extras = [test] extends = - http://dist.plone.org/release/4.3-latest/versions.cfg - sources.cfg - versions.cfg + https://raw.githubusercontent.com/collective/buildout.plonetest/master/qa.cfg + checkouts.cfg -extensions += +extensions = mr.developer + plone.versioncheck -parts = - instance1 +parts += + instance omelette + ploneversioncheck + createcoverage + robot +# coverage +# test-coverage +# plone-helper-scripts develop = . eggs += + Plone + Pillow +# Products.PDBDebugMode +# collective.profiler +# ipdb + pdbp +# plone.reload + +package-extras += + pdbp always-checkout = force -auto-checkout += -[instance1] -recipe = plone.recipe.zope2instance -user = admin:admin -http-address = 8081 -eggs = +[instance] +environment-vars += + PYTHONBREAKPOINT pdbp.set_trace +eggs += ${buildout:eggs} - imio.migrator -zcml = -environment-vars = zope_i18n_compile_mo_files true +zcml += + +[test] +initialization += + os.environ['PYTHONBREAKPOINT'] = 'pdbp.set_trace' [omelette] recipe = collective.recipe.omelette +eggs = ${test:eggs} + +[ploneversioncheck] +recipe = zc.recipe.egg +eggs = plone.versioncheck + +[code-analysis] +recipe = plone.recipe.codeanalysis +pre-commit-hook = True +return-status-codes = True +directory = ${buildout:directory}/src/imio/migrator +flake8-ignore = E123,E124,E501,E126,E127,E128,W391,C901,W503,W504 +flake8-extensions = + flake8-isort + +[robot] +recipe = zc.recipe.egg eggs = - ${buildout:eggs} - ${instance1:eggs} + Pillow + ${test:eggs} + plone.app.robotframework[reload, debug] + +[coverage] +recipe = zc.recipe.egg +eggs = coverage + +[test-coverage] +recipe = collective.recipe.template +input = inline: + #!/bin/bash + export TZ=UTC + ${buildout:directory}/bin/coverage run bin/test $* + ${buildout:directory}/bin/coverage html + ${buildout:directory}/bin/coverage report -m --fail-under=90 + # Fail (exit status 1) if coverage returns exit status 2 (this happens + # when test coverage is below 100%. +output = ${buildout:directory}/bin/test-coverage +mode = 755 + +[plone-helper-scripts] +recipe = zc.recipe.egg +eggs = + Products.CMFPlone + ${instance:eggs} +interpreter = zopepy +scripts = + zopepy + plone-compile-resources diff --git a/checkouts.cfg b/checkouts.cfg new file mode 100644 index 0000000..a0a5284 --- /dev/null +++ b/checkouts.cfg @@ -0,0 +1,19 @@ +[buildout] +always-checkout = force +auto-checkout += + +[remotes] +imio = https://github.com/imio +imio_push = git@github.com:imio +plone = https://github.com/plone +plone_push = git@github.com:plone +ftw = https://github.com/4teamwork +ftw_push = git@github.com:4teamwork +zopefoundation = https://github.com/zopefoundation +zopefoundation_push = git@github.com:zopefoundation +zopesvn = svn://svn.zope.org/repos/main/ +col = https://github.com/collective +col_push = git@github.com:collective + +[sources] +imio.helpers = git ${remotes:imio}/imio.helpers.git pushurl=${remotes:imio_push}/imio.helpers.git diff --git a/dev.cfg b/dev.cfg deleted file mode 100644 index d4d40cc..0000000 --- a/dev.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[buildout] -extends = - base.cfg - -parts += - test - -show-picked-versions = true - -auto-checkout += - -[instance1] -debug-mode = on -verbose-security = on -eggs += - plone.reload -# Products.PDBDebugMode - Products.DocFinderTab - aws.zope2zcmldoc - collective.profiler - ipdb - iw.debug - -zcml += - iw.debug - -[test] -recipe = zc.recipe.testrunner -eggs = - ${buildout:eggs} - imio.migrator [test] -defaults = ['-s', 'imio.migrator', '--auto-color', '--auto-progress'] diff --git a/gha.cfg b/gha.cfg new file mode 100644 index 0000000..ef2e218 --- /dev/null +++ b/gha.cfg @@ -0,0 +1,5 @@ +[buildout] +extends = + test.cfg +eggs-directory = ~/buildout-cache/eggs +download-cache = ~/buildout-cache/downloads diff --git a/requirements-6.1.txt b/requirements-6.1.txt new file mode 100644 index 0000000..aaf0495 --- /dev/null +++ b/requirements-6.1.txt @@ -0,0 +1,3 @@ +-c https://dist.plone.org/release/6.1-latest/requirements.txt +setuptools +zc.buildout diff --git a/setup.py b/setup.py index c8451d4..68e81e6 100644 --- a/setup.py +++ b/setup.py @@ -5,13 +5,12 @@ from setuptools import setup -long_description = ( - open('README.rst').read() + '\n' + open('CHANGES.rst').read() + '\n') +long_description = open("README.rst").read() + "\n" + open("CHANGES.rst").read() + "\n" setup( - name='imio.migrator', - version='1.39.dev0', + name="imio.migrator", + version="2.0.dev0", description="Migration helper tool", long_description=long_description, # Get more from http://pypi.python.org/pypi?%3Aaction=list_classifiers @@ -19,32 +18,35 @@ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", "Framework :: Plone", - "Framework :: Plone :: 4.3", + "Framework :: Plone :: 6.1", "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.13", ], - keywords='migration helpers', - author='IMIO', - author_email='dev@imio.be', - url='https://github.com/imio/imio.migrator', - download_url='https://pypi.org/project/imio.migrator', - license='GPL', - packages=find_packages('src', exclude=['ez_setup']), - namespace_packages=['imio', ], - package_dir={'': 'src'}, + keywords="migration helpers", + author="IMIO", + author_email="dev@imio.be", + url="https://github.com/imio/imio.migrator", + download_url="https://pypi.org/project/imio.migrator", + license="GPL", + packages=find_packages("src", exclude=["ez_setup"]), + namespace_packages=[ + "imio", + ], + package_dir={"": "src"}, include_package_data=True, zip_safe=False, install_requires=[ - 'Plone', - 'imio.helpers>=1.0.1', - 'imio.pyutils>=1.1.1', - 'setuptools', + "Plone", + "imio.helpers>=1.0.1", + "imio.pyutils>=1.1.1", + "setuptools", ], extras_require={ - 'test': [ - 'plone.app.testing', + "test": [ + "plone.app.testing", + "freezegun", ], }, entry_points=""" diff --git a/src/imio/__init__.py b/src/imio/__init__.py index f48ad10..05f0beb 100644 --- a/src/imio/__init__.py +++ b/src/imio/__init__.py @@ -1,6 +1,7 @@ # See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages try: - __import__('pkg_resources').declare_namespace(__name__) + __import__("pkg_resources").declare_namespace(__name__) except ImportError: from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/src/imio/migrator/configure.zcml b/src/imio/migrator/configure.zcml new file mode 100644 index 0000000..a7a6bb4 --- /dev/null +++ b/src/imio/migrator/configure.zcml @@ -0,0 +1,6 @@ + + + diff --git a/src/imio/migrator/migrator.py b/src/imio/migrator/migrator.py index 5c516c6..696d146 100644 --- a/src/imio/migrator/migrator.py +++ b/src/imio/migrator/migrator.py @@ -21,8 +21,9 @@ from imio.pyutils.system import memory from imio.pyutils.system import process_memory from plone import api +from plone.base.interfaces import IBundleRegistry +from plone.base.utils import get_installer from plone.registry.interfaces import IRegistry -from Products.CMFPlone.utils import base_hasattr from Products.GenericSetup.upgrade import normalize_version from Products.ZCatalog.ProgressHandler import ZLogHandler from zope.component import getUtility @@ -32,193 +33,195 @@ import time -logger = logging.getLogger('imio.migrator') -CURRENTLY_MIGRATING_REQ_VALUE = 'imio_migrator_currently_migrating' +logger = logging.getLogger("imio.migrator") +CURRENTLY_MIGRATING_REQ_VALUE = "imio_migrator_currently_migrating" class Migrator(object): """Abstract class for creating a migrator.""" + def __init__(self, context, disable_linkintegrity_checks=False): self.context = context self.portal = context.portal_url.getPortalObject() self.request = self.portal.REQUEST self.ps = self.portal.portal_setup + self.installer = get_installer(self.portal, self.request) self.wfTool = self.portal.portal_workflow self.registry = getUtility(IRegistry) - self.catalog = api.portal.get_tool('portal_catalog') + self.catalog = api.portal.get_tool("portal_catalog") self.startTime = time.time() self.warnings = [] self.request.set(CURRENTLY_MIGRATING_REQ_VALUE, True) self.disable_linkintegrity_checks = disable_linkintegrity_checks if disable_linkintegrity_checks: self.original_link_integrity = disable_link_integrity_checks() - self.run_part = os.getenv('FUNC_PART', '') + self.run_part = os.getenv("FUNC_PART", "") self.display_mem = True def run(self): """Must be overridden. This method does the migration job.""" - raise NotImplementedError('You should have overridden me darling.') + raise NotImplementedError("You should have overridden me darling.") def is_in_part(self, part): """Check if environment variable part is the same as parameter.""" if self.run_part == part: logger.info("DOING PART '{}'".format(part)) return True - elif self.run_part == '': - self.log_mem("PART {}".format(part)) # print intermediate part memory info if run in one step + elif self.run_part == "": + self.log_mem( + "PART {}".format(part) + ) # print intermediate part memory info if run in one step return True return False - def log_mem(self, tag=''): + def log_mem(self, tag=""): """Display in Mb the used memory and in the fourth position of the 'quintet' the available memory""" if self.display_mem: - logger.info('Mem used {} at {}, ({})'.format(process_memory(), tag, memory())) + logger.info( + "Mem used {} at {}, ({})".format(process_memory(), tag, memory()) + ) def warn(self, logger, warning_msg): """Manage warning messages, into logger and saved into self.warnings.""" - logger.warn(warning_msg) + logger.warning(warning_msg) self.warnings.append(warning_msg) def finish(self): """At the end of the migration, you can call this method to log its - duration in minutes.""" + duration in minutes.""" if self.disable_linkintegrity_checks: restore_link_integrity_checks(self.original_link_integrity) self.request.set(CURRENTLY_MIGRATING_REQ_VALUE, False) if not self.warnings: - self.warnings.append('No warnings.') - logger.info('HERE ARE WARNING MESSAGES GENERATED DURING THE MIGRATION : \n{0}'.format( - '\n'.join(self.warnings))) + self.warnings.append("No warnings.") + logger.info( + "HERE ARE WARNING MESSAGES GENERATED DURING THE MIGRATION : \n{0}".format( + "\n".join(self.warnings) + ) + ) logger.info(end_time(self.startTime)) - def refreshDatabase(self, - catalogs=True, - catalogsToRebuild=['portal_catalog'], - workflows=False, - workflowsToUpdate=[], - catalogsToUpdate=('portal_catalog', 'reference_catalog', 'uid_catalog')): + def refreshDatabase( + self, + catalogs=True, + catalogsToRebuild=["portal_catalog"], + workflows=False, + workflowsToUpdate=[], + catalogsToUpdate=("portal_catalog",), + ): """After the migration script has been executed, it can be necessary to - update the Plone catalogs and/or the workflow settings on every - database object if workflow definitions have changed. We can pass - catalog ids we want to 'clear and rebuild' using - p_catalogsToRebuild.""" + update the Plone catalogs and/or the workflow settings on every + database object if workflow definitions have changed. We can pass + catalog ids we want to 'clear and rebuild' using + p_catalogsToRebuild.""" if catalogs: # Manage the catalogs we want to clear and rebuild # We have to call another method as clear=1 passed to refreshCatalog # does not seem to work as expected... for catalogId in catalogsToRebuild: - logger.info('Clearing and rebuilding {0}...'.format(catalogId)) + logger.info("Clearing and rebuilding {0}...".format(catalogId)) catalogObj = getattr(self.portal, catalogId) - if base_hasattr(catalogObj, 'clearFindAndRebuild'): - catalogObj.clearFindAndRebuild() - else: - # special case for the uid_catalog - catalogObj.manage_rebuildCatalog() + catalogObj.clearFindAndRebuild() for catalogId in catalogsToUpdate: if catalogId not in catalogsToRebuild: - logger.info('Refreshing {0}...'.format(catalogId)) + logger.info("Refreshing {0}...".format(catalogId)) catalogObj = getattr(self.portal, catalogId) pghandler = ZLogHandler() catalogObj.refreshCatalog(clear=0, pghandler=pghandler) if workflows: - logger.info('Refresh workflow-related information on every object of the database...') + logger.info( + "Refresh workflow-related information on every object of the database..." + ) if not workflowsToUpdate: - logger.info('Refreshing every workflows...') + logger.info("Refreshing every workflows...") count = self.wfTool.updateRoleMappings() else: wfs = {} for wf_id in workflowsToUpdate: - logger.info('Refreshing workflow(s) "{0}"...'.format( - ", ".join(workflowsToUpdate))) + logger.info( + 'Refreshing workflow(s) "{0}"...'.format( + ", ".join(workflowsToUpdate) + ) + ) wf = self.wfTool.getWorkflowById(wf_id) wfs[wf_id] = wf count = self.wfTool._recursiveUpdateRoleMappings(self.portal, wfs) - logger.info('{0} object(s) updated.'.format(count)) + logger.info("{0} object(s) updated.".format(count)) - def cleanRegistries(self, registries=('portal_javascripts', 'portal_css', 'portal_setup')): + def cleanRegistries(self, registries=("bundles_registry", "portal_setup")): """ - Clean p_registries, remove not found elements. + Clean p_registries, remove not found elements. """ - logger.info('Cleaning registries...') - if 'portal_javascripts' in registries: - jstool = self.portal.portal_javascripts - for script in jstool.getResources(): - scriptId = script.getId() - resourceExists = script.isExternal or self.portal.restrictedTraverse(scriptId, False) and True - if not resourceExists: - # we found a notFound resource, remove it - logger.info('Removing %s from portal_javascripts' % scriptId) - jstool.unregisterResource(scriptId) - jstool.cookResources() - logger.info('portal_javascripts has been cleaned!') - - if 'portal_css' in registries: - csstool = self.portal.portal_css - for sheet in csstool.getResources(): - sheetId = sheet.getId() - resourceExists = sheet.isExternal or self.portal.restrictedTraverse(sheetId, False) and True - if not resourceExists: - # we found a notFound resource, remove it - logger.info('Removing %s from portal_css' % sheetId) - csstool.unregisterResource(sheetId) - csstool.cookResources() - logger.info('portal_css has been cleaned!') + logger.info("Cleaning registries...") + if "bundles_registry" in registries: + bundles = self.registry.collectionOfInterface( + IBundleRegistry, prefix="plone.bundles", check=False + ) + for bundle_name, bundle in bundles.items(): + js = bundle.jscompilation + css = bundle.csscompilation + resource_is_missing = not self.portal.restrictedTraverse( + js, False + ) or not self.portal.restrictedTraverse(css, False) + if resource_is_missing: + # we found a notFound resource (css or js), remove it + logger.info(f"Removing {bundle_name} from bundle registry") + del bundles[bundle_name] + logger.info("Bundle registry has been cleaned!") - if 'portal_setup' in registries: + if "portal_setup" in registries: # clean portal_setup change = False for stepId in self.ps.getSortedImportSteps(): stepMetadata = self.ps.getImportStepMetadata(stepId) # remove invalid steps - if stepMetadata['invalid']: - logger.info('Removing %s step from portal_setup' % stepId) + if stepMetadata["invalid"]: + logger.info("Removing %s step from portal_setup" % stepId) self.ps._import_registry.unregisterStep(stepId) change = True if change: self.ps._p_changed = True - logger.info('portal_setup has been cleaned!') - logger.info('Registries have been cleaned!') + logger.info("portal_setup has been cleaned!") + logger.info("Registries have been cleaned!") def removeUnusedIndexes(self, indexes=[]): - """ Remove unused catalog indexes. """ - logger.info('Removing no more used catalog indexes...') + """Remove unused catalog indexes.""" + logger.info("Removing no more used catalog indexes...") removeIndexes(self.portal, indexes=indexes) - logger.info('Done.') + logger.info("Done.") def removeUnusedColumns(self, columns=[]): - """ Remove unused catalog columns. """ - logger.info('Removing no more used catalog columns...') + """Remove unused catalog columns.""" + logger.info("Removing no more used catalog columns...") removeColumns(self.portal, columns=columns) - logger.info('Done.') + logger.info("Done.") def removeUnusedPortalTypes(self, portal_types=[]): - """ Remove unused portal_types from portal_types and portal_factory.""" - logger.info('Removing no more used {0} portal_types...'.format(', '.join(portal_types))) + """Remove unused portal_types from portal_types and portal_factory.""" + logger.info( + "Removing no more used {0} portal_types...".format(", ".join(portal_types)) + ) # remove from portal_types types = self.portal.portal_types - to_remove = [portal_type for portal_type in portal_types if portal_type in types] + to_remove = [ + portal_type for portal_type in portal_types if portal_type in types + ] if to_remove: types.manage_delObjects(ids=to_remove) - # remove from portal_factory - portal_factory = api.portal.get_tool('portal_factory') - registeredFactoryTypes = [portal_type for portal_type in list(portal_factory.getFactoryTypes().keys()) - if portal_type not in portal_types] - portal_factory.manage_setPortalFactoryTypes(listOfTypeIds=registeredFactoryTypes) - # remove from site_properties.types_not_searched - props = api.portal.get_tool('portal_properties').site_properties - nsTypes = list(props.getProperty('types_not_searched')) + # remove from registry plone.types_not_searched + nsTypes = list(self.registry.get("plone.types_not_searched")) for portal_type_id in portal_types: if portal_type_id in nsTypes: nsTypes.remove(portal_type_id) - props.manage_changeProperties(types_not_searched=tuple(nsTypes)) - logger.info('Done.') + self.registry["plone.types_not_searched"] = tuple(nsTypes) + logger.info("Done.") def clean_orphan_brains(self, query): """Get brains from catalog with p_query and clean brains without an object.""" brains = list(self.catalog(**query)) pghandler = ZLogHandler(steps=1000) - pghandler.init('clean_orphan_brains', len(brains)) - pghandler.info('Cleaning orphan brains (query=%s)' % query) + pghandler.init("clean_orphan_brains", len(brains)) + pghandler.info("Cleaning orphan brains (query=%s)" % query) i = 0 cleaned = 0 for brain in brains: @@ -233,9 +236,11 @@ def clean_orphan_brains(self, query): cleaned += 1 pghandler.finish() logger.warning("clean_orphan_brains cleaned %d orphan brains" % cleaned) - logger.info('Done.') + logger.info("Done.") - def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_types=[]): + def reindexIndexes( + self, idxs=[], update_metadata=False, meta_types=[], portal_types=[] + ): """Reindex index including metadata if p_update_metadata=True. :param idxs: list of indexes to handle @@ -244,16 +249,20 @@ def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_t :param portal_types: list of portal_types to filter on :return: True if batch_number is not defined, else return batch_last """ - catalog = api.portal.get_tool('portal_catalog') + catalog = api.portal.get_tool("portal_catalog") paths = list(catalog._catalog.uids.keys()) pghandler = ZLogHandler(steps=1000) i = 0 pghandler.info( - 'In reindexIndexes, idxs={0}, update_metadata={1}, meta_types={2}, portal_types={3}'.format( - repr(idxs), repr(update_metadata), repr(meta_types), repr(portal_types))) - pghandler.init('reindexIndexes', len(paths)) - pklfile = batch_hashed_filename('imio.migrator.reindexIndexes.pkl', - (idxs, update_metadata, meta_types, portal_types)) + "In reindexIndexes, idxs={0}, update_metadata={1}, meta_types={2}, portal_types={3}".format( + repr(idxs), repr(update_metadata), repr(meta_types), repr(portal_types) + ) + ) + pghandler.init("reindexIndexes", len(paths)) + pklfile = batch_hashed_filename( + "imio.migrator.reindexIndexes.pkl", + (idxs, update_metadata, meta_types, portal_types), + ) batch_keys, batch_config = batch_get_keys(pklfile, loop_length=len(paths)) for p in paths: if batch_skip_key(p, batch_keys, batch_config): @@ -263,10 +272,19 @@ def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_t pghandler.report(i) obj = catalog.resolve_path(p) if obj is None: - logger.error('reindexIndex could not resolve an object from the uid %r.' % p) - elif (not meta_types or obj.meta_type in meta_types) and \ - (not portal_types or obj.portal_type in portal_types): - catalog.catalog_object(obj, p, idxs=idxs, update_metadata=update_metadata, pghandler=pghandler) + logger.error( + "reindexIndex could not resolve an object from the uid %r." % p + ) + elif (not meta_types or obj.meta_type in meta_types) and ( + not portal_types or obj.portal_type in portal_types + ): + catalog.catalog_object( + obj, + p, + idxs=idxs, + update_metadata=update_metadata, + pghandler=pghandler, + ) if batch_handle_key(p, batch_keys, batch_config): break else: @@ -278,17 +296,17 @@ def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_t return batch_globally_finished(batch_keys, batch_config) def reindexIndexesFor(self, idxs=[], **query): - """ Reindex p_idxs on objects of given p_portal_types. """ - catalog = api.portal.get_tool('portal_catalog') + """Reindex p_idxs on objects of given p_portal_types.""" + catalog = api.portal.get_tool("portal_catalog") brains = catalog(**query) pghandler = ZLogHandler(steps=1000) len_brains = len(brains) pghandler.info( 'In reindexIndexesFor, reindexing indexes "{0}" on "{1}" objects ({2})...'.format( - ', '.join(idxs) or '*', - len(brains), - str(query))) - pghandler.init('reindexIndexesFor', len_brains) + ", ".join(idxs) or "*", len(brains), str(query) + ) + ) + pghandler.init("reindexIndexesFor", len_brains) i = 0 for brain in brains: i += 1 @@ -296,35 +314,46 @@ def reindexIndexesFor(self, idxs=[], **query): obj = brain.getObject() obj.reindexObject(idxs=idxs) pghandler.finish() - logger.info('Done.') + logger.info("Done.") def install(self, products): - """ Allows to install a series of products """ - qi = api.portal.get_tool('portal_quickinstaller') + """Allows to install a series of products""" for product in products: logger.info("Install product '{}'".format(product)) - logger.info(qi.installProduct(product, forceProfile=True)) # don't reinstall + logger.info(self.installer.install_product(product)) # don't reinstall def reinstall(self, profiles, ignore_dependencies=False, dependency_strategy=None): - """ Allows to reinstall a series of p_profiles. """ - logger.info('Reinstalling product(s) %s...' % ', '.join([profile.startswith('profile-') and profile[8:] or - profile for profile in profiles])) + """Allows to reinstall a series of p_profiles.""" + logger.info( + "Reinstalling product(s) %s..." + % ", ".join( + [ + profile.startswith("profile-") and profile[8:] or profile + for profile in profiles + ] + ) + ) for profile in profiles: - if not profile.startswith('profile-'): - profile = 'profile-%s' % profile + if not profile.startswith("profile-"): + profile = "profile-%s" % profile try: - self.ps.runAllImportStepsFromProfile(profile, - ignore_dependencies=ignore_dependencies, - dependency_strategy=dependency_strategy) + self.ps.runAllImportStepsFromProfile( + profile, + ignore_dependencies=ignore_dependencies, + dependency_strategy=dependency_strategy, + ) except KeyError: - logger.error('Profile %s not found!' % profile) - logger.info('Done.') + logger.error("Profile %s not found!" % profile) + logger.info("Done.") def upgradeProfile(self, profile, olds=[]): - """ Get upgrade steps and run it. olds can contain a list of dest upgrades to run. """ + """Get upgrade steps and run it. olds can contain a list of dest upgrades to run.""" def run_upgrade_step(step, source, dest): - logger.info('Running upgrade step %s (%s -> %s): %s' % (profile, source, dest, step.title)) + logger.info( + "Running upgrade step %s (%s -> %s): %s" + % (profile, source, dest, step.title) + ) step.doStep(self.ps) # if olds, we get all steps. @@ -332,42 +361,45 @@ def run_upgrade_step(step, source, dest): applied_dests = [] for container in upgrades: if isinstance(container, dict): - if not olds or container['sdest'] in olds: - applied_dests.append((normalize_version(container['sdest']), container['sdest'])) - run_upgrade_step(container['step'], container['ssource'], container['sdest']) + if not olds or container["sdest"] in olds: + applied_dests.append( + (normalize_version(container["sdest"]), container["sdest"]) + ) + run_upgrade_step( + container["step"], container["ssource"], container["sdest"] + ) elif isinstance(container, list): for dic in container: - if not olds or dic['sdest'] in olds: - applied_dests.append((normalize_version(dic['sdest']), dic['sdest'])) - run_upgrade_step(dic['step'], dic['ssource'], dic['sdest']) + if not olds or dic["sdest"] in olds: + applied_dests.append( + (normalize_version(dic["sdest"]), dic["sdest"]) + ) + run_upgrade_step(dic["step"], dic["ssource"], dic["sdest"]) if applied_dests: - current_version = normalize_version(self.ps.getLastVersionForProfile(profile)) + current_version = normalize_version( + self.ps.getLastVersionForProfile(profile) + ) highest_version, dest = sorted(applied_dests)[-1] # check if highest applied version is higher than current version if highest_version > current_version: self.ps.setLastVersionForProfile(profile, dest) - # we update portal_quickinstaller version - pqi = self.portal.portal_quickinstaller - try: - product = profile.split(':')[0] - prod = pqi.get(product) - setattr(prod, 'installedversion', pqi.getProductVersion(product)) - except IndexError as e: - logger.error("Cannot extract product from profile '%s': %s" % (profile, e)) - except AttributeError as e: - logger.error("Cannot get product '%s' from portal_quickinstaller: %s" % (product, e)) def upgradeAll(self, omit=[]): - """ Upgrade all upgrade profiles except those in omit parameter list """ - if self.portal.REQUEST.get('profile_id'): - omit.append(self.portal.REQUEST.get('profile_id')) + """Upgrade all upgrade profiles except those in omit parameter list""" + if self.portal.REQUEST.get("profile_id"): + omit.append(self.portal.REQUEST.get("profile_id")) for profile in self.ps.listProfilesWithUpgrades(): # make sure the profile isn't the current (or must be avoided) and # the profile is well installed - if profile not in omit and self.ps.getLastVersionForProfile(profile) != 'unknown': + if ( + profile not in omit + and self.ps.getLastVersionForProfile(profile) != "unknown" + ): self.upgradeProfile(profile) - def runProfileSteps(self, product, steps=[], profile='default', run_dependencies=False): + def runProfileSteps( + self, product, steps=[], profile="default", run_dependencies=False + ): """Run given steps of a product profile (default is 'default' profile). :param product: product name @@ -377,6 +409,11 @@ def runProfileSteps(self, product, steps=[], profile='default', run_dependencies (default is False) """ for step_id in steps: - logger.info("Running profile step '%s:%s' => %s" % (product, profile, step_id)) - self.ps.runImportStepFromProfile('profile-%s:%s' % (product, profile), step_id, - run_dependencies=run_dependencies) + logger.info( + "Running profile step '%s:%s' => %s" % (product, profile, step_id) + ) + self.ps.runImportStepFromProfile( + "profile-%s:%s" % (product, profile), + step_id, + run_dependencies=run_dependencies, + ) diff --git a/src/imio/migrator/profiles/testing/metadata.xml b/src/imio/migrator/profiles/testing/metadata.xml new file mode 100644 index 0000000..f768b5a --- /dev/null +++ b/src/imio/migrator/profiles/testing/metadata.xml @@ -0,0 +1,4 @@ + + + 1000 + diff --git a/src/imio/migrator/profiles/testing/registry.xml b/src/imio/migrator/profiles/testing/registry.xml new file mode 100644 index 0000000..00466ef --- /dev/null +++ b/src/imio/migrator/profiles/testing/registry.xml @@ -0,0 +1,26 @@ + + + + + True + + ++plone++not-existing.css + + plone + False + False + + + + True + + + ++plone++not-existing.js + plone + False + False + + + diff --git a/src/imio/migrator/profiles/testing2/metadata.xml b/src/imio/migrator/profiles/testing2/metadata.xml new file mode 100644 index 0000000..f768b5a --- /dev/null +++ b/src/imio/migrator/profiles/testing2/metadata.xml @@ -0,0 +1,4 @@ + + + 1000 + diff --git a/src/imio/migrator/profiles/testing2/registry.xml b/src/imio/migrator/profiles/testing2/registry.xml new file mode 100644 index 0000000..025473c --- /dev/null +++ b/src/imio/migrator/profiles/testing2/registry.xml @@ -0,0 +1,15 @@ + + + + + True + + ++plone++my-bundle.css + + plone + False + False + + + diff --git a/src/imio/migrator/testing.py b/src/imio/migrator/testing.py index 3f65ab5..6c7e769 100644 --- a/src/imio/migrator/testing.py +++ b/src/imio/migrator/testing.py @@ -1,24 +1,37 @@ # -*- coding: utf-8 -*- -from plone.app.testing import FunctionalTesting -from plone.app.testing import PloneWithPackageLayer -from plone.testing import z2 -from plone.testing import zca + +from plone.app.testing import applyProfile +from plone.app.testing import IntegrationTesting +from plone.app.testing import PLONE_FIXTURE +from plone.app.testing import PloneSandboxLayer import imio.migrator +import unittest + + +class ImioMigratorLayer(PloneSandboxLayer): + + defaultBases = (PLONE_FIXTURE,) + + def setUpZope(self, app, configurationContext): + """Set up Zope.""" + self.loadZCML(package=imio.migrator, name="testing.zcml") + + def setUpPloneSite(self, portal): + """Set up Plone.""" + portal.portal_workflow.setDefaultChain("simple_publication_workflow") + applyProfile(portal, "imio.migrator:testing") + +FIXTURE = ImioMigratorLayer(name="FIXTURE") +INTEGRATION = IntegrationTesting(bases=(FIXTURE,), name="INTEGRATION") -MIGRATOR_ZCML = zca.ZCMLSandbox(filename="testing.zcml", - package=imio.migrator, - name='MIGRATOR_ZCML') -MIGRATOR_Z2 = z2.IntegrationTesting(bases=(z2.STARTUP, MIGRATOR_ZCML), - name='MIGRATOR_Z2') +class IntegrationTestCase(unittest.TestCase): + """Base class for integration tests.""" -MIGRATOR_TESTING_PROFILE = PloneWithPackageLayer( - zcml_filename="testing.zcml", - zcml_package=imio.migrator, - additional_z2_products=(), - name="MIGRATOR_TESTING_PROFILE") + layer = INTEGRATION -MIGRATOR_TESTING_PROFILE_FUNCTIONAL = FunctionalTesting( - bases=(MIGRATOR_TESTING_PROFILE,), name="MIGRATOR_TESTING_PROFILE_FUNCTIONAL") + def setUp(self): + super(IntegrationTestCase, self).setUp() + self.portal = self.layer["portal"] diff --git a/src/imio/migrator/testing.zcml b/src/imio/migrator/testing.zcml index a7a6bb4..16e7518 100644 --- a/src/imio/migrator/testing.zcml +++ b/src/imio/migrator/testing.zcml @@ -3,4 +3,30 @@ xmlns:genericsetup="http://namespaces.zope.org/genericsetup" i18n_domain="imio.migrator"> + + + + + + + + + + diff --git a/src/imio/migrator/tests/test_migrator.py b/src/imio/migrator/tests/test_migrator.py new file mode 100644 index 0000000..86a3950 --- /dev/null +++ b/src/imio/migrator/tests/test_migrator.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- + +from imio.migrator.migrator import Migrator +from imio.migrator.testing import IntegrationTestCase +from plone import api +from plone.app.testing import login +from plone.app.testing import setRoles +from plone.app.testing import TEST_USER_ID +from plone.app.testing import TEST_USER_NAME +from plone.base.interfaces import IBundleRegistry +from Products.CMFCore.indexing import processQueue +from Products.CMFCore.permissions import AccessContentsInformation +from Products.CMFCore.permissions import View +from Products.CMFCore.utils import _checkPermission +from unittest.mock import patch + + +class TestMigrator(IntegrationTestCase): + + def setUp(self): + self.portal = self.layer["portal"] + self.catalog = api.portal.get_tool("portal_catalog") + self.wf_tool = api.portal.get_tool("portal_workflow") + setRoles(self.portal, TEST_USER_ID, ["Manager"]) + self.folder = api.content.create( + container=self.portal, + type="Folder", + title="Folder", + id="folder", + ) + self.doc = api.content.create( + container=self.folder, + type="Document", + id="doc", + title="Foo", + description="Bar", + ) + self.migrator = Migrator(self.portal) + self.migrator.display_mem = False + processQueue() + + def test_run(self): + with self.assertRaises(NotImplementedError): + self.migrator.run() + + def test_is_in_part(self): + self.migrator.run_part = "RUN_PART" + self.assertTrue(self.migrator.is_in_part("RUN_PART")) + self.assertFalse(self.migrator.is_in_part("OTHER_PART")) + self.migrator.run_part = "" + self.assertTrue(self.migrator.is_in_part("TEST")) + + def test_refreshDatabase(self): + self.catalog.unindexObject(self.folder.doc) + self.assertEqual(len(self.catalog(getId="doc")), 0) + self.migrator.refreshDatabase( + catalogs=True, + catalogsToRebuild=[], + catalogsToUpdate=[], + ) + self.assertEqual(len(self.catalog(getId="doc")), 0) + self.migrator.refreshDatabase(catalogs=False) + self.assertEqual(len(self.catalog(getId="doc")), 0) + self.migrator.refreshDatabase(catalogs=True) + self.assertEqual(len(self.catalog(getId="doc")), 1) + self.catalog.unindexObject(self.folder.doc) + self.assertEqual(len(self.catalog(getId="doc")), 0) + self.migrator.refreshDatabase( + catalogs=True, + catalogsToRebuild=[], + catalogsToUpdate=["portal_catalog"], + ) + self.assertEqual(len(self.catalog(getId="doc")), 0) + self.catalog.reindexObject(self.folder.doc) + self.assertEqual(len(self.catalog(getId="doc")), 1) + self.folder.doc.setTitle("Fred") + self.folder.doc.setDescription("BamBam") + self.migrator.refreshDatabase( + catalogs=True, + catalogsToRebuild=[], + catalogsToUpdate=["portal_catalog"], + ) + brain = self.catalog(getId="doc")[0] + self.assertEqual(brain.getId, "doc") + self.assertEqual(brain.Title, "Fred") + self.assertEqual(brain.Description, "BamBam") + self.catalog.unindexObject(self.folder.doc) + self.assertEqual(len(self.catalog(getId="doc")), 0) + self.migrator.refreshDatabase( + catalogs=True, + catalogsToRebuild=["portal_catalog"], + catalogsToUpdate=[], + ) + self.assertEqual(len(self.catalog(getId="doc")), 1) + self.assertTrue(_checkPermission(View, self.doc)) + self.assertEqual(len(self.catalog(getId="doc")), 1) + wf = self.portal.portal_workflow.getWorkflowsFor(self.doc)[0] + wf.states.private.permission_roles[AccessContentsInformation] = ("Manager",) + wf.states.private.permission_roles[View] = ("Manager",) + setRoles(self.portal, TEST_USER_ID, ["Member"]) + login(self.portal, TEST_USER_NAME) + self.assertTrue(_checkPermission(View, self.doc)) + self.assertEqual(len(self.catalog(getId="doc")), 1) + self.migrator.refreshDatabase( + catalogs=False, + workflows=True, + ) + self.assertFalse(_checkPermission(View, self.doc)) + self.assertEqual(len(self.catalog(getId="doc")), 0) + wf.states.private.permission_roles[AccessContentsInformation] = ("Member",) + wf.states.private.permission_roles[View] = ("Member",) + self.migrator.refreshDatabase( + catalogs=False, + workflows=True, + workflowsToUpdate=["simple_publication_workflow"], + ) + self.assertTrue(_checkPermission(View, self.doc)) + self.assertEqual(len(self.catalog(getId="doc")), 1) + + def test_cleanRegistries(self): + bundles = self.migrator.registry.collectionOfInterface( + IBundleRegistry, prefix="plone.bundles", check=False + ) + self.assertTrue("broken-css-bundle" in bundles.keys()) + self.assertTrue("broken-js-bundle" in bundles.keys()) + self.migrator.cleanRegistries() + bundles = self.migrator.registry.collectionOfInterface( + IBundleRegistry, prefix="plone.bundles", check=False + ) + self.assertFalse("broken-css-bundle" in bundles.keys()) + self.assertFalse("broken-js-bundle" in bundles.keys()) + + def test_removeUnusedPortalTypes(self): + self.assertIn("TempFolder", self.migrator.portal.portal_types) + self.assertIn( + "TempFolder", self.migrator.registry.get("plone.types_not_searched") + ) + self.migrator.removeUnusedPortalTypes(["TempFolder"]) + self.assertNotIn("TempFolder", self.migrator.portal.portal_types) + self.assertNotIn( + "TempFolder", self.migrator.registry.get("plone.types_not_searched") + ) + + def test_clean_orphan_brains(self): + self.assertEqual(len(self.catalog(getId="doc")), 1) + with patch( + "Products.ZCatalog.CatalogBrains.AbstractCatalogBrain.getObject", + side_effect=AttributeError, + ): + self.migrator.clean_orphan_brains({}) + self.assertEqual(len(self.catalog(getId="doc")), 0) + + def test_reindexIndexes(self): + self.folder.doc.setTitle("Fred") + self.folder.doc.setDescription("BamBam") + self.migrator.reindexIndexes(idxs=["Title"], update_metadata=True) + brain = self.catalog(getId="doc")[0] + self.assertEqual(brain.getId, "doc") + self.assertEqual(brain.Title, "Fred") + self.assertEqual(brain.Description, "BamBam") + self.folder.doc.setTitle("Bob") + self.folder.doc.setDescription("BimBim") + self.migrator.reindexIndexes(idxs=["Title"], update_metadata=False) + brain = self.catalog(getId="doc")[0] + self.assertEqual(brain.getId, "doc") + self.assertEqual(brain.Title, "Fred") + self.assertEqual(brain.Description, "BamBam") + + def test_reindexIndexesFor(self): + self.folder.doc.setTitle("Fred") + self.folder.doc.setDescription("BamBam") + self.migrator.reindexIndexesFor(idxs=["Title"], portal_type=["Folder"]) + brain = self.catalog(getId="doc")[0] + self.assertEqual(brain.getId, "doc") + self.assertEqual(brain.Title, "Foo") + self.assertEqual(brain.Description, "Bar") + self.migrator.reindexIndexesFor(idxs=["Title"], portal_type=["Document"]) + brain = self.catalog(getId="doc")[0] + self.assertEqual(brain.getId, "doc") + self.assertEqual(brain.Title, "Fred") + self.assertEqual(brain.Description, "BamBam") + + def test_install(self): + self.assertFalse(self.migrator.installer.is_product_installed("plone.session")) + self.migrator.install(["plone.session"]) + self.assertTrue(self.migrator.installer.is_product_installed("plone.session")) + + def test_reinstall(self): + self.assertFalse(self.migrator.installer.is_product_installed("plone.session")) + self.migrator.install(["plone.session"]) + self.migrator.reinstall(["profile-plone.session:default"]) + self.assertTrue(self.migrator.installer.is_product_installed("plone.session")) + self.migrator.installer.uninstall_product("plone.session") + self.migrator.reinstall(["profile-plone.session:default"]) + self.assertTrue(self.migrator.installer.is_product_installed("plone.session")) + + def test_upgradeProfile(self): + self.migrator.ps.setLastVersionForProfile("imio.migrator:testing", "999") + info = self.migrator.installer.upgrade_info("imio.migrator") + self.assertEqual(info["installedVersion"], "999") + self.migrator.upgradeProfile("imio.migrator:testing") + info = self.migrator.installer.upgrade_info("imio.migrator") + self.assertEqual(info["installedVersion"], "1000") + + def test_upgradeAll(self): + self.migrator.ps.setLastVersionForProfile("imio.migrator:testing", "999") + info = self.migrator.installer.upgrade_info("imio.migrator") + self.assertEqual(info["installedVersion"], "999") + self.migrator.upgradeAll(omit=["imio.migrator:testing"]) + info = self.migrator.installer.upgrade_info("imio.migrator") + self.assertEqual(info["installedVersion"], "999") + self.migrator.upgradeAll() + info = self.migrator.installer.upgrade_info("imio.migrator") + self.assertEqual(info["installedVersion"], "1000") + + def test_runProfileSteps(self): + bundles = self.migrator.registry.collectionOfInterface( + IBundleRegistry, prefix="plone.bundles", check=False + ) + self.assertFalse("my-bundle" in bundles.keys()) + self.migrator.runProfileSteps( + "imio.migrator", + steps=["plone.app.registry"], + profile="testing2", + run_dependencies=False, + ) + bundles = self.migrator.registry.collectionOfInterface( + IBundleRegistry, prefix="plone.bundles", check=False + ) + self.assertTrue("my-bundle" in bundles.keys()) diff --git a/src/imio/migrator/tests/test_utils.py b/src/imio/migrator/tests/test_utils.py new file mode 100644 index 0000000..e288766 --- /dev/null +++ b/src/imio/migrator/tests/test_utils.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +from freezegun import freeze_time +from imio.migrator.migrator import Migrator +from imio.migrator.testing import IntegrationTestCase +from imio.migrator.utils import end_time +from imio.migrator.utils import ensure_upgraded + + +START_TIME = 1735732800.0 # 2025-01-01 12:00:00 + + +class TestUtils(IntegrationTestCase): + + def setUp(self): + self.portal = self.layer["portal"] + self.migrator = Migrator(self.portal) + self.migrator.display_mem = False + + def test_ensure_upgraded(self): + self.migrator.ps.setLastVersionForProfile("imio.migrator:testing", "999") + info = self.migrator.installer.upgrade_info("imio.migrator") + self.assertEqual(info["installedVersion"], "999") + ensure_upgraded("imio.migrator", profile_name="testing") + info = self.migrator.installer.upgrade_info("imio.migrator") + self.assertEqual(info["installedVersion"], "1000") + + @freeze_time("2025-01-01 12:00:05") + def test_end_time_seconds_only(self): + msg = end_time(START_TIME) + self.assertEqual(msg, "Migration finished in 5 second(s).") + + @freeze_time("2025-01-01 12:01:05") + def test_end_time_minutes_and_seconds(self): + msg = end_time(START_TIME) + self.assertEqual(msg, "Migration finished in 1 minute(s), 5 second(s).") + + @freeze_time("2025-01-01 14:02:03") + def test_end_time_hours_minutes_seconds(self): + msg = end_time(START_TIME) + self.assertEqual( + msg, "Migration finished in 2 hour(s), 2 minute(s), 3 second(s)." + ) + + @freeze_time("2025-01-02 13:01:01") + def test_end_time_days_hours_minutes_seconds(self): + msg = end_time(START_TIME) + self.assertEqual( + msg, "Migration finished in 1 day(s), 1 hour(s), 1 minute(s), 1 second(s)." + ) + + @freeze_time("2025-01-01 12:00:10") + def test_end_time_return_seconds_flag(self): + msg, seconds = end_time(START_TIME, return_seconds=True) + self.assertEqual(msg, "Migration finished in 10 second(s).") + self.assertEqual(seconds, 10) + + @freeze_time("2025-01-01 12:00:10") + def test_end_time_total_number(self): + msg = end_time(START_TIME, total_number=50) + self.assertEqual( + msg, + "Migration finished in 10 second(s). Updated 50 elements, that is 5 by second.", + ) + + @freeze_time("2025-01-01 12:00:00") + def test_end_time_total_number_zero_seconds(self): + msg = end_time(START_TIME, total_number=10) + self.assertEqual( + msg, + "Migration finished in 0 second(s). Updated 10 elements, that is 10 by second.", + ) diff --git a/src/imio/migrator/utils.py b/src/imio/migrator/utils.py index b90e379..db6ec33 100644 --- a/src/imio/migrator/utils.py +++ b/src/imio/migrator/utils.py @@ -8,14 +8,16 @@ import time -def end_time(start_time, - base_msg="Migration finished in ", - return_seconds=False, - total_number=None): +def end_time( + start_time, + base_msg="Migration finished in ", + return_seconds=False, + total_number=None, +): """Display a end time message. - If p_return_seconds=True, it will return the msg and the total seconds. - If a integer is given to total_number, the msg is compeleted with - number of elements processed per second.""" + If p_return_seconds=True, it will return the msg and the total seconds. + If a integer is given to total_number, the msg is compeleted with + number of elements processed per second.""" seconds = time.time() - start_time seconds = int(seconds) m, s = divmod(seconds, 60) @@ -23,8 +25,9 @@ def end_time(start_time, d, h = divmod(h, 24) msg = base_msg if d: - msg += "{0} day(s), {1} hour(s), " \ - "{2} minute(s), {3} second(s).".format(d, h, m, s) + msg += "{0} day(s), {1} hour(s), " "{2} minute(s), {3} second(s).".format( + d, h, m, s + ) elif h: msg += "{0} hour(s), {1} minute(s), {2} second(s).".format(h, m, s) elif m: @@ -35,18 +38,21 @@ def end_time(start_time, if total_number is not None: # avoid divide by 0 if seconds = 0 msg += " Updated %d elements, that is %d by second." % ( - total_number, total_number / (seconds or 1)) + total_number, + total_number / (seconds or 1), + ) if return_seconds: return msg, seconds return msg -def ensure_upgraded(package_name): +def ensure_upgraded(package_name, profile_name="default"): """Make sure the given p_package_name is upgraded, this is useful when some - code will rely on fact that a record is in the registry or so. - profile_name must be like "collective.documentgenerator", we will turn - it into a portal_setup compliant profile name.""" + code will rely on fact that a record is in the registry or so. + profile_name must be like "collective.documentgenerator", we will turn + it into a portal_setup compliant profile name.""" from imio.migrator.migrator import Migrator + migrator = Migrator(api.portal.get()) - migrator.upgradeProfile("profile-" + package_name + ":default") + migrator.upgradeProfile("profile-" + package_name + ":" + profile_name) diff --git a/test-6.1.cfg b/test-6.1.cfg new file mode 100644 index 0000000..d708207 --- /dev/null +++ b/test-6.1.cfg @@ -0,0 +1,30 @@ +[buildout] + +extends = + https://raw.githubusercontent.com/collective/buildout.plonetest/master/test-6.1.x.cfg + base.cfg + +#update-versions-file = test-6.1.cfg + +[versions] +# to keep prompt-toolkit < 3 +ipython = 8.3.0 + +ipdb = 0.13.9 +iw.debug = 0.3 +jedi = 0.18.1 +parso = 0.8.3 +pdbp = 1.7.0 + +# Required by: +# ipdb +asttokens = 2.0.8 +backcall = 0.2.0 +executing = 1.1.1 +matplotlib-inline = 0.1.6 +pexpect = 4.8.0 +pickleshare = 0.7.5 +ptyprocess = 0.7.0 +pure-eval = 0.2.2 +stack-data = 0.5.1 +traitlets = 5.4.0