From c027e6a80615a1f5c23842f48a6dd46115cf5173 Mon Sep 17 00:00:00 2001 From: Sergey Klyuykov Date: Sat, 30 Nov 2019 12:53:52 +1000 Subject: [PATCH] Fix build and optimize `CRONTAB` periodic tasks saving performance. --- .gitlab-ci.yml | 8 +++--- Makefile | 14 ++++++++--- polemarch/__init__.py | 2 +- polemarch/main/models/__init__.py | 42 ++++++++++++++++++------------- setup.cfg | 1 + tox.ini | 2 +- tox_build.ini | 7 +++--- 7 files changed, 44 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 09bbb105..0e102bc5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,11 +2,10 @@ image: onegreyonewhite/tox:tox variables: - TESTS: "polemarch.main.tests" GET_SOURCES_ATTEMPTS: 3 ARTIFACT_DOWNLOAD_ATTEMPTS: 3 RESTORE_CACHE_ATTEMPTS: 3 - DJANGO_LOG_LEVEL: 'ERROR' + DJANGO_LOG_LEVEL: 'DEBUG' TOX_ARGS: "--workdir /cache/.tox_polemarch_${CI_BUILD_REF_NAME}" CCACHE_DIR: /cache/.ccache CC: ccache gcc @@ -98,7 +97,6 @@ master_coverage: <<: *branch_tests variables: TOX_ENVS: "py37-coverage" - except: only: refs: - master @@ -202,7 +200,7 @@ publish_pypi: - $PYPI_UPLOAD_PASSWORD - $PYPI_UPLOAD_NAME script: - - twine upload -u ${PYPI_UPLOAD_NAME} -p ${PYPI_UPLOAD_PASSWORD} dist/*.tar.gz dist/*.whl + - make twine publish_release: @@ -220,6 +218,6 @@ publish_release: before_script: - git config --global user.name "${GITLAB_USER_NAME}" - git config --global user.email "${GITLAB_USER_EMAIL}" - - git push https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/vstconsulting/polemarch.git "${CI_COMMIT_TAG}" + - git push https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/vstconsulting/polemarch.git ${CI_COMMIT_TAG} || echo "Failed to upload to github." script: - make test ENVS=release diff --git a/Makefile b/Makefile index 47cb6a67..3886896e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PY = python2 +PY = python PIP = $(PY) -m pip PY_VERSION = $(shell $(PY) --version 2>&1 | tail -c +8) PYTHON_BIN = $(shell $(PY) -c 'import sys, os; print(os.path.dirname(sys.executable))') @@ -12,7 +12,7 @@ TESTS = NAMEBASE = polemarch USER = $(NAMEBASE) NAME = $(NAMEBASE) -VER = $(shell $(PY) -c 'import $(NAME); print($(NAME).__version__)') +VER = $(shell $(PY) setup.py --version | tr -d '\n') PROJECT_CTL = $(NAME)ctl MAIN_APP = main VSTUTILS_REQ = $(shell cat requirements-doc.txt | grep vstutils | sed 's/\[.*\]/[doc]/') @@ -86,7 +86,7 @@ pylint: tox -e pylint build: build-clean print_vars - -rm -rf dist + # -rm -rf dist $(PY) setup.py sdist -v pre_compile: build-clean print_vars @@ -95,7 +95,7 @@ pre_compile: build-clean print_vars $(PIP) install -U $(VSTUTILS) compile: pre_compile - -rm -rf dist + # -rm -rf dist $(PY) setup.py compile -v wheel: pre_compile @@ -208,3 +208,9 @@ deploy: destroy: ansible-playbook destroy-openstack.yml -v --private-key $(OPENSTACK_KEY) $(DEPLOY_ARGS) + +twine: + for file in $(shell find dist/*.{tar.gz,whl} | grep ${NAME} | grep ${VER}); do \ + echo $$file; \ + twine upload -u $(PYPI_UPLOAD_NAME) -p $(PYPI_UPLOAD_PASSWORD) $$file || echo "Filed to upload ${file}"; \ + done diff --git a/polemarch/__init__.py b/polemarch/__init__.py index 2edf8bdc..c6b30fb0 100644 --- a/polemarch/__init__.py +++ b/polemarch/__init__.py @@ -31,6 +31,6 @@ "VST_ROOT_URLCONF": os.getenv("VST_ROOT_URLCONF", 'vstutils.urls'), } -__version__ = "1.5.0" +__version__ = "1.5.1" prepare_environment(**default_settings) diff --git a/polemarch/main/models/__init__.py b/polemarch/main/models/__init__.py index 2a02c8a3..47670ee7 100644 --- a/polemarch/main/models/__init__.py +++ b/polemarch/main/models/__init__.py @@ -221,21 +221,23 @@ def save_to_beat(instance: PeriodicTask, **kwargs) -> NoReturn: if instance.type == "INTERVAL": units = IntervalSchedule.SECONDS secs = instance.get_schedule() - schedule, _ = IntervalSchedule.objects.get_or_create(every=secs, - period=units) - manager.create(interval=schedule, - name=str(instance.id), - task=task, - args=json.dumps([instance.id])) + schedule, _ = IntervalSchedule.objects.get_or_create(every=secs, period=units) + manager.create( + interval=schedule, + name=str(instance.id), + task=task, + args=json.dumps([instance.id]) + ) elif instance.type == "CRONTAB": cron_data = instance.crontab_kwargs + cron_data['timezone'] = settings.TIME_ZONE schedule, _ = CrontabSchedule.objects.get_or_create(**cron_data) - schedule.timezone = settings.TIME_ZONE - schedule.save() - manager.create(crontab=schedule, - name=str(instance.id), - task=task, - args=json.dumps([instance.id])) + manager.create( + crontab=schedule, + name=str(instance.id), + task=task, + args=json.dumps([instance.id]) + ) @receiver(signals.post_delete, sender=PeriodicTask) @@ -244,18 +246,22 @@ def delete_from_beat(instance: PeriodicTask, **kwargs) -> NoReturn: return manager = django_celery_beat.models.PeriodicTask.objects celery_tasks = manager.filter(name=str(instance.id)) + types_dict = { + 'crontab_id': CrontabSchedule, + 'interval_id': IntervalSchedule + } + qs_dict = {i: [] for i in types_dict} for task in celery_tasks: - qs_dict = { - 'crontab_id': CrontabSchedule.objects.all(), - 'interval_id': IntervalSchedule.objects.all(), - } - for field in ['crontab_id', 'interval_id']: + for field in qs_dict: pk = getattr(task, field) if pk is None: continue others = manager.filter(**{field: pk}).exclude(pk=task.id) if not others.exists(): - qs_dict[field].get(id=pk).delete() + qs_dict[field].append(pk) + for key, values in qs_dict.items(): + if values: + types_dict[key].objects.filter(id__in=values).delete() celery_tasks.delete() diff --git a/setup.cfg b/setup.cfg index 391df507..0c3df2dc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ project = 'Polemarch' repo = vstconsulting/polemarch assets = dist/polemarch-{release}.tar.gz + dist/polemarch-{release}-py3-none-any.whl dist/polemarch-{release}-cp35-cp35m-manylinux1_x86_64.whl dist/polemarch-{release}-cp36-cp36m-manylinux1_x86_64.whl dist/polemarch-{release}-cp37-cp37m-manylinux1_x86_64.whl diff --git a/tox.ini b/tox.ini index db0f8f14..5dd9e99e 100644 --- a/tox.ini +++ b/tox.ini @@ -102,7 +102,7 @@ whitelist_externals = rm commands = rm -rf dist build - tox -c tox_build.ini --workdir {toxworkdir} -e py35-build,py35-wheel,py36-wheel,py37-wheel + tox -c tox_build.ini --workdir {toxworkdir} [testenv:builddoc] basepython = python3.6 diff --git a/tox_build.ini b/tox_build.ini index 1c912646..761e5462 100644 --- a/tox_build.ini +++ b/tox_build.ini @@ -1,5 +1,6 @@ [tox] -envlist = py{35,36,37}-{build,wheel,rpm,deb} +# envlist = py35-build,py{35,36,37}-wheel +envlist = py35-{build,wheel} skipsdist = True [testenv] @@ -14,7 +15,7 @@ whitelist_externals = commands = build: make compile PY=python {posargs} wheel: make wheel PY=python {posargs} - wheel: bash -c "for whl in `ls dist/*.whl | grep -v manylinux`; do auditwheel repair $whl -w dist/ && rm $whl; done" + wheel: bash -c "for whl in `ls dist/*.whl | grep -v manylinux | grep -v none-any`; do auditwheel repair $whl -w dist/ && rm $whl; done" deb: make deb PY=python {posargs} rpm: make rpm PY=python {posargs} deps = @@ -25,4 +26,4 @@ deps = csscompressor==0.9.5 {rpm,deb}: virtualenv==16.0 {rpm,deb}: venvctrl - py{35,36}-wheel: auditwheel + py{35,36,37}-wheel: auditwheel