diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3a29e64e3..9061cf7fe 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.0rc22 +current_version = 2.0.1 commit = False tag = False allow_dirty = True diff --git a/docker/Dockerfile b/docker/Dockerfile index d27999203..6492429f6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ ARG BASE_IMAGE # && curl -L -u "${GITHUB_TOKEN}:" "https://github.com/unicef/etools-datamart/archive/${DATAMART_VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ # fi -FROM unicef/datamart-base:${BASE_IMAGE} +FROM unicef/datamart:${BASE_IMAGE}-base #COPY --from=loader /code /code ARG BUILD_DATE @@ -105,6 +105,7 @@ ENV UWSGI_PROTOCOL=http \ UWSGI_DISABLE_LOGGING=false \ UWSGI_DISABLE_WRITE_EXCEPTION=true \ UWSGI_FREEBIND=true \ + UWSGI_HARAKIRI=120 \ UWSGI_HTTP_TIMEOUT=60 \ UWSGI_IGNORE_SIGPIPE=true \ UWSGI_IGNORE_WRITE_ERRORS=true \ diff --git a/docker/Makefile b/docker/Makefile index 676752d31..b72c3115b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=1 DOCKER_PASS?= DOCKER_USER?= -TARGET?=2.0.0rc22 +TARGET?=2.0.1 BASE=2.0 # below vars are used internally BUILD_OPTIONS?=--squash @@ -30,10 +30,10 @@ build-base: cd .. && docker build \ ${BUILD_OPTIONS} \ --no-cache \ - -t ${DOCKER_IMAGE_NAME}-base:${BASE} \ + -t ${DOCKER_IMAGE_NAME}:${BASE}-base \ -f docker/Dockerfile.base . @echo ${DOCKER_PASS} | docker login -u ${DOCKER_USER} --password-stdin - docker push ${DOCKER_IMAGE_NAME}-base:${BASE} + docker push ${DOCKER_IMAGE_NAME}:${BASE}-base .build: diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index 17217f4a4..678b1b0bf 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,7 +1,7 @@ import warnings NAME = 'etools-datamart' -VERSION = __version__ = '2.0.0rc22' +VERSION = __version__ = '2.0.1' __author__ = '' # UserWarning: The psycopg2 wheel package will be renamed from release 2.8; diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index e38e8fa83..e29afee92 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -581,10 +581,9 @@ def load(self, *, verbosity=0, always_update=False, stdout=None, self.results.context = self.context self.fields_to_compare = [f for f in self.mapping.keys() if f not in ["seen"]] if truncate: - cache.set("STATUS:%s" % self.etl_task.task, '[truncating]') self.model.objects.truncate() for i, country in enumerate(countries, 1): - cache.set("STATUS:%s" % self.etl_task.task, country) + cache.set("STATUS:%s" % self.etl_task.task, "%s - %s" % (country, self.results.processed)) self.context['country'] = country if stdout and verbosity > 0: stdout.write(f"{i:>3}/{total_countries} " diff --git a/src/etools_datamart/apps/data/migrations/0089_auto_20190901_0643.py b/src/etools_datamart/apps/data/migrations/0089_auto_20190901_0643.py new file mode 100644 index 000000000..b25ee08f7 --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0089_auto_20190901_0643.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.4 on 2019-09-01 06:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0088_auto_20190828_2216'), + ] + + operations = [ + migrations.AlterField( + model_name='pdindicator', + name='baseline_denominator', + field=models.DecimalField(blank=True, decimal_places=3, max_digits=20, null=True), + ), + migrations.AlterField( + model_name='pdindicator', + name='baseline_numerator', + field=models.DecimalField(blank=True, decimal_places=3, max_digits=20, null=True), + ), + ] diff --git a/src/etools_datamart/apps/data/models/pd_indicator.py b/src/etools_datamart/apps/data/models/pd_indicator.py index 2fac3a966..165e36eea 100644 --- a/src/etools_datamart/apps/data/models/pd_indicator.py +++ b/src/etools_datamart/apps/data/models/pd_indicator.py @@ -66,9 +66,9 @@ class PDIndicator(LocationMixin, DataMartModel): # baseline = models.TextField(blank=True, null=True) # This field type is a guess. baseline_denominator = models.DecimalField(blank=True, null=True, - max_digits=8, decimal_places=3) + max_digits=20, decimal_places=3) baseline_numerator = models.DecimalField(blank=True, null=True, - max_digits=8, decimal_places=3) + max_digits=20, decimal_places=3) # from lower_result lower_result_name = models.CharField(max_length=500, blank=True, null=True) diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index 8f5991bb8..78f3f3f0c 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -151,7 +151,7 @@ def _last_failure(self, obj): def _status(self, obj): css = get_css(obj) - if 'RETRY' in obj.status: + if obj.status and 'RETRY' in obj.status: s = '%s
%s' % (obj.status, obj.results) else: c = cache.get("STATUS:%s" % obj.task) or "" diff --git a/src/unicef_rest_framework/renderers/__init__.py b/src/unicef_rest_framework/renderers/__init__.py index 9309ba7d3..f29b7785e 100644 --- a/src/unicef_rest_framework/renderers/__init__.py +++ b/src/unicef_rest_framework/renderers/__init__.py @@ -8,3 +8,4 @@ from .pdf import PDFRenderer # noqa from .txt import TextRenderer # noqa from .xls import XLSXRenderer # noqa +from .yaml import YAMLRenderer # noqa diff --git a/src/unicef_rest_framework/renderers/yaml.py b/src/unicef_rest_framework/renderers/yaml.py new file mode 100644 index 000000000..71b4e9a46 --- /dev/null +++ b/src/unicef_rest_framework/renderers/yaml.py @@ -0,0 +1,29 @@ +import logging + +from crashlog.middleware import process_exception +from rest_framework_yaml.renderers import YAMLRenderer as BaseRenderer + +from unicef_rest_framework.renderers.mixin import ContentDispositionMixin + +logger = logging.getLogger(__name__) + + +def labelize(v): + return v.replace("_", " ").title() + + +class YAMLRenderer(ContentDispositionMixin, BaseRenderer): + format = 'yaml' + + def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + self.process_response(renderer_context) + if response.status_code != 200: + return '' + try: + return super().render(data, accepted_media_type, renderer_context) + + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception(f'Error processing request {e}') from e diff --git a/src/unicef_rest_framework/views.py b/src/unicef_rest_framework/views.py index 90edfa449..263c58ffb 100644 --- a/src/unicef_rest_framework/views.py +++ b/src/unicef_rest_framework/views.py @@ -8,7 +8,6 @@ from rest_framework import viewsets from rest_framework.authentication import BasicAuthentication, SessionAuthentication, TokenAuthentication from rest_framework_xml.renderers import XMLRenderer -from rest_framework_yaml.renderers import YAMLRenderer from strategy_field.utils import fqn from unicef_rest_framework.pagination import PageFilter @@ -21,8 +20,8 @@ from .negotiation import CT from .ordering import OrderingFilter from .permissions import ServicePermission -from .renderers import (CSVRenderer, HTMLRenderer, IQYRenderer, JSONRenderer, MSJSONRenderer, - MSXmlRenderer, PDFRenderer, TextRenderer, URFBrowsableAPIRenderer, XLSXRenderer,) +from .renderers import (CSVRenderer, HTMLRenderer, IQYRenderer, JSONRenderer, MSJSONRenderer, MSXmlRenderer, + PDFRenderer, TextRenderer, URFBrowsableAPIRenderer, XLSXRenderer, YAMLRenderer,) class classproperty(object):