diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..9d0ba33 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +source = picklefield +branch = True + +[report] +exclude_lines = + pragma: no cover + except ImportError: diff --git a/.gitignore b/.gitignore index 1340094..4bd06b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ -*.pyc -*.pyo -/build -/dist -/temp -/src/django_picklefield.egg-info - +syntax:glob +*.py[co] +dist/ +django_picklefield.egg-info/* +build/ +.coverage +.tox diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..aae3926 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,35 @@ +dist: bionic +sudo: false +language: python +cache: pip +python: + - 2.7 + - 3.5 + - 3.6 + - 3.7 + - 3.8 +stages: + - lint + - test +jobs: + fast_finish: true + include: + - { stage: lint, env: TOXENV=flake8, python: 3.6 } + - { stage: lint, env: TOXENV=isort, python: 3.6 } + +install: +- pip install tox coveralls tox-travis +script: +- tox +after_success: +- if [ -f .coverage ]; then coveralls; fi +before_deploy: "pip install django" +deploy: + provider: pypi + user: charettes + password: + secure: Ix/iLGKu+usjaNGPIKRzvgdZz8TVxReiMFLorgMpHb9P29tgIBQSDzVmo8gzUi+LF4SQs2AdnuYJY15ap68JX7iLfCuWWYIvIqgMLb3W64jfUDg9KpLJQYCv6cTKQHoDVtVTg0jXJmauEYMMYuqmdrSgZ6rSfpsbajMrveCzVuStb7kd5H4WPX3/j4JfCmr07CyAi85rLAnaIfmCYADWC4nrYll7+0Ugke64C4KstY0l6HYBNTT5ZwJvXSNd0VQd31b5okv8bwjGXzeVrbgIAe+c1p0lS11hsNqfLL/rrnlvgC7P7568TxGwWVmIDJm131r1K7skLscEfCuPbJJ3oqljxLQMQIJ4JbR12HtRwB93dOnJOgCbqySAJAZHUvkm2r3VAO8BJK3xYnJg/2IGrxolDQxwJi2SMEgwqQZx3thTFKXvviiWn5dnLjiNRpSOHUzayCVYga7WAgVksKVMbmtcNrJWwrAHtWLxo9vnM2W+6JNFeXiGIIwGu7qJ/Sz/qCuv/uKibbFqhUJfATXf5HhSJ3yIM/Ef04/SSpBqkYcFcrGLdzzW2Bpv4jscv4EcNyGDxnzeJtvXWVceoW0g7XweUn4b5P+NvOSV4YCjJ04BU4r/upV7+ulkQxnCWot2MD0zhvlodldn+6tGfdBbYJ/KlBHQjwdjFFKaxqRkuyA= + distributions: sdist bdist_wheel + on: + tags: true + condition: "$TRAVIS_PYTHON_VERSION = 2.7" diff --git a/LICENSE b/LICENSE index 19459ea..c97bbdd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ Copyright (c) 2009-2010 Gintautas Miliauskas +Copyright (c) 2011-2018 Simon Charette Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9c8317c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE +include README.rst \ No newline at end of file diff --git a/README b/README.rst similarity index 54% rename from README rename to README.rst index fb3f937..6639493 100644 --- a/README +++ b/README.rst @@ -1,3 +1,30 @@ +django-picklefield +================== + +.. image:: https://img.shields.io/pypi/l/django-picklefield.svg?style=flat + :target: https://pypi.python.org/pypi/django-picklefield/ + :alt: License + +.. image:: https://img.shields.io/pypi/v/django-picklefield.svg?style=flat + :target: https://pypi.python.org/pypi/django-picklefield/ + :alt: Latest Version + +.. image:: https://travis-ci.org/gintas/django-picklefield.svg?branch=master + :target: https://travis-ci.org/gintas/django-picklefield + :alt: Build Status + +.. image:: https://coveralls.io/repos/gintas/django-picklefield/badge.svg?branch=master + :target: https://coveralls.io/r/gintas/django-picklefield?branch=master + :alt: Coverage Status + +.. image:: https://img.shields.io/pypi/pyversions/django-picklefield.svg?style=flat + :target: https://pypi.python.org/pypi/django-picklefield/ + :alt: Supported Python Versions + +.. image:: https://img.shields.io/pypi/wheel/django-picklefield.svg?style=flat + :target: https://pypi.python.org/pypi/django-picklefield/ + :alt: Wheel Status + ----- About ----- @@ -5,13 +32,13 @@ About **django-picklefield** provides an implementation of a pickled object field. Such fields can contain any picklable objects. -The implementation is taken and adopted from Django snippet #1694 - by Taavi Taijala, which is in -turn based on Django snippet #513 -by Oliver Beattie. +The implementation is taken and adopted from `Django snippet #1694`_ by Taavi +Taijala, which is in turn based on `Django snippet #513`_ by Oliver Beattie. django-picklefield is available under the MIT license. +.. _Django snippet #1694: http://www.djangosnippets.org/snippets/1694/ +.. _Django snippet #513: http://www.djangosnippets.org/snippets/513/ ----- Usage @@ -20,13 +47,17 @@ Usage First of all, you need to have **django-picklefield** installed; for your convenience, recent versions should be available from PyPI. -To use, just define a field in your model:: +To use, just define a field in your model: + +.. code:: python >>> from picklefield.fields import PickledObjectField ... class SomeObject(models.Model): ... args = PickledObjectField() -and assign whatever you like (as long as it's picklable) to the field:: +and assign whatever you like (as long as it's picklable) to the field: + +.. code:: python >>> obj = SomeObject() >>> obj.args = ['fancy', {'objects': 'inside'}] @@ -44,6 +75,23 @@ base64-encoded pickles, which allows reliable deserialization, but such a format is not convenient for parsing in the browser. By overriding ``value_to_string()`` you can choose a more convenient serialization format. +Fields now accept the boolean key word argument `copy`, which defaults to +`True`. The `copy` is necessary for lookups to work correctly. If you don't +care about performing lookups on the picklefield, you can set `copy=False` to +save on some memory usage. This an be especially beneficial for very large +object trees. + +------------- +Running tests +------------- + +The full test suite can be run with `Tox`_:: + + >>> pip install tox + >>> tox + +.. _Tox: https://testrun.org/tox/latest/ + -------------- Original notes -------------- @@ -114,55 +162,117 @@ since it is never a good idea to have a PickledObjectField be user editable. Changes ------- +Changes in version 2.1.0 +======================== + +* Added official support for Django 2.2 (thanks to joehybird). +* Dropped support for Django 2.0 and 2.1 (thanks to joehybird). +* Dropped support for Python 3.4 (thanks to joehybidd). + +Changes in version 2.0.0 +======================== + +* Silenced ``RemovedInDjango30Warning`` warnings on Django 2.0+ (thanks to + canarduck). +* Restructured project directories. +* Disallowed the usage of empty strings for ``PickledObjectField``. That makes + ``.save()``, ``.create()``, etc. raise ``IntegrityError`` if `null` is not + ``True`` and no default value was specified like built-in fields do + (thanks to Attila-Mihaly Balazs). +* Added a check for mutable default values to ``PickledObjectField``. + +Changes in version 1.1.0 +======================== + +* Added support for Django 2.1 and dropped support for Django < 1.11. + +Changes in version 1.0.0 +======================== + +* Added a new option to prevent a copy of the object before pickling: `copy=True` +* Dropped support for Django 1.4 +* Dropped support for Django 1.7 +* Dropped support for Python 3.2 +* Added support for Python 3.6 + +Changes in version 0.3.2 +======================== + +* Dropped support for Django 1.3. +* Dropped support for Python 2.5. +* Silenced deprecation warnings on Django 1.8+. + +Changes in version 0.3.1 +======================== + +* Favor the built in json module (thanks to Simon Charette). + +Changes in version 0.3.0 +======================== + +* Python 3 support (thanks to Rafal Stozek). + +Changes in version 0.2.0 +======================== + +* Allow pickling of subclasses of django.db.models.Model (thanks to Simon + Charette). + +Changes in version 0.1.9 +======================== + +* Added `connection` and `prepared` parameters to `get_db_prep_value()` too + (thanks to Matthew Schinckel). + Changes in version 0.1.8 ======================== - * Updated link to code repository. +* Updated link to code repository. Changes in version 0.1.7 ======================== - * Added `connection` and `prepared` parameters to `get_db_prep_lookup()` to - get rid of deprecation warnings in Django 1.2. +* Added `connection` and `prepared` parameters to `get_db_prep_lookup()` to + get rid of deprecation warnings in Django 1.2. Changes in version 0.1.6 ======================== - * Fixed South support (thanks aehlke@github). +* Fixed South support (thanks aehlke@github). Changes in version 0.1.5 ======================== - * Added support for South. - * Changed default to null=False, as is common throughout Django. +* Added support for South. +* Changed default to null=False, as is common throughout Django. Changes in version 0.1.4 ======================== - * Updated copyright statements. +* Updated copyright statements. Changes in version 0.1.3 ======================== - * Updated serialization tests (thanks to Michael Fladischer). +* Updated serialization tests (thanks to Michael Fladischer). Changes in version 0.1.2 ======================== - * Added Simplified BSD licence. +* Added Simplified BSD licence. Changes in version 0.1.1 ======================== - * Added test for serialization. - * Added note about JSON serialization for browser. - * Added support for different pickle protocol versions (thanks to Michael - Fladischer). +* Added test for serialization. +* Added note about JSON serialization for browser. +* Added support for different pickle protocol versions (thanks to Michael + Fladischer). Changes in version 0.1 ====================== - * First public release. +* First public release. -------- diff --git a/picklefield/__init__.py b/picklefield/__init__.py new file mode 100644 index 0000000..2f0479c --- /dev/null +++ b/picklefield/__init__.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals + +import django.utils.version + +from .constants import DEFAULT_PROTOCOL +from .fields import PickledObjectField + +__all__ = 'VERSION', '__version__', 'DEFAULT_PROTOCOL', 'PickledObjectField' + +VERSION = (2, 1, 1, 'final', 0) + +__version__ = django.utils.version.get_version(VERSION) diff --git a/picklefield/constants.py b/picklefield/constants.py new file mode 100644 index 0000000..d3c1743 --- /dev/null +++ b/picklefield/constants.py @@ -0,0 +1,3 @@ +from __future__ import unicode_literals + +DEFAULT_PROTOCOL = 2 diff --git a/src/picklefield/fields.py b/picklefield/fields.py similarity index 53% rename from src/picklefield/fields.py rename to picklefield/fields.py index 0e2de16..45a4f31 100644 --- a/src/picklefield/fields.py +++ b/picklefield/fields.py @@ -1,17 +1,21 @@ -"""Pickle field implementation for Django.""" +from __future__ import unicode_literals +from base64 import b64decode, b64encode from copy import deepcopy -from base64 import b64encode, b64decode from zlib import compress, decompress -try: - from cPickle import loads, dumps -except ImportError: - from pickle import loads, dumps +from django import VERSION as DJANGO_VERSION +from django.core import checks from django.db import models -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text + +from .constants import DEFAULT_PROTOCOL + +try: + from cPickle import loads, dumps # pragma: no cover +except ImportError: + from pickle import loads, dumps # pragma: no cover -from picklefield import DEFAULT_PROTOCOL class PickledObject(str): """ @@ -24,30 +28,54 @@ class PickledObject(str): remove PickledObject and its references, you won't be able to pass in pre-encoded values anymore, but you can always just pass in the python objects themselves. + """ + + +class _ObjectWrapper(object): + """ + A class used to wrap object that have properties that may clash with the + ORM internals. + For example, objects with the `prepare_database_save` property such as + `django.db.Model` subclasses won't work under certain conditions and the + same apply for trying to retrieve any `callable` object. """ + __slots__ = ('_obj',) + + def __init__(self, obj): + self._obj = obj + +def wrap_conflictual_object(obj): + if hasattr(obj, 'prepare_database_save') or callable(obj): + obj = _ObjectWrapper(obj) + return obj -def dbsafe_encode(value, compress_object=False, pickle_protocol=DEFAULT_PROTOCOL): + +def dbsafe_encode(value, compress_object=False, pickle_protocol=DEFAULT_PROTOCOL, copy=True): # We use deepcopy() here to avoid a problem with cPickle, where dumps # can generate different character streams for same lookup value if # they are referenced differently. # The reason this is important is because we do all of our lookups as # simple string matches, thus the character streams must be the same # for the lookups to work properly. See tests.py for more information. - if not compress_object: - value = b64encode(dumps(deepcopy(value), pickle_protocol)) - else: - value = b64encode(compress(dumps(deepcopy(value), pickle_protocol))) + if copy: + # Copy can be very expensive if users aren't going to perform lookups + # on the value anyway. + value = deepcopy(value) + value = dumps(value, protocol=pickle_protocol) + if compress_object: + value = compress(value) + value = b64encode(value).decode() # decode bytes to str return PickledObject(value) def dbsafe_decode(value, compress_object=False): - if not compress_object: - value = loads(b64decode(value)) - else: - value = loads(decompress(b64decode(value))) - return value + value = value.encode() # encode str to bytes + value = b64decode(value) + if compress_object: + value = decompress(value) + return loads(value) class PickledObjectField(models.Field): @@ -60,12 +88,12 @@ class PickledObjectField(models.Field): can still do lookups using None). This way, it is still possible to use the ``isnull`` lookup type correctly. """ - - __metaclass__ = models.SubfieldBase + empty_strings_allowed = False def __init__(self, *args, **kwargs): self.compress = kwargs.pop('compress', False) self.protocol = kwargs.pop('protocol', DEFAULT_PROTOCOL) + self.copy = kwargs.pop('copy', True) kwargs.setdefault('editable', False) super(PickledObjectField, self).__init__(*args, **kwargs) @@ -88,12 +116,39 @@ def get_default(self): # If the field doesn't have a default, then we punt to models.Field. return super(PickledObjectField, self).get_default() + def _check_default(self): + if self.has_default() and isinstance(self.default, (list, dict, set)): + return [ + checks.Warning( + "%s default should be a callable instead of a mutable instance so " + "that it's not shared between all field instances." % ( + self.__class__.__name__, + ), + hint=( + 'Use a callable instead, e.g., use `%s` instead of ' + '`%r`.' % ( + type(self.default).__name__, + self.default, + ) + ), + obj=self, + id='picklefield.E001', + ) + ] + else: + return [] + + def check(self, **kwargs): + errors = super(PickledObjectField, self).check(**kwargs) + errors.extend(self._check_default()) + return errors + def to_python(self, value): """ B64decode and unpickle the object, optionally decompressing it. If an error is raised in de-pickling and we're sure the value is - a definite pickle, the error is allowed to propogate. If we + a definite pickle, the error is allowed to propagate. If we aren't sure if the value is a pickle or not, then we catch the error and return the original value instead. @@ -101,14 +156,28 @@ def to_python(self, value): if value is not None: try: value = dbsafe_decode(value, self.compress) - except: + except Exception: # If the value is a definite pickle; and an error is raised in - # de-pickling it should be allowed to propogate. + # de-pickling it should be allowed to propagate. if isinstance(value, PickledObject): raise + else: + if isinstance(value, _ObjectWrapper): + return value._obj return value - def get_db_prep_value(self, value): + def pre_save(self, model_instance, add): + value = super(PickledObjectField, self).pre_save(model_instance, add) + return wrap_conflictual_object(value) + + if DJANGO_VERSION < (2, 0): + def from_db_value(self, value, expression, connection, context): # pragma: no cover + return self.to_python(value) # pragma: no cover + else: + def from_db_value(self, value, expression, connection): # pragma: no cover + return self.to_python(value) # pragma: no cover + + def get_db_prep_value(self, value, connection=None, prepared=False): """ Pickle and b64encode the object, optionally compressing it. @@ -120,41 +189,26 @@ def get_db_prep_value(self, value): """ if value is not None and not isinstance(value, PickledObject): - # We call force_unicode here explicitly, so that the encoded string + # We call force_text here explicitly, so that the encoded string # isn't rejected by the postgresql_psycopg2 backend. Alternatively, # we could have just registered PickledObject with the psycopg # marshaller (telling it to store it like it would a string), but # since both of these methods result in the same value being stored, # doing things this way is much easier. - value = force_unicode(dbsafe_encode(value, self.compress, self.protocol)) + value = force_text(dbsafe_encode(value, self.compress, self.protocol, self.copy)) return value def value_to_string(self, obj): - value = self._get_val_from_obj(obj) + value = self.value_from_object(obj) return self.get_db_prep_value(value) def get_internal_type(self): return 'TextField' - def get_db_prep_lookup(self, lookup_type, value, connection=None, prepared=False): - if lookup_type not in ['exact', 'in', 'isnull']: - raise TypeError('Lookup type %s is not supported.' % lookup_type) - # The Field model already calls get_db_prep_value before doing the - # actual lookup, so all we need to do is limit the lookup types. - try: - return super(PickledObjectField, self).get_db_prep_lookup( - lookup_type, value, connection=connection, prepared=prepared) - except TypeError: - # Try not to break on older versions of Django, where the - # `connection` and `prepared` parameters are not available. - return super(PickledObjectField, self).get_db_prep_lookup( - lookup_type, value) - - -# South support; see http://south.aeracode.org/docs/tutorial/part4.html#simple-inheritance -try: - from south.modelsinspector import add_introspection_rules -except ImportError: - pass -else: - add_introspection_rules([], [r"^picklefield\.fields\.PickledObjectField"]) + def get_lookup(self, lookup_name): + """ + We need to limit the lookup types. + """ + if lookup_name not in ['exact', 'in', 'isnull']: + raise TypeError('Lookup type %s is not supported.' % lookup_name) + return super(PickledObjectField, self).get_lookup(lookup_name) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b4b3ce2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +[flake8] +max-line-length = 119 + +[isort] +combine_as_imports=true +include_trailing_comma=true +multi_line_output=5 +not_skip=__init__.py + +[wheel] +universal = 1 diff --git a/setup.py b/setup.py index 1655ef8..24ec734 100644 --- a/setup.py +++ b/setup.py @@ -1,39 +1,45 @@ -# Copyright (c) 2009-2010 Gintautas Miliauskas -# Copyright (c) 2009, Taavi Taijala -# Copyright (c) 2007, Oliver Beattie -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. +from __future__ import unicode_literals -from setuptools import setup, find_packages +from setuptools import find_packages, setup -setup(name='django-picklefield', - version='0.1.8', +import picklefield + +with open('README.rst') as file_: + long_description = file_.read() + +setup( + name='django-picklefield', + version=picklefield.__version__, description='Pickled object field for Django', - long_description=open('README').read(), - author='Gintautas Miliauskas', - author_email='gintautas@miliauskas.lt', + long_description=long_description, + author='Simon Charette', + author_email='charette.s+django-picklefiel@gmail.com', url='http://github.com/gintas/django-picklefield', - packages=find_packages('src'), - package_dir={'' : 'src'}, + license='MIT', classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', + 'Environment :: Web Environment', 'Framework :: Django', + 'Framework :: Django :: 1.11', + 'Framework :: Django :: 2.2', + 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', - ] + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + keywords=['django pickle model field'], + packages=find_packages(exclude=['tests', 'tests.*']), + install_requires=['Django>=1.11'], + extras_require={ + 'tests': ['tox'], + }, ) diff --git a/src/picklefield/__init__.py b/src/picklefield/__init__.py deleted file mode 100644 index d8269fd..0000000 --- a/src/picklefield/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Pickle field implementation for Django.""" - -DEFAULT_PROTOCOL = 2 - -from picklefield.fields import PickledObjectField # reexport diff --git a/src/picklefield/tests.py b/src/picklefield/tests.py deleted file mode 100644 index a18bb68..0000000 --- a/src/picklefield/tests.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Unit tests for django-picklefield.""" - -from django.test import TestCase -from django.db import models -from django.core import serializers -from picklefield.fields import PickledObjectField - -class TestingModel(models.Model): - pickle_field = PickledObjectField() - compressed_pickle_field = PickledObjectField(compress=True) - default_pickle_field = PickledObjectField(default=({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5])) - -class MinimalTestingModel(models.Model): - pickle_field = PickledObjectField() - -class TestCustomDataType(str): - pass - -class PickledObjectFieldTests(TestCase): - def setUp(self): - self.testing_data = ( - {1:2, 2:4, 3:6, 4:8, 5:10}, - 'Hello World', - (1, 2, 3, 4, 5), - [1, 2, 3, 4, 5], - TestCustomDataType('Hello World'), - ) - return super(PickledObjectFieldTests, self).setUp() - - def testDataIntegriry(self): - """ - Tests that data remains the same when saved to and fetched from - the database, whether compression is enabled or not. - - """ - for value in self.testing_data: - model_test = TestingModel(pickle_field=value, compressed_pickle_field=value) - model_test.save() - model_test = TestingModel.objects.get(id__exact=model_test.id) - # Make sure that both the compressed and uncompressed fields return - # the same data, even thought it's stored differently in the DB. - self.assertEquals(value, model_test.pickle_field) - self.assertEquals(value, model_test.compressed_pickle_field) - model_test.delete() - - # Make sure the default value for default_pickled_field gets stored - # correctly and that it isn't converted to a string. - model_test = TestingModel() - model_test.save() - model_test = TestingModel.objects.get(id__exact=model_test.id) - self.assertEquals(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]), model_test.default_pickle_field) - - def testLookups(self): - """ - Tests that lookups can be performed on data once stored in the - database, whether compression is enabled or not. - - One problem with cPickle is that it will sometimes output - different streams for the same object, depending on how they are - referenced. It should be noted though, that this does not happen - for every object, but usually only with more complex ones. - - >>> from pickle import dumps - >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, \ - ... 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) - >>> dumps(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, \ - ... 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5])) - "((dp0\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np1\n(I1\nI2\nI3\nI4\nI5\ntp2\n(lp3\nI1\naI2\naI3\naI4\naI5\natp4\n." - >>> dumps(t) - "((dp0\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np1\n(I1\nI2\nI3\nI4\nI5\ntp2\n(lp3\nI1\naI2\naI3\naI4\naI5\natp4\n." - >>> # Both dumps() are the same using pickle. - - >>> from cPickle import dumps - >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) - >>> dumps(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5])) - "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." - >>> dumps(t) - "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\n(I1\nI2\nI3\nI4\nI5\nt(lp2\nI1\naI2\naI3\naI4\naI5\natp3\n." - >>> # But with cPickle the two dumps() are not the same! - >>> # Both will generate the same object when loads() is called though. - - We can solve this by calling deepcopy() on the value before - pickling it, as this copies everything to a brand new data - structure. - - >>> from cPickle import dumps - >>> from copy import deepcopy - >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) - >>> dumps(deepcopy(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]))) - "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." - >>> dumps(deepcopy(t)) - "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." - >>> # Using deepcopy() beforehand means that now both dumps() are idential. - >>> # It may not be necessary, but deepcopy() ensures that lookups will always work. - - Unfortunately calling copy() alone doesn't seem to fix the - problem as it lies primarily with complex data types. - - >>> from cPickle import dumps - >>> from copy import copy - >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) - >>> dumps(copy(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]))) - "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." - >>> dumps(copy(t)) - "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\n(I1\nI2\nI3\nI4\nI5\nt(lp2\nI1\naI2\naI3\naI4\naI5\natp3\n." - - """ - for value in self.testing_data: - model_test = TestingModel(pickle_field=value, compressed_pickle_field=value) - model_test.save() - # Make sure that we can do an ``exact`` lookup by both the - # pickle_field and the compressed_pickle_field. - model_test = TestingModel.objects.get(pickle_field__exact=value, compressed_pickle_field__exact=value) - self.assertEquals(value, model_test.pickle_field) - self.assertEquals(value, model_test.compressed_pickle_field) - # Make sure that ``in`` lookups also work correctly. - model_test = TestingModel.objects.get(pickle_field__in=[value], compressed_pickle_field__in=[value]) - self.assertEquals(value, model_test.pickle_field) - self.assertEquals(value, model_test.compressed_pickle_field) - # Make sure that ``is_null`` lookups are working. - self.assertEquals(1, TestingModel.objects.filter(pickle_field__isnull=False).count()) - self.assertEquals(0, TestingModel.objects.filter(pickle_field__isnull=True).count()) - model_test.delete() - - # Make sure that lookups of the same value work, even when referenced - # differently. See the above docstring for more info on the issue. - value = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) - model_test = TestingModel(pickle_field=value, compressed_pickle_field=value) - model_test.save() - # Test lookup using an assigned variable. - model_test = TestingModel.objects.get(pickle_field__exact=value) - self.assertEquals(value, model_test.pickle_field) - # Test lookup using direct input of a matching value. - model_test = TestingModel.objects.get( - pickle_field__exact = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]), - compressed_pickle_field__exact = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]), - ) - self.assertEquals(value, model_test.pickle_field) - model_test.delete() - - def testSerialization(self): - model_test = MinimalTestingModel(pickle_field={'foo': 'bar'}) - json_test = serializers.serialize('json', [model_test]) - self.assertEquals(json_test, - '[{"pk": null,' - ' "model": "picklefield.minimaltestingmodel",' - ' "fields": {"pickle_field": "gAJ9cQFVA2Zvb3ECVQNiYXJxA3Mu"}}]') - for deserialized_test in serializers.deserialize('json', json_test): - self.assertEquals(deserialized_test.object, - model_test) diff --git a/src/picklefield/models.py b/tests/__init__.py similarity index 100% rename from src/picklefield/models.py rename to tests/__init__.py diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 0000000..3c80b6e --- /dev/null +++ b/tests/models.py @@ -0,0 +1,34 @@ +from __future__ import unicode_literals + +from datetime import date + +from django.db import models +from picklefield import PickledObjectField + +S1 = 'Hello World' +T1 = (1, 2, 3, 4, 5) +L1 = [1, 2, 3, 4, 5] +D1 = {1: 1, 2: 4, 3: 6, 4: 8, 5: 10} +D2 = {1: 2, 2: 4, 3: 6, 4: 8, 5: 10} + + +class TestCopyDataType(str): + def __deepcopy__(self, memo): + raise ValueError('Please dont copy me') + + +class TestCustomDataType(str): + pass + + +class TestingModel(models.Model): + pickle_field = PickledObjectField() + compressed_pickle_field = PickledObjectField(compress=True) + default_pickle_field = PickledObjectField(default=(D1, S1, T1, L1)) + callable_pickle_field = PickledObjectField(default=date.today) + non_copying_field = PickledObjectField(copy=False, default=TestCopyDataType('boom!')) + nullable_pickle_field = PickledObjectField(null=True) + + +class MinimalTestingModel(models.Model): + pickle_field = PickledObjectField() diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..e2a67fa --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,13 @@ +from __future__ import unicode_literals + +SECRET_KEY = 'not-anymore' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + } +} + +INSTALLED_APPS = [ + 'tests', +] diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000..f162588 --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,246 @@ +import json +from datetime import date + +from django.core import checks, serializers +from django.db import IntegrityError, models +from django.test import SimpleTestCase, TestCase +from django.test.utils import isolate_apps +from picklefield.fields import ( + PickledObjectField, dbsafe_encode, wrap_conflictual_object, +) + +from .models import ( + D1, D2, L1, S1, T1, MinimalTestingModel, TestCopyDataType, + TestCustomDataType, TestingModel, +) + +try: + from unittest.mock import patch # pragma: no cover +except ImportError: + from mock import patch # pragma: no cover + + +class PickledObjectFieldTests(TestCase): + def setUp(self): + self.testing_data = (D2, S1, T1, L1, + TestCustomDataType(S1), + MinimalTestingModel) + return super(PickledObjectFieldTests, self).setUp() + + def test_data_integrity(self): + """ + Tests that data remains the same when saved to and fetched from + the database, whether compression is enabled or not. + """ + for value in self.testing_data: + model_test = TestingModel(pickle_field=value, compressed_pickle_field=value) + model_test.save() + model_test = TestingModel.objects.get(id__exact=model_test.id) + # Make sure that both the compressed and uncompressed fields return + # the same data, even thought it's stored differently in the DB. + self.assertEqual(value, model_test.pickle_field) + self.assertEqual(value, model_test.compressed_pickle_field) + self.assertIsNone(model_test.nullable_pickle_field) + # Make sure we can also retrieve the model + model_test.save() + model_test.delete() + + # Make sure the default value for default_pickled_field gets stored + # correctly and that it isn't converted to a string. + model_test = TestingModel(pickle_field=1, compressed_pickle_field=1) + model_test.save() + model_test = TestingModel.objects.get(id__exact=model_test.id) + self.assertEqual((D1, S1, T1, L1), model_test.default_pickle_field) + self.assertEqual(date.today(), model_test.callable_pickle_field) + + def test_lookups(self): + """ + Tests that lookups can be performed on data once stored in the + database, whether compression is enabled or not. + + One problem with cPickle is that it will sometimes output + different streams for the same object, depending on how they are + referenced. It should be noted though, that this does not happen + for every object, but usually only with more complex ones. + + >>> from pickle import dumps + >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, \ + ... 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) + >>> dumps(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, \ + ... 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5])) + "((dp0\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np1\n(I1\nI2\nI3\nI4\nI5\ntp2\n(lp3\nI1\naI2\naI3\naI4\naI5\natp4\n." + >>> dumps(t) + "((dp0\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np1\n(I1\nI2\nI3\nI4\nI5\ntp2\n(lp3\nI1\naI2\naI3\naI4\naI5\natp4\n." + >>> # Both dumps() are the same using pickle. + + >>> from cPickle import dumps + >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) + >>> dumps(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5])) + "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." + >>> dumps(t) + "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\n(I1\nI2\nI3\nI4\nI5\nt(lp2\nI1\naI2\naI3\naI4\naI5\natp3\n." + >>> # But with cPickle the two dumps() are not the same! + >>> # Both will generate the same object when loads() is called though. + + We can solve this by calling deepcopy() on the value before + pickling it, as this copies everything to a brand new data + structure. + + >>> from cPickle import dumps + >>> from copy import deepcopy + >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) + >>> dumps(deepcopy(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]))) + "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." + >>> dumps(deepcopy(t)) + "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." + >>> # Using deepcopy() beforehand means that now both dumps() are identical. + >>> # It may not be necessary, but deepcopy() ensures that lookups will always work. + + Unfortunately calling copy() alone doesn't seem to fix the + problem as it lies primarily with complex data types. + + >>> from cPickle import dumps + >>> from copy import copy + >>> t = ({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]) + >>> dumps(copy(({1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5]))) + "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\np2\n(I1\nI2\nI3\nI4\nI5\ntp3\n(lp4\nI1\naI2\naI3\naI4\naI5\nat." + >>> dumps(copy(t)) + "((dp1\nI1\nI1\nsI2\nI4\nsI3\nI6\nsI4\nI8\nsI5\nI10\nsS'Hello World'\n(I1\nI2\nI3\nI4\nI5\nt(lp2\nI1\naI2\naI3\naI4\naI5\natp3\n." + + """ # noqa + for value in self.testing_data: + model_test = TestingModel(pickle_field=value, compressed_pickle_field=value) + model_test.save() + # Make sure that we can do an ``exact`` lookup by both the + # pickle_field and the compressed_pickle_field. + wrapped_value = wrap_conflictual_object(value) + model_test = TestingModel.objects.get(pickle_field__exact=wrapped_value, + compressed_pickle_field__exact=wrapped_value) + self.assertEqual(value, model_test.pickle_field) + self.assertEqual(value, model_test.compressed_pickle_field) + # Make sure that ``in`` lookups also work correctly. + model_test = TestingModel.objects.get(pickle_field__in=[wrapped_value], + compressed_pickle_field__in=[wrapped_value]) + self.assertEqual(value, model_test.pickle_field) + self.assertEqual(value, model_test.compressed_pickle_field) + # Make sure that ``is_null`` lookups are working. + self.assertEqual(1, TestingModel.objects.filter(pickle_field__isnull=False).count()) + self.assertEqual(0, TestingModel.objects.filter(pickle_field__isnull=True).count()) + model_test.delete() + + # Make sure that lookups of the same value work, even when referenced + # differently. See the above docstring for more info on the issue. + value = (D1, S1, T1, L1) + model_test = TestingModel(pickle_field=value, compressed_pickle_field=value) + model_test.save() + # Test lookup using an assigned variable. + model_test = TestingModel.objects.get(pickle_field__exact=value) + self.assertEqual(value, model_test.pickle_field) + # Test lookup using direct input of a matching value. + model_test = TestingModel.objects.get( + pickle_field__exact=(D1, S1, T1, L1), + compressed_pickle_field__exact=(D1, S1, T1, L1), + ) + self.assertEqual(value, model_test.pickle_field) + model_test.delete() + + def test_limit_lookups_type(self): + """ + Test that picklefield supports lookup type limit + """ + with self.assertRaisesMessage(TypeError, 'Lookup type gte is not supported'): + TestingModel.objects.filter(pickle_field__gte=1) + + def test_serialization(self): + model = MinimalTestingModel(pk=1, pickle_field={'foo': 'bar'}) + serialized = serializers.serialize('json', [model]) + data = json.loads(serialized) + + # determine output at runtime, because pickle output in python 3 + # is different (but compatible with python 2) + p = dbsafe_encode({'foo': 'bar'}) + + self.assertEqual(data, [{ + 'pk': 1, + 'model': 'tests.minimaltestingmodel', + 'fields': {"pickle_field": p}}, + ]) + + for deserialized_test in serializers.deserialize('json', serialized): + self.assertEqual(deserialized_test.object, model) + + def test_no_copy(self): + TestingModel.objects.create( + pickle_field='Copy Me', + compressed_pickle_field='Copy Me', + non_copying_field=TestCopyDataType('Dont Copy Me') + ) + + with self.assertRaises(ValueError): + TestingModel.objects.create( + pickle_field=TestCopyDataType('BOOM!'), + compressed_pickle_field='Copy Me', + non_copying_field='Dont copy me' + ) + + def test_empty_strings_not_allowed(self): + with self.assertRaises(IntegrityError): + MinimalTestingModel.objects.create() + + def test_decode_error(self): + def mock_decode_error(*args, **kwargs): + raise Exception() + + model = MinimalTestingModel.objects.create(pickle_field={'foo': 'bar'}) + model.save() + + self.assertEqual( + {'foo': 'bar'}, MinimalTestingModel.objects.get(pk=model.pk).pickle_field + ) + + with patch('picklefield.fields.dbsafe_decode', mock_decode_error): + encoded_value = dbsafe_encode({'foo': 'bar'}) + self.assertEqual(encoded_value, MinimalTestingModel.objects.get(pk=model.pk).pickle_field) + + +@isolate_apps('tests') +class PickledObjectFieldCheckTests(SimpleTestCase): + def test_mutable_default_check(self): + class Model(models.Model): + list_field = PickledObjectField(default=[]) + dict_field = PickledObjectField(default={}) + set_field = PickledObjectField(default=set()) + + msg = ( + "PickledObjectField default should be a callable instead of a mutable instance so " + "that it's not shared between all field instances." + ) + + self.assertEqual(Model().check(), [ + checks.Warning( + msg=msg, + hint='Use a callable instead, e.g., use `list` instead of `[]`.', + obj=Model._meta.get_field('list_field'), + id='picklefield.E001', + ), + checks.Warning( + msg=msg, + hint='Use a callable instead, e.g., use `dict` instead of `{}`.', + obj=Model._meta.get_field('dict_field'), + id='picklefield.E001', + ), + checks.Warning( + msg=msg, + hint='Use a callable instead, e.g., use `set` instead of `%s`.' % repr(set()), + obj=Model._meta.get_field('set_field'), + id='picklefield.E001', + ) + ]) + + def test_non_mutable_default_check(self): + class Model(models.Model): + list_field = PickledObjectField(default=list) + dict_field = PickledObjectField(default=dict) + set_field = PickledObjectField(default=set) + + self.assertEqual(Model().check(), []) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..72304b2 --- /dev/null +++ b/tox.ini @@ -0,0 +1,50 @@ +[tox] +skipsdist = true +args_are_paths = false +envlist = + flake8, + isort, + py27-1.11, + py35-{1.11,2.2}, + py36-{1.11,2.2,master}, + py37-{1.11,2.2,master}, + py38-{2.2,master}, + +[tox:travis] +2.7 = py27 +3.5 = py35 +3.6 = py36 +3.7 = py37 +3.8 = py38 + +[testenv] +basepython = + py27: python2.7 + py35: python3.5 + py36: python3.6 + py37: python3.7 + py38: python3.8 +usedevelop = true +commands = + {envpython} -R -Wonce {envbindir}/coverage run --branch -m django test -v2 --settings=tests.settings {posargs} + coverage report -m +deps = + py27: mock + coverage + 1.11: Django>=1.11,<2.0 + 2.0: Django>=2.0,<2.1 + 2.1: Django>=2.1,<2.2 + 2.2: Django>=2.2,<3.0 + master: https://github.com/django/django/archive/master.tar.gz + +[testenv:flake8] +usedevelop = false +basepython = python2.7 +commands = flake8 +deps = flake8 + +[testenv:isort] +usedevelop = false +basepython = python2.7 +commands = isort --recursive --check-only --diff picklefield tests +deps = isort==4.2.5