diff --git a/AUTHORS b/AUTHORS index 4eac5eb2d..2e7b56fcb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -244,4 +244,5 @@ that much better: * Stanislav Kaledin (https://github.com/sallyruthstruik) * Dmitry Yantsen (https://github.com/mrTable) * Renjianxin (https://github.com/Davidrjx) - * Erdenezul Batmunkh (https://github.com/erdenezul) \ No newline at end of file + * Erdenezul Batmunkh (https://github.com/erdenezul) + * Andy Yankovsky (https://github.com/werat) diff --git a/docs/apireference.rst b/docs/apireference.rst index 625d4a8bd..05ba3f733 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -87,7 +87,9 @@ Fields .. autoclass:: mongoengine.fields.DictField .. autoclass:: mongoengine.fields.MapField .. autoclass:: mongoengine.fields.ReferenceField +.. autoclass:: mongoengine.fields.LazyReferenceField .. autoclass:: mongoengine.fields.GenericReferenceField +.. autoclass:: mongoengine.fields.GenericLazyReferenceField .. autoclass:: mongoengine.fields.CachedReferenceField .. autoclass:: mongoengine.fields.BinaryField .. autoclass:: mongoengine.fields.FileField diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d41ae7e63..3ced284e2 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -22,7 +22,7 @@ objects** as class attributes to the document class:: class Page(Document): title = StringField(max_length=200, required=True) - date_modified = DateTimeField(default=datetime.datetime.now) + date_modified = DateTimeField(default=datetime.datetime.utcnow) As BSON (the binary format for storing data in mongodb) is order dependent, documents are serialized based on their field order. @@ -80,6 +80,7 @@ are as follows: * :class:`~mongoengine.fields.FloatField` * :class:`~mongoengine.fields.GenericEmbeddedDocumentField` * :class:`~mongoengine.fields.GenericReferenceField` +* :class:`~mongoengine.fields.GenericLazyReferenceField` * :class:`~mongoengine.fields.GeoPointField` * :class:`~mongoengine.fields.ImageField` * :class:`~mongoengine.fields.IntField` @@ -87,6 +88,7 @@ are as follows: * :class:`~mongoengine.fields.MapField` * :class:`~mongoengine.fields.ObjectIdField` * :class:`~mongoengine.fields.ReferenceField` +* :class:`~mongoengine.fields.LazyReferenceField` * :class:`~mongoengine.fields.SequenceField` * :class:`~mongoengine.fields.SortedListField` * :class:`~mongoengine.fields.StringField` @@ -224,7 +226,7 @@ store; in this situation a :class:`~mongoengine.fields.DictField` is appropriate user = ReferenceField(User) answers = DictField() - survey_response = SurveyResponse(date=datetime.now(), user=request.user) + survey_response = SurveyResponse(date=datetime.utcnow(), user=request.user) response_form = ResponseForm(request.POST) survey_response.answers = response_form.cleaned_data() survey_response.save() @@ -618,7 +620,7 @@ collection after a given period. See the official documentation for more information. A common usecase might be session data:: class Session(Document): - created = DateTimeField(default=datetime.now) + created = DateTimeField(default=datetime.utcnow) meta = { 'indexes': [ {'fields': ['created'], 'expireAfterSeconds': 3600} diff --git a/docs/tutorial.rst b/docs/tutorial.rst index ea1a04c1d..bcd0d17f4 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -86,7 +86,7 @@ of them stand out as particularly intuitive solutions. Posts ^^^^^ -Happily mongoDB *isn't* a relational database, so we're not going to do it that +Happily MongoDB *isn't* a relational database, so we're not going to do it that way. As it turns out, we can use MongoDB's schemaless nature to provide us with a much nicer solution. We will store all of the posts in *one collection* and each post type will only store the fields it needs. If we later want to add diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 43f328108..fddd945a2 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -351,7 +351,8 @@ def delete(self): def update(self, **update): """ - Updates the embedded documents with the given update values. + Updates the embedded documents with the given replacement values. This + function does not support mongoDB update operators such as ``inc__``. .. note:: The embedded document changes are not automatically saved diff --git a/mongoengine/document.py b/mongoengine/document.py index f1622934f..71929cf1a 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -280,6 +280,9 @@ def modify(self, query=None, **update): elif query[id_field] != self.pk: raise InvalidQueryError('Invalid document modify query: it must modify only this document.') + # Need to add shard key to query, or you get an error + query.update(self._object_key) + updated = self._qs(**query).modify(new=True, **update) if updated is None: return False diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7932f73ac..a661874a5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -614,6 +614,7 @@ class EmbeddedDocumentField(BaseField): """ def __init__(self, document_type, **kwargs): + # XXX ValidationError raised outside of the "validate" method. if not ( isinstance(document_type, six.string_types) or issubclass(document_type, EmbeddedDocument) @@ -919,8 +920,11 @@ def __init__(self, basecls=None, field=None, *args, **kwargs): self.field = field self._auto_dereference = False self.basecls = basecls or BaseField + + # XXX ValidationError raised outside of the "validate" method. if not issubclass(self.basecls, BaseField): self.error('DictField only accepts dict values') + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) @@ -969,6 +973,7 @@ class MapField(DictField): """ def __init__(self, field=None, *args, **kwargs): + # XXX ValidationError raised outside of the "validate" method. if not isinstance(field, BaseField): self.error('Argument to MapField constructor must be a valid ' 'field') @@ -1028,6 +1033,7 @@ def __init__(self, document_type, dbref=False, A reference to an abstract document type is always stored as a :class:`~pymongo.dbref.DBRef`, regardless of the value of `dbref`. """ + # XXX ValidationError raised outside of the "validate" method. if ( not isinstance(document_type, six.string_types) and not issubclass(document_type, Document) @@ -1082,6 +1088,8 @@ def to_mongo(self, document): if isinstance(document, Document): # We need the id from the saved object to create the DBRef id_ = document.pk + + # XXX ValidationError raised outside of the "validate" method. if id_ is None: self.error('You can only reference documents once they have' ' been saved to the database') @@ -1121,7 +1129,6 @@ def prepare_query_value(self, op, value): return self.to_mongo(value) def validate(self, value): - if not isinstance(value, (self.document_type, LazyReference, DBRef, ObjectId)): self.error('A ReferenceField only accepts DBRef, LazyReference, ObjectId or documents') @@ -1129,11 +1136,14 @@ def validate(self, value): self.error('You can only reference documents once they have been ' 'saved to the database') - if self.document_type._meta.get('abstract') and \ - not isinstance(value, self.document_type): + if ( + self.document_type._meta.get('abstract') and + not isinstance(value, self.document_type) + ): self.error( '%s is not an instance of abstract reference type %s' % ( - self.document_type._class_name) + self.document_type._class_name + ) ) def lookup_member(self, member_name): @@ -1156,6 +1166,7 @@ def __init__(self, document_type, fields=None, auto_sync=True, **kwargs): if fields is None: fields = [] + # XXX ValidationError raised outside of the "validate" method. if ( not isinstance(document_type, six.string_types) and not issubclass(document_type, Document) @@ -1230,6 +1241,7 @@ def to_mongo(self, document, use_db_field=True, fields=None): id_field_name = self.document_type._meta['id_field'] id_field = self.document_type._fields[id_field_name] + # XXX ValidationError raised outside of the "validate" method. if isinstance(document, Document): # We need the id from the saved object to create the DBRef id_ = document.pk @@ -1238,7 +1250,6 @@ def to_mongo(self, document, use_db_field=True, fields=None): ' been saved to the database') else: self.error('Only accept a document object') - # TODO: should raise here or will fail next statement value = SON(( ('_id', id_field.to_mongo(id_)), @@ -1256,6 +1267,7 @@ def prepare_query_value(self, op, value): if value is None: return None + # XXX ValidationError raised outside of the "validate" method. if isinstance(value, Document): if value.pk is None: self.error('You can only reference documents once they have' @@ -1269,7 +1281,6 @@ def prepare_query_value(self, op, value): raise NotImplementedError def validate(self, value): - if not isinstance(value, self.document_type): self.error('A CachedReferenceField only accepts documents') @@ -1330,6 +1341,8 @@ def __init__(self, *args, **kwargs): elif isinstance(choice, type) and issubclass(choice, Document): self.choices.append(choice._class_name) else: + # XXX ValidationError raised outside of the "validate" + # method. self.error('Invalid choices provided: must be a list of' 'Document subclasses and/or six.string_typess') @@ -1393,6 +1406,7 @@ def to_mongo(self, document): # We need the id from the saved object to create the DBRef id_ = document.id if id_ is None: + # XXX ValidationError raised outside of the "validate" method. self.error('You can only reference documents once they have' ' been saved to the database') else: @@ -2190,8 +2204,11 @@ class MultiPolygonField(GeoJsonBaseField): class LazyReferenceField(BaseField): """A really lazy reference to a document. - Unlike the :class:`~mongoengine.fields.ReferenceField` it must be manually - dereferenced using it ``fetch()`` method. + Unlike the :class:`~mongoengine.fields.ReferenceField` it will + **not** be automatically (lazily) dereferenced on access. + Instead, access will return a :class:`~mongoengine.base.LazyReference` class + instance, allowing access to `pk` or manual dereference by using + ``fetch()`` method. .. versionadded:: 0.15 """ @@ -2209,6 +2226,7 @@ def __init__(self, document_type, passthrough=False, dbref=False, automatically call `fetch()` and try to retrive the field on the fetched document. Note this only work getting field (not setting or deleting). """ + # XXX ValidationError raised outside of the "validate" method. if ( not isinstance(document_type, six.string_types) and not issubclass(document_type, Document) @@ -2316,10 +2334,12 @@ def lookup_member(self, member_name): class GenericLazyReferenceField(GenericReferenceField): - """A reference to *any* :class:`~mongoengine.document.Document` subclass - that will be automatically dereferenced on access (lazily). - Unlike the :class:`~mongoengine.fields.GenericReferenceField` it must be - manually dereferenced using it ``fetch()`` method. + """A reference to *any* :class:`~mongoengine.document.Document` subclass. + Unlike the :class:`~mongoengine.fields.GenericReferenceField` it will + **not** be automatically (lazily) dereferenced on access. + Instead, access will return a :class:`~mongoengine.base.LazyReference` class + instance, allowing access to `pk` or manual dereference by using + ``fetch()`` method. .. note :: * Any documents used as a generic reference must be registered in the diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 6f9c372c6..e5611226a 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -486,8 +486,9 @@ def update(self, upsert=False, multi=True, write_concern=None, ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. - :param full_result: Return the full result rather than just the number - updated. + :param full_result: Return the full result dictionary rather than just the number + updated, e.g. return + ``{'n': 2, 'nModified': 2, 'ok': 1.0, 'updatedExisting': True}``. :param update: Django-style update keyword arguments .. versionadded:: 0.2 diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index a9874ddf5..7758ddcf9 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -101,21 +101,8 @@ def query(_doc_cls=None, **kwargs): value = value['_id'] elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict): - # Raise an error if the in/nin/all/near param is not iterable. We need a - # special check for BaseDocument, because - although it's iterable - using - # it as such in the context of this method is most definitely a mistake. - BaseDocument = _import_class('BaseDocument') - if isinstance(value, BaseDocument): - raise TypeError("When using the `in`, `nin`, `all`, or " - "`near`-operators you can\'t use a " - "`Document`, you must wrap your object " - "in a list (object -> [object]).") - elif not hasattr(value, '__iter__'): - raise TypeError("The `in`, `nin`, `all`, or " - "`near`-operators must be applied to an " - "iterable (e.g. a list).") - else: - value = [field.prepare_query_value(op, v) for v in value] + # Raise an error if the in/nin/all/near param is not iterable. + value = _prepare_query_for_iterable(field, op, value) # If we're querying a GenericReferenceField, we need to alter the # key depending on the value: @@ -284,9 +271,15 @@ def update(_doc_cls=None, **update): if isinstance(field, GeoJsonBaseField): value = field.to_mongo(value) - if op == 'push' and isinstance(value, (list, tuple, set)): + if op == 'pull': + if field.required or value is not None: + if match == 'in' and not isinstance(value, dict): + value = _prepare_query_for_iterable(field, op, value) + else: + value = field.prepare_query_value(op, value) + elif op == 'push' and isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] - elif op in (None, 'set', 'push', 'pull'): + elif op in (None, 'set', 'push'): if field.required or value is not None: value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): @@ -443,3 +436,22 @@ def _infer_geometry(value): raise InvalidQueryError('Invalid $geometry data. Can be either a ' 'dictionary or (nested) lists of coordinate(s)') + + +def _prepare_query_for_iterable(field, op, value): + # We need a special check for BaseDocument, because - although it's iterable - using + # it as such in the context of this method is most definitely a mistake. + BaseDocument = _import_class('BaseDocument') + + if isinstance(value, BaseDocument): + raise TypeError("When using the `in`, `nin`, `all`, or " + "`near`-operators you can\'t use a " + "`Document`, you must wrap your object " + "in a list (object -> [object]).") + + if not hasattr(value, '__iter__'): + raise TypeError("The `in`, `nin`, `all`, or " + "`near`-operators must be applied to an " + "iterable (e.g. a list).") + + return [field.prepare_query_value(op, v) for v in value] diff --git a/tests/document/instance.py b/tests/document/instance.py index 555cf6ace..38c7fcafb 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1341,6 +1341,23 @@ class Site(Document): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") + def test_update_list_field(self): + """Test update on `ListField` with $pull + $in. + """ + class Doc(Document): + foo = ListField(StringField()) + + Doc.drop_collection() + doc = Doc(foo=['a', 'b', 'c']) + doc.save() + + # Update + doc = Doc.objects.first() + doc.update(pull__foo__in=['a', 'c']) + + doc = Doc.objects.first() + self.assertEqual(doc.foo, ['b']) + def test_embedded_update_db_field(self): """Test update on `EmbeddedDocumentField` fields when db_field is other than default. diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index 20a0c2784..a72497b7b 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -28,12 +28,16 @@ def test_transform_query(self): {'name': {'$exists': True}}) def test_transform_update(self): + class LisDoc(Document): + foo = ListField(StringField()) + class DicDoc(Document): dictField = DictField() class Doc(Document): pass + LisDoc.drop_collection() DicDoc.drop_collection() Doc.drop_collection() @@ -50,6 +54,9 @@ class Doc(Document): update = transform.update(DicDoc, pull__dictField__test=doc) self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict)) + + update = transform.update(LisDoc, pull__foo__in=['a']) + self.assertEqual(update, {'$pull': {'foo': {'$in': ['a']}}}) def test_transform_update_push(self): """Ensure the differences in behvaior between 'push' and 'push_all'"""