From b8cd4fa7b81c58dfff3bca8367bbca42670e2c58 Mon Sep 17 00:00:00 2001 From: Johan Forberg Date: Tue, 5 Feb 2019 23:57:15 +0100 Subject: [PATCH 1/7] Add date search functionality. --- backend/backend/api/entries.py | 8 ++++- backend/backend/app.py | 2 +- backend/backend/db.py | 58 ++++++++++++++++++++-------------- backend/test/fixtures.py | 6 ++-- backend/test/test_db.py | 48 ++++++++++++++++++++++++++-- frontend/src/logbook.js | 2 +- frontend/src/search.js | 32 +++++++++++++++++++ 7 files changed, 124 insertions(+), 32 deletions(-) diff --git a/backend/backend/api/entries.py b/backend/backend/api/entries.py index 5398fb3..896564c 100644 --- a/backend/backend/api/entries.py +++ b/backend/backend/api/entries.py @@ -3,7 +3,7 @@ from flask import request, send_file from flask_restful import Resource, marshal, marshal_with, abort from webargs.fields import (Integer, Str, Boolean, Dict, List, - Nested, Email, LocalDateTime) + Nested, Email, LocalDateTime, Date) from webargs.flaskparser import use_args from ..db import Entry, Logbook, EntryLock @@ -138,6 +138,8 @@ def put(self, args, entry_id, logbook_id=None): "attachments": Str(), "attribute": List(Str(validate=lambda s: len(s.split(":")) == 2)), "metadata": List(Str(validate=lambda s: len(s.split(":")) == 2)), + "from_date": Date(), + "to_date": Date(), "archived": Boolean(), "ignore_children": Boolean(), "n": Integer(missing=50), @@ -169,6 +171,8 @@ def get(self, args, logbook_id=None): attachment_filter=args.get("attachments"), attribute_filter=attributes, metadata_filter=metadata, + from_date=args.get("from_date"), + to_date=args.get("to_date"), n=args["n"], offset=args.get("offset"), sort_by_timestamp=args.get("sort_by_timestamp")) entries = logbook.get_entries(**search_args) @@ -182,6 +186,8 @@ def get(self, args, logbook_id=None): attachment_filter=args.get("attachments"), attribute_filter=attributes, metadata_filter=metadata, + from_date=args.get("from_date"), + to_date=args.get("to_date"), n=args["n"], offset=args.get("offset"), sort_by_timestamp=args.get("sort_by_timestamp")) entries = Entry.search(**search_args) diff --git a/backend/backend/app.py b/backend/backend/app.py index 44ed026..386d908 100644 --- a/backend/backend/app.py +++ b/backend/backend/app.py @@ -24,7 +24,7 @@ static_url_path="/static") app.config.from_envvar('ELOGY_CONFIG_FILE') -app.secret_key = app.config["SECRET"] +app.secret_key = app.config.get("SECRET", "not-very-secret") # add some hooks for debugging purposes @app.before_request diff --git a/backend/backend/db.py b/backend/backend/db.py index e2c59d0..05cdcb4 100644 --- a/backend/backend/db.py +++ b/backend/backend/db.py @@ -8,13 +8,13 @@ from playhouse.sqlite_ext import SqliteExtDatabase, JSONField, fn from peewee import (IntegerField, CharField, TextField, BooleanField, DateTimeField, ForeignKeyField, sqlite3) -from peewee import Model, DoesNotExist, DeferredRelation, Entity +from peewee import Model, DoesNotExist, Entity from .utils import CustomJSONEncoder # defer the actual db setup to later, when we have read the config -db = SqliteExtDatabase(None) +db = SqliteExtDatabase(None, regexp_function=True) class CustomJSONField(JSONField): @@ -29,12 +29,12 @@ def setup_database(db_name, close=True): # TODO: support further configuration options, see FlaskDB db_dependencies_installed() db.init(db_name) - Logbook.create_table(fail_silently=True) - LogbookChange.create_table(fail_silently=True) - Entry.create_table(fail_silently=True) - EntryChange.create_table(fail_silently=True) - EntryLock.create_table(fail_silently=True) - Attachment.create_table(fail_silently=True) + Logbook.create_table(safe=True) + LogbookChange.create_table(safe=True) + Entry.create_table(safe=True) + EntryChange.create_table(safe=True) + EntryLock.create_table(safe=True) + Attachment.create_table(safe=True) # print("\n".join(line[0] for line in db.execute_sql("pragma compile_options;"))) if close: db.close() # important @@ -95,7 +95,7 @@ class Meta: description = TextField(null=True) template = TextField(null=True) template_content_type = CharField(default="text/html; charset=UTF-8") - parent = ForeignKeyField("self", null=True, related_name="children") + parent = ForeignKeyField("self", null=True, backref="children") attributes = JSONField(default=[]) metadata = JSONField(default={}) archived = BooleanField(default=False) @@ -275,7 +275,7 @@ class LogbookChange(Model): class Meta: database = db - logbook = ForeignKeyField(Logbook, related_name="changes") + logbook = ForeignKeyField(Logbook, backref="changes") changed = CustomJSONField() @@ -299,7 +299,7 @@ def get_old_value(self, attr): try: change = (LogbookChange.select() .where((LogbookChange.logbook == self.logbook) & - (LogbookChange.changed.extract(attr) != None) & + (LogbookChange.changed[attr] != None) & (LogbookChange.id > self.id)) .order_by(LogbookChange.id) .get()) @@ -320,7 +320,7 @@ def get_new_value(self, attr): try: change = (LogbookChange.select() .where((LogbookChange.logbook == self.logbook) & - (LogbookChange.changed.extract(attr) != None) & + (LogbookChange.changed[attr] != None) & (LogbookChange.id > self.id)) .order_by(LogbookChange.id) .get()) @@ -351,7 +351,7 @@ def __getattr__(self, attr): return getattr(self.change.logbook, attr) -DeferredEntry = DeferredRelation() +# DeferredEntry = DeferredRelation() # class EntrySearch(FTS5Model): @@ -400,7 +400,7 @@ class Entry(Model): class Meta: database = db - logbook = ForeignKeyField(Logbook, related_name="entries") + logbook = ForeignKeyField(Logbook, backref="entries") title = CharField(null=True) authors = JSONField(default=[]) content = TextField(null=True) @@ -416,7 +416,7 @@ class Meta: # logbooks. created_at = UTCDateTimeField(default=datetime.utcnow) last_changed_at = UTCDateTimeField(null=True) - follows = ForeignKeyField("self", null=True, related_name="followups") + follows = ForeignKeyField("self", null=True, backref="followups") archived = BooleanField(default=False) def __str__(self): @@ -571,6 +571,7 @@ def search(cls, logbook=None, followups=False, attribute_filter=None, content_filter=None, title_filter=None, author_filter=None, attachment_filter=None, metadata_filter=None, + from_date=None, to_date=None, sort_by_timestamp=True): # Note: this is all pretty messy. The reason we're building @@ -740,11 +741,20 @@ def search(cls, logbook=None, followups=False, # Check if we're searching, in that case we want to show all entries. if followups or any([title_filter, content_filter, author_filter, metadata_filter, attribute_filter, attachment_filter]): - query += " GROUP BY thread" + query += " GROUP BY thread HAVING 1" else: # We're not searching. In this case we'll only show query += " GROUP BY thread HAVING entry.follows_id IS NULL" + # Since we're using timestamp, which is an aggregate, it must be filtered + # here instead of in the where clause. + if from_date: + query += " AND timestamp >= ?\n" + variables.append(from_date) + if to_date: + query += " AND timestamp <= ?\n" + variables.append(to_date) + # sort newest first, taking into account the last edit if any # TODO: does this make sense? Should we only consider creation date? order_by = sort_by_timestamp and "timestamp" or "entry.created_at" @@ -753,7 +763,7 @@ def search(cls, logbook=None, followups=False, query += " LIMIT {}".format(n) if offset: query += " OFFSET {}".format(offset) - logging.debug("query=%r, variables=%r" % (query, variables)) + logging.warning("query=%r, variables=%r" % (query, variables)) return Entry.raw(query, *variables) @classmethod @@ -804,7 +814,7 @@ def search_(cls, logbook=None, followups=False, # We're using the SQLite JSON1 extension to pick the # attribute value out of the JSON encoded field. # TODO: regexp? - attr = Entry.attributes.extract(name) + attr = Entry.attributes[name] # Note: The reason we're just using 'contains' here # (it's a substring match) is to support "multioption" # attributes. They are represented as a JSON array and @@ -815,7 +825,7 @@ def search_(cls, logbook=None, followups=False, if metadata_filter: for name, value in metadata_filter: - field = Entry.metadata.extract(name) + field = Entry.metadata[name] result = result.where(field.contains(value)) # TODO: how to group the results properly? If searching, we @@ -832,7 +842,7 @@ def search_(cls, logbook=None, followups=False, return result -DeferredEntry.set_model(Entry) +#DeferredEntry.set_model(Entry) class EntryChange(Model): @@ -853,7 +863,7 @@ class EntryChange(Model): class Meta: database = db - entry = ForeignKeyField(Entry, related_name="changes") + entry = ForeignKeyField(Entry, backref="changes") changed = CustomJSONField() @@ -877,7 +887,7 @@ def get_old_value(self, attr): try: change = (EntryChange.select() .where((EntryChange.entry == self.entry) & - (EntryChange.changed.extract(attr) != None) & + (EntryChange.changed[attr] != None) & (EntryChange.id > self.id)) .order_by(EntryChange.id) .get()) @@ -898,7 +908,7 @@ def get_new_value(self, attr): try: change = (EntryChange.select() .where((EntryChange.entry == self.entry) & - (EntryChange.changed.extract(attr) != None) & + (EntryChange.changed[attr] != None) & (EntryChange.id > self.id)) .order_by(EntryChange.id) .get()) @@ -1002,7 +1012,7 @@ class Meta: database = db order_by = ("id",) - entry = ForeignKeyField(Entry, null=True, related_name="attachments") + entry = ForeignKeyField(Entry, null=True, backref="attachments") filename = CharField(null=True) timestamp = UTCDateTimeField(default=datetime.utcnow) path = CharField() # path within the upload folder diff --git a/backend/test/fixtures.py b/backend/test/fixtures.py index 5681c0b..2d3ccd5 100644 --- a/backend/test/fixtures.py +++ b/backend/test/fixtures.py @@ -19,7 +19,7 @@ def elogy(request): os.environ["ELOGY_CONFIG_FILE"] = "../test/config.py" def run_elogy(): - from elogy.app import app + from backend.app import app app.run() proc = Process(target=run_elogy) @@ -33,7 +33,7 @@ def run_elogy(): @pytest.fixture(scope="function") def db(request): - from elogy.db import db, setup_database + from backend.db import db, setup_database setup_database(":memory:", close=False) return db @@ -45,7 +45,7 @@ def db(request): @pytest.fixture(scope="module") def elogy_client(request): os.environ["ELOGY_CONFIG_FILE"] = "../test/config.py" - from elogy.app import app + from backend.app import app with app.test_client() as c: yield c try: diff --git a/backend/test/test_db.py b/backend/test/test_db.py index 06b4c4a..9d3f492 100644 --- a/backend/test/test_db.py +++ b/backend/test/test_db.py @@ -1,8 +1,9 @@ +from datetime import datetime from operator import attrgetter from .fixtures import db -from elogy.db import Entry -from elogy.db import Logbook, LogbookRevision +from backend.db import Entry +from backend.db import Logbook, LogbookRevision # Logbook @@ -309,6 +310,49 @@ def test_entry_title_search(db): assert result.title == "Third entry" +def test_entry_date_search(db): + lb = Logbook.create(name="Logbook1") + + entries = [ + { + "logbook": lb, + "title": "A", + "content": "This content is great!", + "created_at": datetime(2019, 1, 15, 12, 0, 0) + }, + { + "logbook": lb, + "title": "B", + "content": "Some very neat content.", + "created_at": datetime(2019, 1, 17, 12, 0, 0) + }, + { + "logbook": lb, + "title": "C", + "content": "Not so bad content either.", + "created_at": datetime(2019, 1, 19, 12, 0, 0) + } + ] + + # create entries + for entry in entries: + entry = Entry.create(**entry) + entry.save() + + # include the from date + results = list(Entry.search(logbook=lb, from_date="2019-01-17")) + assert {r.title for r in results} == {"B", "C"} + + # exclude the to date + # TODO does this make sense or should we include the date? + results = list(Entry.search(logbook=lb, to_date="2019-01-17")) + assert {r.title for r in results} == {"A"} + + # date interval + results = list(Entry.search(logbook=lb, from_date="2019-01-16", to_date="2019-01-18")) + assert {r.title for r in results} == {"B"} + + def test_entry_search_followups(db): lb = Logbook.create(name="Logbook1") diff --git a/frontend/src/logbook.js b/frontend/src/logbook.js index e6cc261..077b09e 100644 --- a/frontend/src/logbook.js +++ b/frontend/src/logbook.js @@ -194,7 +194,7 @@ class Logbook extends React.Component { ? parseInt(this.props.match.params.entryId, 10) : null, query = parseQuery(this.props.location.search), - filter = ["title", "content", "authors", "attachments"] + filter = ["title", "content", "authors", "attachments", "from_date", "to_date"] .filter(key => query[key]) .map(key => ( diff --git a/frontend/src/search.js b/frontend/src/search.js index f9d57b5..784e422 100644 --- a/frontend/src/search.js +++ b/frontend/src/search.js @@ -11,6 +11,8 @@ class QuickSearch extends React.Component { title: null, authors: null, attachments: null, + from_date: null, + to_date: null, ignore_children: false }; } @@ -54,6 +56,8 @@ class QuickSearch extends React.Component { title: null, authors: null, attachments: null, + from_date: null, + to_date: null, ignore_children: false }, this.onSubmit.bind(this, history, event) @@ -115,6 +119,34 @@ class QuickSearch extends React.Component { placeholder="Attachments" onChange={this.onChange.bind(this)} /> + + + + + + + + + + + +
From: + +
+ To: + + +