From 801cd6d9c70de3535a34863452bcc713b024103d Mon Sep 17 00:00:00 2001 From: pauliyobo Date: Mon, 30 Sep 2024 21:04:29 +0200 Subject: [PATCH] Refactor and handling database migrations --- alembic.ini | 2 +- alembic/env.py | 7 +- .../85028f013e6d_zero_migration_revision.py | 30 +++++ bookworm/annotation/__init__.py | 2 +- bookworm/annotation/annotation_models.py | 121 ------------------ bookworm/annotation/annotator.py | 4 +- bookworm/database/__init__.py | 36 ++++-- bookworm/database/models.py | 111 +++++++++++++++- 8 files changed, 171 insertions(+), 142 deletions(-) create mode 100644 alembic/versions/85028f013e6d_zero_migration_revision.py delete mode 100644 bookworm/annotation/annotation_models.py diff --git a/alembic.ini b/alembic.ini index 72cc6990..2a4cc32b 100644 --- a/alembic.ini +++ b/alembic.ini @@ -60,7 +60,7 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = sqlite:///.appdata/database/database2.sqlite [post_write_hooks] diff --git a/alembic/env.py b/alembic/env.py index fcf0e08a..5d5b6a0a 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -8,8 +8,7 @@ from alembic.autogenerate import rewriter from alembic.operations import ops -from bookworm.database import Base, get_db_url -from bookworm.annotation.annotation_models import * +from bookworm.database import * @@ -54,7 +53,7 @@ def run_migrations_offline() -> None: script output. """ - url = get_db_url() + url = config.get_main_option("sqlalchemy.url") context.configure( url=url, target_metadata=target_metadata, @@ -73,7 +72,7 @@ def run_migrations_online() -> None: and associate a connection with the context. """ - connectable = create_engine(get_db_url()) + connectable = create_engine(config.get_main_option("sqlalchemy.url")) with connectable.connect() as connection: context.configure( diff --git a/alembic/versions/85028f013e6d_zero_migration_revision.py b/alembic/versions/85028f013e6d_zero_migration_revision.py new file mode 100644 index 00000000..be60ebd2 --- /dev/null +++ b/alembic/versions/85028f013e6d_zero_migration_revision.py @@ -0,0 +1,30 @@ +"""Zero Migration revision + +Revision ID: 85028f013e6d +Revises: 28099038d8d6 +Create Date: 2024-09-30 14:48:49.349114 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import bookworm + +# revision identifiers, used by Alembic. +revision: str = '85028f013e6d' +down_revision: Union[str, None] = '28099038d8d6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/bookworm/annotation/__init__.py b/bookworm/annotation/__init__.py index f03d7524..9c953f59 100644 --- a/bookworm/annotation/__init__.py +++ b/bookworm/annotation/__init__.py @@ -3,6 +3,7 @@ import wx from bookworm import config, speech +from bookworm.database.models import Note, Quote from bookworm.logger import logger from bookworm.resources import sounds from bookworm.service import BookwormService @@ -20,7 +21,6 @@ AnnotationSettingsPanel, AnnotationsMenuIds, ) -from .annotation_models import Note, Quote from .annotator import Bookmarker, NoteTaker, Quoter log = logger.getChild(__name__) diff --git a/bookworm/annotation/annotation_models.py b/bookworm/annotation/annotation_models.py deleted file mode 100644 index f41f574d..00000000 --- a/bookworm/annotation/annotation_models.py +++ /dev/null @@ -1,121 +0,0 @@ -# coding: utf-8 - -""" -Database models for Annotations. -""" - -from datetime import datetime - -import sqlalchemy as sa -from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import deferred, relationship, synonym - -from bookworm.database import GetOrCreateMixin, Base - - -class TaggedMixin: - """Provides a generic many-to-many relationship - to a dynamically generated tags table using - the `table-per-related` pattern. - - .. admonition::: the dynamically generated table is shared by this model - class and all it's subclasses. - - Adapted from oy-cms. - Copyright (c) 2018 Musharraf Omer - """ - - @staticmethod - def _prepare_association_table(table_name, remote1, remote2): - return sa.Table( - table_name, - Base.metadata, - sa.Column(f"{remote1}_id", sa.Integer, sa.ForeignKey(f"{remote1}.id")), - sa.Column(f"{remote2}_id", sa.Integer, sa.ForeignKey(f"{remote2}.id")), - ) - - @declared_attr - def tags(cls): - if not hasattr(cls, "Tag"): - # Create the Tag model - tag_attrs = { - "id": sa.Column(sa.Integer, primary_key=True), - "title": sa.Column(sa.String(512), nullable=False, unique=True, index=True), - "items": relationship( - cls, - secondary=lambda: cls.__tags_association_table__, - backref="related_tags", - ), - } - cls.Tag = type( - f"{cls.__name__}Tag", (GetOrCreateMixin, Base), tag_attrs - ) - # The many-to-many association table - cls.__tags_association_table__ = cls._prepare_association_table( - table_name=f"{cls.__tablename__}s_tags", - remote1=cls.__tablename__, - remote2=cls.Tag.__tablename__, - ) - return association_proxy( - "related_tags", - "title", - creator=lambda t: cls.Tag.get_or_create(title=t.lower()), - ) - - -class AnnotationBase(Base): - __abstract__ = True - __tablename__ = "annotation_base" - id = sa.Column(sa.Integer, primary_key=True) - title = sa.Column(sa.String(255), nullable=False) - page_number = sa.Column(sa.Integer, nullable=False) - position = sa.Column(sa.Integer, nullable=False, default=0) - section_title = sa.Column(sa.String(1024), nullable=False) - section_identifier = sa.Column(sa.String(1024), nullable=False) - date_created = sa.Column(sa.DateTime, default=datetime.utcnow) - date_updated = sa.Column(sa.DateTime, onupdate=datetime.utcnow) - - @declared_attr - def text_column(cls): - return synonym("title") - - @declared_attr - def book_id(cls): - return sa.Column(sa.Integer, sa.ForeignKey("book.id"), nullable=False) - - @declared_attr - def book(cls): - reverse_name = f"{cls.__name__.lower()}s" - return relationship("Book", foreign_keys=[cls.book_id], backref=reverse_name) - - -class Bookmark(AnnotationBase): - """Represents a user-defined bookmark.""" - __tablename__ = "bookmark" - - -class TaggedContent(AnnotationBase, TaggedMixin): - __abstract__ = True - __tablename__ = "tagged_content" - - @declared_attr - def content(cls): - return deferred(sa.Column(sa.Text, nullable=False)) - - @declared_attr - def text_column(cls): - return synonym("content") - - -class Note(TaggedContent): - """Represents user comments (notes).""" - __tablename__ = "note" - - -class Quote(TaggedContent): - """Represents a highlight (quote) from the book.""" - __tablename__ = "quote" - - start_pos = sa.Column(sa.Integer, nullable=False) - end_pos = sa.Column(sa.Integer, nullable=False) diff --git a/bookworm/annotation/annotator.py b/bookworm/annotation/annotator.py index 988625d2..f468a950 100644 --- a/bookworm/annotation/annotator.py +++ b/bookworm/annotation/annotator.py @@ -6,11 +6,9 @@ import sqlalchemy as sa from bookworm import config -from bookworm.database.models import Book +from bookworm.database.models import Book, Bookmark, Note, Quote from bookworm.logger import logger -from .annotation_models import Bookmark, Note, Quote - log = logger.getChild(__name__) # The bakery caches query objects to avoid recompiling them into strings in every call diff --git a/bookworm/database/__init__.py b/bookworm/database/__init__.py index f6a4ca29..d6913fb8 100644 --- a/bookworm/database/__init__.py +++ b/bookworm/database/__init__.py @@ -10,7 +10,8 @@ from alembic import command from alembic.config import Config -from sqlalchemy import create_engine +from alembic.migration import MigrationContext +from sqlalchemy import create_engine, text from sqlalchemy.orm import scoped_session, sessionmaker from bookworm import app @@ -18,14 +19,7 @@ from bookworm import paths from bookworm.paths import db_path as get_db_path -from .models import ( - Book, - Base, - DocumentPositionInfo, - GetOrCreateMixin, - PinnedDocument, - RecentDocument, -) +from .models import * log = logger.getChild(__name__) @@ -33,8 +27,21 @@ def get_db_url() -> str: db_path = os.path.join(get_db_path(), "database.sqlite") return f"sqlite:///{db_path}" -def init_database(): - engine = create_engine(get_db_url()) +def init_database(engine = None, url: str = None) -> bool: + if not url: + url = get_db_url() + if not engine: + engine = create_engine(get_db_url()) + log.debug(f"Using url {url} ") + with engine.connect() as conn: + context = MigrationContext.configure(conn) + rev = context.get_current_revision() + # let's check for the book table + # Should it be too ambiguous, we'd have to revisit what tables should be checked to determine whether the DB is at the baseline point + cursor = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table';")) + tables = [row[0] for row in cursor.fetchall()] + is_baseline = tables != None and "book" in tables and rev == None + log.info(f"Current revision is {rev}") log.info("Running database migrations and setup") cfg_file = "" script_location = "alembic" @@ -44,8 +51,15 @@ def init_database(): cfg = Config(Path(cfg_file, "alembic.ini")) cfg.set_main_option('script_location', str(script_location)) + cfg.set_main_option("sqlalchemy.url", url) + if rev == None: + if is_baseline: + log.info("No revision was found, but the database appears to be at the baseline required to begin tracking.") + log.info("Stamping alembic revision") + command.stamp(cfg, "28099038d8d6") command.upgrade(cfg, "head") Base.session = scoped_session( sessionmaker(engine, autocommit=False, autoflush=False) ) + return True diff --git a/bookworm/database/models.py b/bookworm/database/models.py index d0a7ab6f..e7fb8501 100644 --- a/bookworm/database/models.py +++ b/bookworm/database/models.py @@ -9,9 +9,10 @@ import sqlalchemy as sa from sqlalchemy import types +from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import class_mapper, mapper, Query, scoped_session, declarative_base +from sqlalchemy.orm import deferred, relationship, synonym, class_mapper, mapper, Query, scoped_session, declarative_base from bookworm.document.uri import DocumentUri from bookworm.logger import logger @@ -163,3 +164,111 @@ def clear_all(cls): for item in cls.query.all(): session.delete(item) session.commit() + +# annotation models + +class TaggedMixin: + """Provides a generic many-to-many relationship + to a dynamically generated tags table using + the `table-per-related` pattern. + + .. admonition::: the dynamically generated table is shared by this model + class and all it's subclasses. + + Adapted from oy-cms. + Copyright (c) 2018 Musharraf Omer + """ + + @staticmethod + def _prepare_association_table(table_name, remote1, remote2): + return sa.Table( + table_name, + Base.metadata, + sa.Column(f"{remote1}_id", sa.Integer, sa.ForeignKey(f"{remote1}.id")), + sa.Column(f"{remote2}_id", sa.Integer, sa.ForeignKey(f"{remote2}.id")), + ) + + @declared_attr + def tags(cls): + if not hasattr(cls, "Tag"): + # Create the Tag model + tag_attrs = { + "id": sa.Column(sa.Integer, primary_key=True), + "title": sa.Column(sa.String(512), nullable=False, unique=True, index=True), + "items": relationship( + cls, + secondary=lambda: cls.__tags_association_table__, + backref="related_tags", + ), + } + cls.Tag = type( + f"{cls.__name__}Tag", (GetOrCreateMixin, Base), tag_attrs + ) + # The many-to-many association table + cls.__tags_association_table__ = cls._prepare_association_table( + table_name=f"{cls.__tablename__}s_tags", + remote1=cls.__tablename__, + remote2=cls.Tag.__tablename__, + ) + return association_proxy( + "related_tags", + "title", + creator=lambda t: cls.Tag.get_or_create(title=t.lower()), + ) + + +class AnnotationBase(Base): + __abstract__ = True + __tablename__ = "annotation_base" + id = sa.Column(sa.Integer, primary_key=True) + title = sa.Column(sa.String(255), nullable=False) + page_number = sa.Column(sa.Integer, nullable=False) + position = sa.Column(sa.Integer, nullable=False, default=0) + section_title = sa.Column(sa.String(1024), nullable=False) + section_identifier = sa.Column(sa.String(1024), nullable=False) + date_created = sa.Column(sa.DateTime, default=datetime.utcnow) + date_updated = sa.Column(sa.DateTime, onupdate=datetime.utcnow) + + @declared_attr + def text_column(cls): + return synonym("title") + + @declared_attr + def book_id(cls): + return sa.Column(sa.Integer, sa.ForeignKey("book.id"), nullable=False) + + @declared_attr + def book(cls): + reverse_name = f"{cls.__name__.lower()}s" + return relationship("Book", foreign_keys=[cls.book_id], backref=reverse_name) + + +class Bookmark(AnnotationBase): + """Represents a user-defined bookmark.""" + __tablename__ = "bookmark" + + +class TaggedContent(AnnotationBase, TaggedMixin): + __abstract__ = True + __tablename__ = "tagged_content" + + @declared_attr + def content(cls): + return deferred(sa.Column(sa.Text, nullable=False)) + + @declared_attr + def text_column(cls): + return synonym("content") + + +class Note(TaggedContent): + """Represents user comments (notes).""" + __tablename__ = "note" + + +class Quote(TaggedContent): + """Represents a highlight (quote) from the book.""" + __tablename__ = "quote" + + start_pos = sa.Column(sa.Integer, nullable=False) + end_pos = sa.Column(sa.Integer, nullable=False)