From 9f8e8585fd2329218d953ebcd0a403392fc1bd66 Mon Sep 17 00:00:00 2001 From: David Straub Date: Tue, 30 Jan 2024 18:50:07 +0100 Subject: [PATCH] Implement check & repair endpoint (fixes #479) --- gramps_webapi/api/__init__.py | 4 +- gramps_webapi/api/check.py | 66 +++++++++++++++++++ gramps_webapi/api/resources/trees.py | 27 +++++++- gramps_webapi/api/tasks.py | 8 +++ gramps_webapi/auth/const.py | 2 + tests/test_endpoints/test_repair.py | 95 ++++++++++++++++++++++++++++ 6 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 gramps_webapi/api/check.py create mode 100644 tests/test_endpoints/test_repair.py diff --git a/gramps_webapi/api/__init__.py b/gramps_webapi/api/__init__.py index 115eb100..a09a2a85 100644 --- a/gramps_webapi/api/__init__.py +++ b/gramps_webapi/api/__init__.py @@ -30,9 +30,9 @@ from .media import get_media_handler from .resources.base import Resource from .resources.bookmarks import ( + BookmarkEditResource, BookmarkResource, BookmarksResource, - BookmarkEditResource, ) from .resources.citations import CitationResource, CitationsResource from .resources.config import ConfigResource, ConfigsResource @@ -93,6 +93,7 @@ from .resources.transactions import TransactionsResource from .resources.translations import TranslationResource, TranslationsResource from .resources.trees import ( + CheckTreeResource, DisableTreeResource, EnableTreeResource, TreeResource, @@ -186,6 +187,7 @@ def register_endpt(resource: Type[Resource], url: str, name: str): register_endpt(TreesResource, "/trees/", "trees") register_endpt(DisableTreeResource, "/trees//disable", "disable_tree") register_endpt(EnableTreeResource, "/trees//enable", "enable_tree") +register_endpt(CheckTreeResource, "/trees//repair", "repair_tree") # Types register_endpt(CustomTypeResource, "/types/custom/", "custom-type") register_endpt(CustomTypesResource, "/types/custom/", "custom-types") diff --git a/gramps_webapi/api/check.py b/gramps_webapi/api/check.py new file mode 100644 index 00000000..4bcfa4e9 --- /dev/null +++ b/gramps_webapi/api/check.py @@ -0,0 +1,66 @@ +"""Check and repair a Gramps database.""" + +from gramps.gen.db import DbTxn, DbWriteBase +from gramps.gen.dbstate import DbState +from gramps.plugins.tool.check import CheckIntegrity + + +def check_database(db_handle: DbWriteBase): + with DbTxn("Check Integrity", db_handle, batch=True) as trans: + db_handle.disable_signals() + dbstate = DbState() + dbstate.change_database(db_handle) + checker = CheckIntegrity(dbstate, None, trans) + + # start with empty objects, broken links can be corrected below + # then. This is done before fixing encoding and missing photos, + # since otherwise we will be trying to fix empty records which are + # then going to be deleted. + checker.cleanup_empty_objects() + checker.fix_encoding() + checker.fix_alt_place_names() + checker.fix_ctrlchars_in_notes() + # checker.cleanup_missing_photos(cli=1) # should not be done on Web API + checker.cleanup_deleted_name_formats() + + prev_total = -1 + total = 0 + + while prev_total != total: + prev_total = total + + checker.check_for_broken_family_links() + checker.check_parent_relationships() + checker.cleanup_empty_families(1) + checker.cleanup_duplicate_spouses() + + total = checker.family_errors() + + checker.fix_duplicated_grampsid() + checker.check_events() + checker.check_person_references() + checker.check_family_references() + checker.check_place_references() + checker.check_source_references() + checker.check_citation_references() + checker.check_media_references() + checker.check_repo_references() + checker.check_note_references() + checker.check_tag_references() + # checker.check_checksum() # should not be done on Web API + checker.check_media_sourceref() + # checker.check_note_links() # requires Gramps 5.2 + checker.check_backlinks() + + # rebuilding reference maps needs to be done outside of a transaction + # to avoid nesting transactions. + if checker.bad_backlinks: + checker.progress.set_pass("Rebuilding reference maps...", 6) + db_handle.reindex_reference_map(checker.callback) + + db_handle.enable_signals() + db_handle.request_rebuild() + + errs = checker.build_report() + text = checker.text.getvalue() + return {"num_errors": errs, "message": text} diff --git a/gramps_webapi/api/resources/trees.py b/gramps_webapi/api/resources/trees.py index 5070699f..4922f8c3 100644 --- a/gramps_webapi/api/resources/trees.py +++ b/gramps_webapi/api/resources/trees.py @@ -24,7 +24,7 @@ import uuid from typing import Dict, List, Optional -from flask import abort, current_app +from flask import abort, current_app, jsonify from gramps.gen.config import config from webargs import fields from werkzeug.security import safe_join @@ -36,11 +36,13 @@ PERM_EDIT_OTHER_TREE, PERM_EDIT_TREE, PERM_EDIT_TREE_QUOTA, + PERM_REPAIR_TREE, PERM_VIEW_OTHER_TREE, ) from ...const import TREE_MULTI from ...dbmanager import WebDbManager from ..auth import has_permissions, require_permissions +from ..tasks import AsyncResult, check_repair_database, make_task_response, run_task from ..util import abort_with_message, get_tree_from_jwt, list_trees, use_args from . import ProtectedResource @@ -224,3 +226,26 @@ class EnableTreeResource(DisableEnableTreeResource): def post(self, tree_id: str): """Disable a tree.""" return self._post_disable_enable_tree(tree_id=tree_id, disabled=False) + + +class CheckTreeResource(ProtectedResource): + """Resource for checking & repairing a Gramps database.""" + + def post(self, tree_id: str): + """Check & repair a Gramps database (tree).""" + require_permissions([PERM_REPAIR_TREE]) + user_tree_id = get_tree_from_jwt() + if tree_id == "-": + # own tree + tree_id = user_tree_id + else: + validate_tree_id(tree_id) + if tree_id != user_tree_id: + abort_with_message(403, "Not allowed to repair other trees") + task = run_task( + check_repair_database, + tree=tree_id, + ) + if isinstance(task, AsyncResult): + return make_task_response(task) + return jsonify(task), 201 diff --git a/gramps_webapi/api/tasks.py b/gramps_webapi/api/tasks.py index 125e7ab8..cbd23835 100644 --- a/gramps_webapi/api/tasks.py +++ b/gramps_webapi/api/tasks.py @@ -29,6 +29,7 @@ from flask import current_app from ..auth import get_owner_emails +from .check import check_database from .emails import email_confirm_email, email_new_user, email_reset_pw from .export import prepare_options, run_export from .media import get_media_handler @@ -230,3 +231,10 @@ def media_ocr( handle, db_handle=db_handle ) return handler.get_ocr(lang=lang, output_format=output_format) + + +@shared_task() +def check_repair_database(tree: str): + """Check and repair a Gramps database (tree)""" + db_handle = get_db_outside_request(tree=tree, view_private=True, readonly=False) + return check_database(db_handle) diff --git a/gramps_webapi/auth/const.py b/gramps_webapi/auth/const.py index c712b4c4..3d6fe944 100644 --- a/gramps_webapi/auth/const.py +++ b/gramps_webapi/auth/const.py @@ -62,6 +62,7 @@ PERM_TRIGGER_REINDEX = "TriggerReindex" PERM_EDIT_NAME_GROUP = "EditNameGroup" PERM_EDIT_TREE = "EditTree" +PERM_REPAIR_TREE = "RepairTree" PERMISSIONS = {} @@ -97,6 +98,7 @@ PERM_IMPORT_FILE, PERM_TRIGGER_REINDEX, PERM_EDIT_TREE, + PERM_REPAIR_TREE, } PERMISSIONS[ROLE_ADMIN] = PERMISSIONS[ROLE_OWNER] | { diff --git a/tests/test_endpoints/test_repair.py b/tests/test_endpoints/test_repair.py new file mode 100644 index 00000000..933f066e --- /dev/null +++ b/tests/test_endpoints/test_repair.py @@ -0,0 +1,95 @@ +# +# Gramps Web API - A RESTful API for the Gramps genealogy program +# +# Copyright (C) 2024 David Straub +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +"""Tests for the database repair endpoint.""" + +import os +import unittest +from unittest.mock import patch + +from gramps.cli.clidbman import CLIDbManager +from gramps.gen.dbstate import DbState + +from gramps_webapi.app import create_app +from gramps_webapi.auth import add_user, user_db +from gramps_webapi.auth.const import ROLE_GUEST, ROLE_OWNER +from gramps_webapi.const import ENV_CONFIG_FILE, TEST_AUTH_CONFIG + + +class TestRepair(unittest.TestCase): + """Test database repair.""" + + @classmethod + def setUpClass(cls): + cls.name = "Test Web API Repair" + cls.dbman = CLIDbManager(DbState()) + dirpath, _ = cls.dbman.create_new_db_cli(cls.name, dbid="sqlite") + tree = os.path.basename(dirpath) + with patch.dict("os.environ", {ENV_CONFIG_FILE: TEST_AUTH_CONFIG}): + cls.app = create_app() + cls.app.config["TESTING"] = True + cls.client = cls.app.test_client() + with cls.app.app_context(): + user_db.create_all() + add_user(name="user", password="123", role=ROLE_GUEST, tree=tree) + add_user(name="owner", password="123", role=ROLE_OWNER, tree=tree) + rv = cls.client.post( + "/api/token/", json={"username": "owner", "password": "123"} + ) + access_token = rv.json["access_token"] + cls.headers = {"Authorization": f"Bearer {access_token}"} + + @classmethod + def tearDownClass(cls): + cls.dbman.remove_database(cls.name) + + def test_repair_empty_database(self): + """Test Repairing the empty database.""" + rv = self.client.post("/api/trees/-/repair", headers=self.headers) + assert rv.status_code == 201 + assert rv.json["num_errors"] == 0 + assert rv.json["message"] == "" + + def test_repair_empty_person(self): + """Test Repairing an empty person.""" + rv = self.client.post("/api/people/", json={}, headers=self.headers) + assert rv.status_code == 201 + rv = self.client.get("/api/people/", headers=self.headers) + assert rv.status_code == 200 + assert len(rv.json) == 1 + rv = self.client.post("/api/trees/-/repair", headers=self.headers) + assert rv.status_code == 201 + assert rv.json["num_errors"] == 1 + rv = self.client.get("/api/people/", headers=self.headers) + assert rv.status_code == 200 + assert len(rv.json) == 0 + + def test_repair_empty_event(self): + """Test Repairing an empty event.""" + rv = self.client.post("/api/events/", json={}, headers=self.headers) + assert rv.status_code == 201 + rv = self.client.get("/api/events/", headers=self.headers) + assert rv.status_code == 200 + assert len(rv.json) == 1 + rv = self.client.post("/api/trees/-/repair", headers=self.headers) + assert rv.status_code == 201 + assert rv.json["num_errors"] == 1 + rv = self.client.get("/api/events/", headers=self.headers) + assert rv.status_code == 200 + assert len(rv.json) == 0