Implement check & repair endpoint (fixes gramps-project#479)

gramps_webapi/api/__init__.py

@@ -30,9 +30,9 @@
 from .media import get_media_handler
 from .resources.base import Resource
 from .resources.bookmarks import (
+    BookmarkEditResource,
     BookmarkResource,
     BookmarksResource,
-    BookmarkEditResource,
 )
 from .resources.citations import CitationResource, CitationsResource
 from .resources.config import ConfigResource, ConfigsResource
@@ -93,6 +93,7 @@
 from .resources.transactions import TransactionsResource
 from .resources.translations import TranslationResource, TranslationsResource
 from .resources.trees import (
+    CheckTreeResource,
     DisableTreeResource,
     EnableTreeResource,
     TreeResource,
@@ -186,6 +187,7 @@ def register_endpt(resource: Type[Resource], url: str, name: str):
 register_endpt(TreesResource, "/trees/", "trees")
 register_endpt(DisableTreeResource, "/trees/<string:tree_id>/disable", "disable_tree")
 register_endpt(EnableTreeResource, "/trees/<string:tree_id>/enable", "enable_tree")
+register_endpt(CheckTreeResource, "/trees/<string:tree_id>/repair", "repair_tree")
 # Types
 register_endpt(CustomTypeResource, "/types/custom/<string:datatype>", "custom-type")
 register_endpt(CustomTypesResource, "/types/custom/", "custom-types")

gramps_webapi/api/check.py

@@ -0,0 +1,66 @@
+"""Check and repair a Gramps database."""
+
+from gramps.gen.db import DbTxn, DbWriteBase
+from gramps.gen.dbstate import DbState
+from gramps.plugins.tool.check import CheckIntegrity
+
+
+def check_database(db_handle: DbWriteBase):
+    with DbTxn("Check Integrity", db_handle, batch=True) as trans:
+        db_handle.disable_signals()
+        dbstate = DbState()
+        dbstate.change_database(db_handle)
+        checker = CheckIntegrity(dbstate, None, trans)
+
+        # start with empty objects, broken links can be corrected below
+        # then. This is done before fixing encoding and missing photos,
+        # since otherwise we will be trying to fix empty records which are
+        # then going to be deleted.
+        checker.cleanup_empty_objects()
+        checker.fix_encoding()
+        checker.fix_alt_place_names()
+        checker.fix_ctrlchars_in_notes()
+        # checker.cleanup_missing_photos(cli=1)  # should not be done on Web API
+        checker.cleanup_deleted_name_formats()
+
+        prev_total = -1
+        total = 0
+
+        while prev_total != total:
+            prev_total = total
+
+            checker.check_for_broken_family_links()
+            checker.check_parent_relationships()
+            checker.cleanup_empty_families(1)
+            checker.cleanup_duplicate_spouses()
+
+            total = checker.family_errors()
+
+        checker.fix_duplicated_grampsid()
+        checker.check_events()
+        checker.check_person_references()
+        checker.check_family_references()
+        checker.check_place_references()
+        checker.check_source_references()
+        checker.check_citation_references()
+        checker.check_media_references()
+        checker.check_repo_references()
+        checker.check_note_references()
+        checker.check_tag_references()
+        # checker.check_checksum()  # should not be done on Web API
+        checker.check_media_sourceref()
+        # checker.check_note_links()  # requires Gramps 5.2
+        checker.check_backlinks()
+
+    # rebuilding reference maps needs to be done outside of a transaction
+    # to avoid nesting transactions.
+    if checker.bad_backlinks:
+        checker.progress.set_pass("Rebuilding reference maps...", 6)
+        db_handle.reindex_reference_map(checker.callback)
+
+    db_handle.enable_signals()
+    db_handle.request_rebuild()
+
+    errs = checker.build_report()
+    text = checker.text.getvalue()
+    return {"num_errors": errs, "message": text}

gramps_webapi/api/resources/trees.py

@@ -24,7 +24,7 @@
 import uuid
 from typing import Dict, List, Optional
 
-from flask import abort, current_app
+from flask import abort, current_app, jsonify
 from gramps.gen.config import config
 from webargs import fields
 from werkzeug.security import safe_join
@@ -36,11 +36,13 @@
     PERM_EDIT_OTHER_TREE,
     PERM_EDIT_TREE,
     PERM_EDIT_TREE_QUOTA,
+    PERM_REPAIR_TREE,
     PERM_VIEW_OTHER_TREE,
 )
 from ...const import TREE_MULTI
 from ...dbmanager import WebDbManager
 from ..auth import has_permissions, require_permissions
+from ..tasks import AsyncResult, check_repair_database, make_task_response, run_task
 from ..util import abort_with_message, get_tree_from_jwt, list_trees, use_args
 from . import ProtectedResource
 
@@ -224,3 +226,26 @@ class EnableTreeResource(DisableEnableTreeResource):
     def post(self, tree_id: str):
         """Disable a tree."""
         return self._post_disable_enable_tree(tree_id=tree_id, disabled=False)
+
+
+class CheckTreeResource(ProtectedResource):
+    """Resource for checking & repairing a Gramps database."""
+
+    def post(self, tree_id: str):
+        """Check & repair a Gramps database (tree)."""
+        require_permissions([PERM_REPAIR_TREE])
+        user_tree_id = get_tree_from_jwt()
+        if tree_id == "-":
+            # own tree
+            tree_id = user_tree_id
+        else:
+            validate_tree_id(tree_id)
+            if tree_id != user_tree_id:
+                abort_with_message(403, "Not allowed to repair other trees")
+        task = run_task(
+            check_repair_database,
+            tree=tree_id,
+        )
+        if isinstance(task, AsyncResult):
+            return make_task_response(task)
+        return jsonify(task), 201

gramps_webapi/api/tasks.py

@@ -29,6 +29,7 @@
 from flask import current_app
 
 from ..auth import get_owner_emails
+from .check import check_database
 from .emails import email_confirm_email, email_new_user, email_reset_pw
 from .export import prepare_options, run_export
 from .media import get_media_handler
@@ -230,3 +231,10 @@ def media_ocr(
         handle, db_handle=db_handle
     )
     return handler.get_ocr(lang=lang, output_format=output_format)
+
+
+@shared_task()
+def check_repair_database(tree: str):
+    """Check and repair a Gramps database (tree)"""
+    db_handle = get_db_outside_request(tree=tree, view_private=True, readonly=False)
+    return check_database(db_handle)

gramps_webapi/auth/const.py

@@ -62,6 +62,7 @@
 PERM_TRIGGER_REINDEX = "TriggerReindex"
 PERM_EDIT_NAME_GROUP = "EditNameGroup"
 PERM_EDIT_TREE = "EditTree"
+PERM_REPAIR_TREE = "RepairTree"
 
 PERMISSIONS = {}
 
@@ -97,6 +98,7 @@
     PERM_IMPORT_FILE,
     PERM_TRIGGER_REINDEX,
     PERM_EDIT_TREE,
+    PERM_REPAIR_TREE,
 }
 
 PERMISSIONS[ROLE_ADMIN] = PERMISSIONS[ROLE_OWNER] | {

tests/test_endpoints/test_repair.py

@@ -0,0 +1,95 @@
+#
+# Gramps Web API - A RESTful API for the Gramps genealogy program
+#
+# Copyright (C) 2024      David Straub
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Tests for the database repair endpoint."""
+
+import os
+import unittest
+from unittest.mock import patch
+
+from gramps.cli.clidbman import CLIDbManager
+from gramps.gen.dbstate import DbState
+
+from gramps_webapi.app import create_app
+from gramps_webapi.auth import add_user, user_db
+from gramps_webapi.auth.const import ROLE_GUEST, ROLE_OWNER
+from gramps_webapi.const import ENV_CONFIG_FILE, TEST_AUTH_CONFIG
+
+
+class TestRepair(unittest.TestCase):
+    """Test database repair."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.name = "Test Web API Repair"
+        cls.dbman = CLIDbManager(DbState())
+        dirpath, _ = cls.dbman.create_new_db_cli(cls.name, dbid="sqlite")
+        tree = os.path.basename(dirpath)
+        with patch.dict("os.environ", {ENV_CONFIG_FILE: TEST_AUTH_CONFIG}):
+            cls.app = create_app()
+        cls.app.config["TESTING"] = True
+        cls.client = cls.app.test_client()
+        with cls.app.app_context():
+            user_db.create_all()
+            add_user(name="user", password="123", role=ROLE_GUEST, tree=tree)
+            add_user(name="owner", password="123", role=ROLE_OWNER, tree=tree)
+        rv = cls.client.post(
+            "/api/token/", json={"username": "owner", "password": "123"}
+        )
+        access_token = rv.json["access_token"]
+        cls.headers = {"Authorization": f"Bearer {access_token}"}
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.dbman.remove_database(cls.name)
+
+    def test_repair_empty_database(self):
+        """Test Repairing the empty database."""
+        rv = self.client.post("/api/trees/-/repair", headers=self.headers)
+        assert rv.status_code == 201
+        assert rv.json["num_errors"] == 0
+        assert rv.json["message"] == ""
+
+    def test_repair_empty_person(self):
+        """Test Repairing an empty person."""
+        rv = self.client.post("/api/people/", json={}, headers=self.headers)
+        assert rv.status_code == 201
+        rv = self.client.get("/api/people/", headers=self.headers)
+        assert rv.status_code == 200
+        assert len(rv.json) == 1
+        rv = self.client.post("/api/trees/-/repair", headers=self.headers)
+        assert rv.status_code == 201
+        assert rv.json["num_errors"] == 1
+        rv = self.client.get("/api/people/", headers=self.headers)
+        assert rv.status_code == 200
+        assert len(rv.json) == 0
+
+    def test_repair_empty_event(self):
+        """Test Repairing an empty event."""
+        rv = self.client.post("/api/events/", json={}, headers=self.headers)
+        assert rv.status_code == 201
+        rv = self.client.get("/api/events/", headers=self.headers)
+        assert rv.status_code == 200
+        assert len(rv.json) == 1
+        rv = self.client.post("/api/trees/-/repair", headers=self.headers)
+        assert rv.status_code == 201
+        assert rv.json["num_errors"] == 1
+        rv = self.client.get("/api/events/", headers=self.headers)
+        assert rv.status_code == 200
+        assert len(rv.json) == 0