Merge branch 'review-individuals' of 'https://github.com/evamillan/grimoirelab-sortinghat'

Merges #923
Closes #923

Author: sduenas

releases/unreleased/mark-individuals-as-reviewed.yml

@@ -0,0 +1,17 @@
+---
+title: Mark individuals as reviewed
+category: added
+author: Eva Millán <evamillan@bitergia.com>
+issue: null
+notes: >
+  Individuals can now be marked as reviewed to keep track
+  of which profiles have already been checked and when.
+  A profile can be marked as reviewed more than once, it
+  will show the date of the last review. If there have
+  been any changes to the profile data since the last review,
+  the review button displays a warning icon.
+
+  The list of individuals can be filtered by whether they
+  have been reviewed and by their review date. The list can
+  also be ordered by review date. 
+

sortinghat/cli/client/schema.py

@@ -82,7 +82,7 @@ class GroupFilterType(sgqlc.types.Input):
 
 class IdentityFilterType(sgqlc.types.Input):
     __schema__ = sh_schema
-    __field_names__ = ('uuid', 'term', 'is_locked', 'is_bot', 'gender', 'country', 'source', 'enrollment', 'enrollment_parent_org', 'enrollment_date', 'is_enrolled', 'last_updated')
+    __field_names__ = ('uuid', 'term', 'is_locked', 'is_bot', 'gender', 'country', 'source', 'enrollment', 'enrollment_parent_org', 'enrollment_date', 'is_enrolled', 'last_updated', 'last_reviewed', 'is_reviewed')
     uuid = sgqlc.types.Field(String, graphql_name='uuid')
     term = sgqlc.types.Field(String, graphql_name='term')
     is_locked = sgqlc.types.Field(Boolean, graphql_name='isLocked')
@@ -95,6 +95,8 @@ class IdentityFilterType(sgqlc.types.Input):
     enrollment_date = sgqlc.types.Field(String, graphql_name='enrollmentDate')
     is_enrolled = sgqlc.types.Field(Boolean, graphql_name='isEnrolled')
     last_updated = sgqlc.types.Field(String, graphql_name='lastUpdated')
+    last_reviewed = sgqlc.types.Field(String, graphql_name='lastReviewed')
+    is_reviewed = sgqlc.types.Field(Boolean, graphql_name='isReviewed')
 
 
 class OperationFilterType(sgqlc.types.Input):
@@ -350,14 +352,15 @@ class IdentityType(sgqlc.types.Type):
 
 class IndividualType(sgqlc.types.Type):
     __schema__ = sh_schema
-    __field_names__ = ('created_at', 'last_modified', 'mk', 'is_locked', 'identities', 'profile', 'enrollments')
+    __field_names__ = ('created_at', 'last_modified', 'mk', 'is_locked', 'identities', 'profile', 'enrollments', 'last_reviewed')
     created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name='createdAt')
     last_modified = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name='lastModified')
     mk = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name='mk')
     is_locked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name='isLocked')
     identities = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(IdentityType))), graphql_name='identities')
     profile = sgqlc.types.Field('ProfileType', graphql_name='profile')
     enrollments = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(EnrollmentType))), graphql_name='enrollments')
+    last_reviewed = sgqlc.types.Field(String, graphql_name='lastReviewed')
 
 
 class JobPaginatedType(sgqlc.types.Type):
@@ -615,6 +618,13 @@ class Refresh(sgqlc.types.Type):
     token = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name='token')
 
 
+class Review(sgqlc.types.Type):
+    __schema__ = sh_schema
+    __field_names__ = ('uuid', 'individual')
+    uuid = sgqlc.types.Field(String, graphql_name='uuid')
+    individual = sgqlc.types.Field(IndividualType, graphql_name='individual')
+
+
 class SortingHatMutation(sgqlc.types.Type):
     __schema__ = sh_schema
     __field_names__ = ('add_organization', 'delete_organization', 'add_team', 'delete_team', 'add_domain', 'delete_domain', 'add_identity', 'delete_identity', 'update_profile', 'move_identity', 'lock', 'unlock', 'merge', 'unmerge_identities', 'enroll', 'withdraw', 'update_enrollment', 'recommend_affiliations', 'recommend_matches', 'recommend_gender', 'affiliate', 'unify', 'genderize', 'add_recommender_exclusion_term', 'delete_recommender_exclusion_term', 'token_auth', 'verify_token', 'refresh_token')
@@ -832,6 +842,13 @@ class SortingHatMutation(sgqlc.types.Type):
             ('term', sgqlc.types.Arg(String, graphql_name='term', default=None)),
         ))
     )
+    review = sgqlc.types.Field(
+        Review,
+        graphql_name='review',
+        args=sgqlc.types.ArgDict((
+            ('uuid', sgqlc.types.Arg(String, graphql_name='uuid', default=None)),
+        ))
+    )
     token_auth = sgqlc.types.Field(
         ObtainJSONWebToken,
         graphql_name='tokenAuth',

sortinghat/core/api.py

@@ -22,7 +22,7 @@
 
 import logging
 
-from grimoirelab_toolkit.datetime import datetime_to_utc
+from grimoirelab_toolkit.datetime import datetime_to_utc, datetime_utcnow
 
 from .db import (find_individual_by_uuid,
                  find_identity,
@@ -57,7 +57,8 @@
                  delete_enrollment,
                  move_domain,
                  move_team,
-                 move_alias)
+                 move_alias,
+                 review as review_db)
 from .errors import (InvalidValueError,
                      AlreadyExistsError,
                      NotFoundError,
@@ -1639,3 +1640,41 @@ def delete_merge_recommendations(ctx):
     logger.info("Merge recommendations deleted")
 
     return recommendations
+
+
+@atomic_using_tenant
+def review(ctx, uuid):
+    """Mark an individual as reviewed.
+
+    This function sets the individual identified by `uuid` as last
+    reviewed on the current date.
+
+    :param ctx: context from where this method is called
+    :param uuid: identifier of the individual which will be reviewed
+
+    :returns: an individual with its last review value updated
+
+    :raises InvalidValueError: raised when `uuid` is `None` or an empty string
+    :raises NotFoundError: when the identity with the
+        given `uuid` does not exist.
+    """
+    if uuid is None:
+        raise InvalidValueError(msg="'uuid' cannot be None")
+    if uuid == '':
+        raise InvalidValueError(msg="'uuid' cannot be an empty string")
+
+    trxl = TransactionsLog.open('review', ctx)
+
+    try:
+        individual = find_individual_by_uuid(uuid)
+    except NotFoundError as exc:
+        raise exc
+
+    review_date = datetime_utcnow()
+    individual = review_db(trxl, individual, review_date)
+
+    trxl.close()
+
+    logger.info(f"Individual {uuid} successfully updated")
+
+    return individual

sortinghat/core/db.py

@@ -1352,3 +1352,35 @@ def delete_merge_recommendations(trxl, recommendations):
     trxl.log_operation(op_type=Operation.OpType.DELETE, entity_type='merge_recommendation',
                        timestamp=datetime_utcnow(), args=op_args,
                        target='merge_recommendations')
+
+
+def review(trxl, individual, review_date):
+    """Mark a given individual as reviewed.
+
+    Sets the given `review_date` as the given `individual` last reviewed
+    date.
+
+    :param trxl: TransactionsLog object from the method calling this one
+    :param individual: individual to review
+    :param review_date: date of the last review
+
+    :returns: the individual with last_reviewed parameter updated
+    """
+    if individual.is_locked:
+        raise LockedIdentityError(uuid=individual.mk)
+
+    op_args = {
+        'mk': individual.mk,
+        'last_reviewed': copy.deepcopy(str(review_date))
+    }
+    try:
+        individual.last_reviewed = review_date
+        individual.save()
+    except django.db.utils.IntegrityError as exc:
+        _handle_integrity_error(Individual, exc)
+
+    trxl.log_operation(op_type=Operation.OpType.UPDATE, entity_type='individual',
+                       timestamp=datetime_utcnow(), args=op_args,
+                       target=op_args['mk'])
+
+    return individual

sortinghat/core/migrations/0010_alter_individual_options_individual_last_reviewed.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.13 on 2024-08-29 15:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0009_tenant_perm_group'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='individual',
+            options={'ordering': ('last_modified', 'created_at', 'profile__name', 'last_reviewed')},
+        ),
+        migrations.AddField(
+            model_name='individual',
+            name='last_reviewed',
+            field=models.DateTimeField(default=None, null=True),
+        ),
+    ]

sortinghat/core/models.py

@@ -243,10 +243,11 @@ def __str__(self):
 class Individual(EntityBase):
     mk = CharField(max_length=MAX_SIZE_CHAR_FIELD, primary_key=True)
     is_locked = BooleanField(default=False)
+    last_reviewed = DateTimeField(null=True, default=None)
 
     class Meta:
         db_table = 'individuals'
-        ordering = ('last_modified', 'created_at', 'profile__name',)
+        ordering = ('last_modified', 'created_at', 'profile__name', 'last_reviewed')
 
     def __str__(self):
         return self.mk

sortinghat/core/schema.py

@@ -64,7 +64,8 @@
                   update_enrollment,
                   merge_organizations,
                   delete_scheduled_task,
-                  update_scheduled_task)
+                  update_scheduled_task,
+                  review)
 from .context import SortingHatContext
 from .decorators import (check_auth, check_permissions)
 from .errors import InvalidFilterError, EqualIndividualError, InvalidValueError
@@ -490,6 +491,16 @@ class IdentityFilterType(graphene.InputObjectType):
                      two dates, following ISO-8601 format. Examples:\n* `>=2020-10-12T09:35:06.13045+01:00` \
                      \n * `2020-10-12T00:00:00..2020-11-22T00:00:00`.'
     )
+    is_reviewed = graphene.Boolean(
+        required=False,
+        description='Filters individuals by whether they have been marked as reviewed.'
+    )
+    last_reviewed = graphene.String(
+        required=False,
+        description='Filter with a comparison operator (>, >=, <, <=) and a date OR with a range operator (..) between\
+                     two dates, following ISO-8601 format. Examples:\n* `>=2020-10-12T09:35:06.13045+01:00` \
+                     \n * `2020-10-12T00:00:00..2020-11-22T00:00:00`.'
+    )
 
 
 class TransactionFilterType(graphene.InputObjectType):
@@ -1572,6 +1583,27 @@ def mutate(self, info, task_id, data):
         )
 
 
+class Review(graphene.Mutation):
+    class Arguments:
+        uuid = graphene.String()
+
+    uuid = graphene.Field(lambda: graphene.String)
+    individual = graphene.Field(lambda: IndividualType)
+
+    @check_permissions(['core.change_profile'])
+    def mutate(self, info, uuid):
+        user = info.context.user
+        tenant = get_db_tenant()
+        ctx = SortingHatContext(user=user, tenant=tenant)
+
+        individual = review(ctx, uuid)
+
+        return Review(
+            uuid=uuid,
+            individual=individual
+        )
+
+
 class SortingHatQuery:
 
     countries = graphene.Field(
@@ -1870,6 +1902,35 @@ def resolve_individuals(self, info, filters=None,
                 elif operator == '..':
                     query = query.filter(last_modified__range=(date1, date2))
 
+        if filters and 'is_reviewed' in filters:
+            query = query.filter(mk__in=Subquery(Individual.objects
+                                                 .filter(last_reviewed__isnull=not filters['is_reviewed'])
+                                                 .values_list('mk')))
+
+        if filters and 'last_reviewed' in filters:
+            # Accepted date format is ISO 8601, YYYY-MM-DDTHH:MM:SS
+            try:
+                filter_data = parse_date_filter(filters['last_reviewed'])
+            except ValueError as e:
+                raise InvalidFilterError(filter_name='last_reviewed', msg=e)
+            except InvalidDateError as e:
+                raise InvalidFilterError(filter_name='last_reviewed', msg=e)
+
+            date1 = filter_data['date1']
+            date2 = filter_data['date2']
+            if filter_data['operator']:
+                operator = filter_data['operator']
+                if operator == '<':
+                    query = query.filter(last_reviewed__lt=date1)
+                elif operator == '<=':
+                    query = query.filter(last_reviewed__lte=date1)
+                elif operator == '>':
+                    query = query.filter(last_reviewed__gt=date1)
+                elif operator == '>=':
+                    query = query.filter(last_reviewed__gte=date1)
+                elif operator == '..':
+                    query = query.filter(last_reviewed__range=(date1, date2))
+
         return IdentityPaginatedType.create_paginated_result(query,
                                                              page,
                                                              page_size=page_size)
@@ -2217,6 +2278,9 @@ class SortingHatMutation(graphene.ObjectType):
     delete_merge_recommendations = DeleteMergeRecommendations.Field(
         description='Remove all unapplied merge recommendations.'
     )
+    review = Review.Field(
+        description='Mark an individual as reviewed on the current date.'
+    )
 
     # JWT authentication
     token_auth = graphql_jwt.ObtainJSONWebToken.Field()