feat(api): add events table, track deduplication events (#321)

* feat(api): added events table, logging deduplication events

* feat(api): add events admin

api/.env-sample

@@ -1,4 +1,5 @@
 DEBUG=on
+LOG_SQL_QUERIES=False
 SECRET_KEY=this_should_be_a_super_secret_key
 DATABASE_URL=sqlite:///db.sqlite3
 DATABASE_URL_FOR_DOCKER=postgres://passport_scorer:passport_scorer_pwd@postgres:5432/passport_scorer

api/account/deduplication/fifo.py

@@ -3,7 +3,7 @@
 
 import api_logging as logging
 from account.models import Community
-from registry.models import Stamp
+from registry.models import Event, Stamp
 
 log = logging.getLogger(__name__)
 
@@ -15,19 +15,45 @@ async def afifo(
     deduped_passport = copy.deepcopy(fifo_passport)
     affected_passports = []
     if "stamps" in fifo_passport:
-        for stamp in fifo_passport["stamps"]:
-            stamp_hash = stamp["credential"]["credentialSubject"]["hash"]
+        dedup_event_data = []
+        new_stamp_hashes = [
+            stamp["credential"]["credentialSubject"]["hash"]
+            for stamp in fifo_passport["stamps"]
+        ]
 
-            existing_stamps = (
-                Stamp.objects.filter(hash=stamp_hash, passport__community=community)
-                .exclude(passport__address=address)
-                .select_related("passport")
+        existing_stamps = (
+            Stamp.objects.filter(
+                hash__in=new_stamp_hashes, passport__community=community
             )
+            .exclude(passport__address=address)
+            .select_related("passport")
+        )
 
-            async for existing_stamp in existing_stamps:
-                existing_stamp_passport = existing_stamp.passport
-                await existing_stamp.adelete()
-                await existing_stamp_passport.asave()
-                affected_passports.append(existing_stamp_passport)
+        async for existing_stamp in existing_stamps:
+            existing_stamp_passport = existing_stamp.passport
+            affected_passports.append(existing_stamp_passport)
+            dedup_event_data.append(
+                {
+                    "hash": existing_stamp.hash,
+                    "provider": existing_stamp.provider,
+                    "prev_owner": existing_stamp_passport.address,
+                    "address": address,
+                    "community_id": community.pk,
+                }
+            )
+
+        await existing_stamps.adelete()
+
+        if dedup_event_data:
+            await Event.objects.abulk_create(
+                [
+                    Event(
+                        action=Event.Action.FIFO_DEDUPLICATION,
+                        address=data["prev_owner"],
+                        data=data,
+                    )
+                    for data in dedup_event_data
+                ]
+            )
 
     return (deduped_passport, affected_passports)

api/account/deduplication/lifo.py

@@ -3,7 +3,7 @@
 
 import api_logging as logging
 from account.models import Community
-from registry.models import Stamp
+from registry.models import Event, Stamp
 
 log = logging.getLogger(__name__)
 
@@ -14,18 +14,41 @@ async def alifo(
 ) -> Tuple[dict, list | None]:
     deduped_passport = copy.deepcopy(lifo_passport)
     deduped_passport["stamps"] = []
+
     if "stamps" in lifo_passport:
+        stamp_hashes = [
+            stamp["credential"]["credentialSubject"]["hash"]
+            for stamp in lifo_passport["stamps"]
+        ]
+
+        clashing_stamps = (
+            Stamp.objects.filter(hash__in=stamp_hashes, passport__community=community)
+            .exclude(passport__address=address)
+            .values("hash", "passport__address", "provider")
+        )
+
+        clashing_hashes = {stamp["hash"] async for stamp in clashing_stamps}
+
         for stamp in lifo_passport["stamps"]:
-            stamp_hash = stamp["credential"]["credentialSubject"]["hash"]
-
-            # query db to see if hash already exists, if so remove stamp from passport
-            if (
-                not await Stamp.objects.filter(
-                    hash=stamp_hash, passport__community=community
-                )
-                .exclude(passport__address=address)
-                .aexists()
-            ):
+            if stamp["credential"]["credentialSubject"]["hash"] not in clashing_hashes:
                 deduped_passport["stamps"].append(copy.deepcopy(stamp))
 
+        if clashing_stamps.aexists():
+            await Event.objects.abulk_create(
+                [
+                    Event(
+                        action=Event.Action.LIFO_DEDUPLICATION,
+                        address=address,
+                        data={
+                            "hash": stamp["hash"],
+                            "provider": stamp["provider"],
+                            "owner": stamp["passport__address"],
+                            "address": address,
+                            "community_id": community.pk,
+                        },
+                    )
+                    async for stamp in clashing_stamps
+                ]
+            )
+
     return (deduped_passport, None)

api/registry/admin.py

@@ -1,5 +1,5 @@
 from django.contrib import admin
-from registry.models import Passport, Score, Stamp
+from registry.models import Event, Passport, Score, Stamp
 
 
 class PassportAdmin(admin.ModelAdmin):
@@ -49,6 +49,26 @@ def get_queryset(self, request):
         return queryset
 
 
+class EventAdmin(admin.ModelAdmin):
+    list_display = [
+        "action",
+        "created_at",
+        "address",
+        "data",
+    ]
+
+    list_filter = [
+        "action",
+    ]
+
+    search_fields = [
+        "created_at",
+        "address",
+        "data",
+    ]
+
+
 admin.site.register(Passport, PassportAdmin)
 admin.site.register(Stamp, StampAdmin)
 admin.site.register(Score, ScoreAdmin)
+admin.site.register(Event, EventAdmin)

api/registry/migrations/0013_event.py

@@ -0,0 +1,40 @@
+# Generated by Django 4.2.3 on 2023-07-19 15:34
+
+import account.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("registry", "0012_alter_score_status"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "action",
+                    models.CharField(
+                        choices=[
+                            ("FDP", "Fifo Deduplication"),
+                            ("LDP", "Lifo Deduplication"),
+                        ],
+                        max_length=3,
+                    ),
+                ),
+                ("address", account.models.EthAddressField(blank=True, max_length=42)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("data", models.JSONField()),
+            ],
+        ),
+    ]

api/registry/models.py

@@ -67,3 +67,23 @@ class Status:
 
     def __str__(self):
         return f"Score #{self.id}, score={self.score}, last_score_timestamp={self.last_score_timestamp}, status={self.status}, error={self.error}, evidence={self.evidence}, passport_id={self.passport_id}"
+
+
+class Event(models.Model):
+    # Example usage:
+    #   obj.action = Event.Action.FIFO_DEDUPLICATION
+    class Action(models.TextChoices):
+        FIFO_DEDUPLICATION = "FDP"
+        LIFO_DEDUPLICATION = "LDP"
+
+    action = models.CharField(
+        max_length=3,
+        choices=Action.choices,
+        blank=False,
+    )
+
+    address = EthAddressField(blank=True, max_length=42)
+
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    data = models.JSONField()