feat(api): added events table, logging deduplication events

api/.env-sample

@@ -1,4 +1,5 @@
 DEBUG=on
+LOG_SQL_QUERIES=False
 SECRET_KEY=this_should_be_a_super_secret_key
 DATABASE_URL=sqlite:///db.sqlite3
 DATABASE_URL_FOR_DOCKER=postgres://passport_scorer:passport_scorer_pwd@postgres:5432/passport_scorer

api/account/deduplication/fifo.py

@@ -3,7 +3,7 @@
 
 import api_logging as logging
 from account.models import Community
-from registry.models import Stamp
+from registry.models import Event, Stamp
 
 log = logging.getLogger(__name__)
 
@@ -15,19 +15,45 @@ async def afifo(
     deduped_passport = copy.deepcopy(fifo_passport)
     affected_passports = []
     if "stamps" in fifo_passport:
-        for stamp in fifo_passport["stamps"]:
-            stamp_hash = stamp["credential"]["credentialSubject"]["hash"]
+        dedup_event_data = []
+        new_stamp_hashes = [
+            stamp["credential"]["credentialSubject"]["hash"]
+            for stamp in fifo_passport["stamps"]
+        ]
 
-            existing_stamps = (
-                Stamp.objects.filter(hash=stamp_hash, passport__community=community)
-                .exclude(passport__address=address)
-                .select_related("passport")
+        existing_stamps = (
+            Stamp.objects.filter(
+                hash__in=new_stamp_hashes, passport__community=community
             )
+            .exclude(passport__address=address)
+            .select_related("passport")
+        )
 
-            async for existing_stamp in existing_stamps:
-                existing_stamp_passport = existing_stamp.passport
-                await existing_stamp.adelete()
-                await existing_stamp_passport.asave()
-                affected_passports.append(existing_stamp_passport)
+        async for existing_stamp in existing_stamps:
+            existing_stamp_passport = existing_stamp.passport
+            affected_passports.append(existing_stamp_passport)
+            dedup_event_data.append(
+                {
+                    "hash": existing_stamp.hash,
+                    "provider": existing_stamp.provider,
+                    "prev_owner": existing_stamp_passport.address,
+                    "address": address,
+                    "community_id": community.pk,
+                }
+            )
+
+        await existing_stamps.adelete()
+
+        if dedup_event_data:
+            await Event.objects.abulk_create(
+                [
+                    Event(
+                        action=Event.Action.FIFO_DEDUPLICATION,
+                        address=data["prev_owner"],
+                        data=data,
+                    )
+                    for data in dedup_event_data
+                ]
+            )
 
     return (deduped_passport, affected_passports)

api/account/deduplication/lifo.py

@@ -3,7 +3,7 @@
 
 import api_logging as logging
 from account.models import Community
-from registry.models import Stamp
+from registry.models import Event, Stamp
 
 log = logging.getLogger(__name__)
 
@@ -14,18 +14,41 @@ async def alifo(
 ) -> Tuple[dict, list | None]:
     deduped_passport = copy.deepcopy(lifo_passport)
     deduped_passport["stamps"] = []
+
     if "stamps" in lifo_passport:
+        stamp_hashes = [
+            stamp["credential"]["credentialSubject"]["hash"]
+            for stamp in lifo_passport["stamps"]
+        ]
+
+        clashing_stamps = (
+            Stamp.objects.filter(hash__in=stamp_hashes, passport__community=community)
+            .exclude(passport__address=address)
+            .values("hash", "passport__address", "provider")
+        )
+
+        clashing_hashes = {stamp["hash"] async for stamp in clashing_stamps}
+
         for stamp in lifo_passport["stamps"]:
-            stamp_hash = stamp["credential"]["credentialSubject"]["hash"]
-
-            # query db to see if hash already exists, if so remove stamp from passport
-            if (
-                not await Stamp.objects.filter(
-                    hash=stamp_hash, passport__community=community
-                )
-                .exclude(passport__address=address)
-                .aexists()
-            ):
+            if stamp["credential"]["credentialSubject"]["hash"] not in clashing_hashes:
                 deduped_passport["stamps"].append(copy.deepcopy(stamp))
 
+        if clashing_stamps.aexists():
+            await Event.objects.abulk_create(
+                [
+                    Event(
+                        action=Event.Action.LIFO_DEDUPLICATION,
+                        address=address,
+                        data={
+                            "hash": stamp["hash"],
+                            "provider": stamp["provider"],
+                            "owner": stamp["passport__address"],
+                            "address": address,
+                            "community_id": community.pk,
+                        },
+                    )
+                    async for stamp in clashing_stamps
+                ]
+            )
+
     return (deduped_passport, None)

api/registry/migrations/0013_event.py

@@ -0,0 +1,40 @@
+# Generated by Django 4.2.3 on 2023-07-19 15:34
+
+import account.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("registry", "0012_alter_score_status"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "action",
+                    models.CharField(
+                        choices=[
+                            ("FDP", "Fifo Deduplication"),
+                            ("LDP", "Lifo Deduplication"),
+                        ],
+                        max_length=3,
+                    ),
+                ),
+                ("address", account.models.EthAddressField(blank=True, max_length=42)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("data", models.JSONField()),
+            ],
+        ),
+    ]

api/registry/models.py

@@ -67,3 +67,23 @@ class Status:
 
     def __str__(self):
         return f"Score #{self.id}, score={self.score}, last_score_timestamp={self.last_score_timestamp}, status={self.status}, error={self.error}, evidence={self.evidence}, passport_id={self.passport_id}"
+
+
+class Event(models.Model):
+    # Example usage:
+    #   obj.action = Event.Action.FIFO_DEDUPLICATION
+    class Action(models.TextChoices):
+        FIFO_DEDUPLICATION = "FDP"
+        LIFO_DEDUPLICATION = "LDP"
+
+    action = models.CharField(
+        max_length=3,
+        choices=Action.choices,
+        blank=False,
+    )
+
+    address = EthAddressField(blank=True, max_length=42)
+
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    data = models.JSONField()

api/registry/test/test_score_passport.py

@@ -3,12 +3,13 @@
 from decimal import Decimal
 from unittest.mock import call, patch
 
+from account.deduplication import Rules
 from account.models import Account, AccountAPIKey, Community
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.test import Client, TransactionTestCase
 from registry.api.v2 import SubmitPassportPayload, get_score, submit_passport
-from registry.models import Passport, Score, Stamp
+from registry.models import Event, Passport, Score, Stamp
 from registry.tasks import score_passport_passport, score_registry_passport
 from web3 import Web3
 
@@ -312,6 +313,11 @@ def test_lifo_duplicate_stamp_scoring(self):
             ):
                 score_registry_passport(self.community.pk, passport.address)
 
+            assert (
+                Event.objects.filter(action=Event.Action.LIFO_DEDUPLICATION).count()
+                == 0
+            )
+
             # Score passport with duplicates (one duplicate from original passport,
             # one duplicate from already existing stamp)
             with patch(
@@ -327,6 +333,11 @@ def test_lifo_duplicate_stamp_scoring(self):
 
             assert (Score.objects.get(passport=passport).score) == Decimal("3")
 
+            assert (
+                Event.objects.filter(action=Event.Action.LIFO_DEDUPLICATION).count()
+                == 2
+            )
+
             deduplicated_stamps = Stamp.objects.filter(
                 passport=passport_with_duplicates
             )
@@ -336,10 +347,157 @@ def test_lifo_duplicate_stamp_scoring(self):
                 Score.objects.get(passport=passport_with_duplicates).score
             ) == Decimal("1")
 
+            passport.requires_calculation = True
+            passport.save()
             # Re-score original passport, just to make sure it doesn't change
             with patch(
                 "registry.atasks.aget_passport", return_value=mock_passport_data
             ):
                 score_registry_passport(self.community.pk, passport.address)
 
             assert (Score.objects.get(passport=passport).score) == Decimal("3")
+            assert (
+                Event.objects.filter(action=Event.Action.LIFO_DEDUPLICATION).count()
+                == 2
+            )
+
+    def test_fifo_duplicate_stamp_scoring(self):
+        with patch(
+            "scorer_weighted.models.settings.GITCOIN_PASSPORT_WEIGHTS",
+            {
+                "Google": 1,
+                "Ens": 2,
+                "POAP": 4,
+            },
+        ):
+            fifo_community = Community.objects.create(
+                name="My Community",
+                description="My Community description",
+                account=self.user_account,
+                rule=Rules.FIFO.value,
+            )
+
+        passport, _ = Passport.objects.update_or_create(
+            address=self.account.address,
+            community_id=fifo_community.pk,
+            requires_calculation=True,
+        )
+
+        passport_for_already_existing_stamp, _ = Passport.objects.update_or_create(
+            address=self.account_2.address,
+            community_id=fifo_community.pk,
+            requires_calculation=True,
+        )
+
+        passport_with_duplicates, _ = Passport.objects.update_or_create(
+            address=self.account_3.address,
+            community_id=fifo_community.pk,
+            requires_calculation=True,
+        )
+
+        already_existing_stamp = {
+            "provider": "POAP",
+            "credential": {
+                "type": ["VerifiableCredential"],
+                "credentialSubject": {
+                    "id": settings.TRUSTED_IAM_ISSUER,
+                    "hash": "0x1111",
+                    "provider": "Gitcoin",
+                },
+                "issuer": settings.TRUSTED_IAM_ISSUER,
+                "issuanceDate": "2023-02-06T23:22:58.848Z",
+                "expirationDate": "2099-02-06T23:22:58.848Z",
+            },
+        }
+
+        Stamp.objects.update_or_create(
+            hash=already_existing_stamp["credential"]["credentialSubject"]["hash"],
+            passport=passport_for_already_existing_stamp,
+            defaults={
+                "provider": already_existing_stamp["provider"],
+                "credential": json.dumps(already_existing_stamp["credential"]),
+            },
+        )
+
+        mock_passport_data_with_duplicates = {
+            "stamps": [
+                mock_passport_data["stamps"][0],
+                already_existing_stamp,
+                {
+                    "provider": "Google",
+                    "credential": {
+                        "type": ["VerifiableCredential"],
+                        "credentialSubject": {
+                            "id": settings.TRUSTED_IAM_ISSUER,
+                            "hash": "0x12121",
+                            "provider": "Google",
+                        },
+                        "issuer": settings.TRUSTED_IAM_ISSUER,
+                        "issuanceDate": "2023-02-06T23:22:58.848Z",
+                        "expirationDate": "2099-02-06T23:22:58.848Z",
+                    },
+                },
+            ]
+        }
+
+        with patch("registry.atasks.validate_credential", side_effect=mock_validate):
+            # Score original passport
+            with patch(
+                "registry.atasks.aget_passport", return_value=mock_passport_data
+            ):
+                score_registry_passport(fifo_community.pk, passport.address)
+
+            assert (
+                Event.objects.filter(action=Event.Action.FIFO_DEDUPLICATION).count()
+                == 0
+            )
+
+            assert Stamp.objects.filter(passport=passport).count() == 3
+            assert (Score.objects.get(passport=passport).score) == Decimal("3")
+
+            # Score passport with duplicates (one duplicate from original passport,
+            # one duplicate from already existing stamp)
+            with patch(
+                "registry.atasks.aget_passport",
+                return_value=mock_passport_data_with_duplicates,
+            ):
+                score_registry_passport(
+                    fifo_community.pk, passport_with_duplicates.address
+                )
+
+            # One stamp should be removed from original passport
+            original_stamps = Stamp.objects.filter(passport=passport)
+            assert len(original_stamps) == 2
+
+            assert (Score.objects.get(passport=passport).score) == Decimal("1")
+
+            assert (
+                Event.objects.filter(action=Event.Action.FIFO_DEDUPLICATION).count()
+                == 2
+            )
+
+            new_stamps = Stamp.objects.filter(passport=passport_with_duplicates)
+            assert len(new_stamps) == 3
+
+            assert (
+                Score.objects.get(passport=passport_with_duplicates).score
+            ) == Decimal("7")
+
+            passport.requires_calculation = True
+            passport.save()
+            # Re-score original passport, it should get the full score again
+            with patch(
+                "registry.atasks.aget_passport", return_value=mock_passport_data
+            ):
+                score_registry_passport(fifo_community.pk, passport.address)
+
+            assert (
+                Event.objects.filter(action=Event.Action.FIFO_DEDUPLICATION).count()
+                == 3
+            )
+
+            assert (Score.objects.get(passport=passport).score) == Decimal("3")
+
+            assert (
+                Score.objects.get(passport=passport_with_duplicates).score
+            ) == Decimal("5")