Merge pull request #900 from kobotoolbox/daily-counters-user-null

Make user not nullable in DailyXFormSubmissionCounter

onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py

@@ -6,6 +6,8 @@
 from django.db.models.functions import ExtractYear, ExtractMonth
 from django.utils import timezone
 
+from onadata.apps.logger.utils import delete_null_user_daily_counters
+
 
 def populate_missing_monthly_counters(apps, schema_editor):
 
@@ -66,6 +68,10 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.RunPython(
+            delete_null_user_daily_counters,
+            migrations.RunPython.noop,
+        ),
         migrations.RunPython(
             populate_missing_monthly_counters,
             migrations.RunPython.noop,

onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py

@@ -0,0 +1,19 @@
+from django.conf import settings
+from django.db import migrations
+
+from onadata.apps.logger.utils import delete_null_user_daily_counters
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('logger', '0030_backfill_lost_monthly_counters'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            delete_null_user_daily_counters,
+            migrations.RunPython.noop,
+        ),
+    ]

onadata/apps/logger/migrations/0032_alter_daily_submission_counter_user.py

@@ -0,0 +1,18 @@
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('logger', '0031_remove_null_user_daily_counters'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dailyxformsubmissioncounter',
+            name='user',
+            field=models.ForeignKey('auth.User', related_name='daily_users', null=False, on_delete=models.CASCADE),
+        ),
+    ]

onadata/apps/logger/models/daily_xform_submission_counter.py

@@ -7,7 +7,7 @@
 
 class DailyXFormSubmissionCounter(models.Model):
     date = models.DateField()
-    user = models.ForeignKey(User, related_name='daily_counts', null=True, on_delete=models.CASCADE)
+    user = models.ForeignKey(User, related_name='daily_counts', on_delete=models.CASCADE)
     xform = models.ForeignKey(
         'logger.XForm', related_name='daily_counters', null=True, on_delete=models.CASCADE
     )

onadata/apps/logger/utils.py

@@ -0,0 +1,37 @@
+def delete_null_user_daily_counters(apps, *args):
+    """
+    Find any DailyXFormCounters without a user, assign them to a user if we can, otherwise delete them
+    This function is reused between two migrations, logger.0030 and logger.0031.
+    If/when those migrations get squashed, please delete this function
+    """
+    DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter')  # noqa
+
+    counters_without_users = DailyXFormSubmissionCounter.objects.filter(user=None)
+
+    if not counters_without_users.exists():
+        return
+
+    # Associate each daily counter with user=None with a user based on its xform
+    batch = []
+    batch_size = 5000
+    for counter in (
+            counters_without_users
+            .exclude(xform=None)
+            .exclude(xform__user=None)
+            .iterator(chunk_size=batch_size)
+    ):
+        counter.user = counter.xform.user
+        # don't add a user to duplicate counters, so they get deleted when we're done looping
+        if DailyXFormSubmissionCounter.objects.filter(
+                date=counter.date, xform=counter.xform
+        ).exclude(user=None).exists():
+            continue
+        batch.append(counter)
+        if len(batch) >= batch_size:
+            DailyXFormSubmissionCounter.objects.bulk_update(batch, ['user_id'])
+            batch = []
+    if batch:
+        DailyXFormSubmissionCounter.objects.bulk_update(batch, ['user_id'])
+
+    # Delete daily counters without a user to avoid creating invalid monthly counters
+    DailyXFormSubmissionCounter.objects.filter(user=None).delete()