diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index 6f21d7716..4c4887788 100644 --- a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -6,6 +6,8 @@ from django.db.models.functions import ExtractYear, ExtractMonth from django.utils import timezone +from onadata.apps.logger.utils import delete_null_user_daily_counters + def populate_missing_monthly_counters(apps, schema_editor): @@ -66,6 +68,10 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython( + delete_null_user_daily_counters, + migrations.RunPython.noop, + ), migrations.RunPython( populate_missing_monthly_counters, migrations.RunPython.noop, diff --git a/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py b/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py new file mode 100644 index 000000000..8be5f80f0 --- /dev/null +++ b/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py @@ -0,0 +1,19 @@ +from django.conf import settings +from django.db import migrations + +from onadata.apps.logger.utils import delete_null_user_daily_counters + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0030_backfill_lost_monthly_counters'), + ] + + operations = [ + migrations.RunPython( + delete_null_user_daily_counters, + migrations.RunPython.noop, + ), + ] diff --git a/onadata/apps/logger/migrations/0032_alter_daily_submission_counter_user.py b/onadata/apps/logger/migrations/0032_alter_daily_submission_counter_user.py new file mode 100644 index 000000000..0bbd1dde7 --- /dev/null +++ b/onadata/apps/logger/migrations/0032_alter_daily_submission_counter_user.py @@ -0,0 +1,18 @@ +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0031_remove_null_user_daily_counters'), + ] + + operations = [ + migrations.AlterField( + model_name='dailyxformsubmissioncounter', + name='user', + field=models.ForeignKey('auth.User', related_name='daily_users', null=False, on_delete=models.CASCADE), + ), + ] diff --git a/onadata/apps/logger/models/daily_xform_submission_counter.py b/onadata/apps/logger/models/daily_xform_submission_counter.py index 25abadc3b..a6b962eca 100644 --- a/onadata/apps/logger/models/daily_xform_submission_counter.py +++ b/onadata/apps/logger/models/daily_xform_submission_counter.py @@ -7,7 +7,7 @@ class DailyXFormSubmissionCounter(models.Model): date = models.DateField() - user = models.ForeignKey(User, related_name='daily_counts', null=True, on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='daily_counts', on_delete=models.CASCADE) xform = models.ForeignKey( 'logger.XForm', related_name='daily_counters', null=True, on_delete=models.CASCADE ) diff --git a/onadata/apps/logger/utils.py b/onadata/apps/logger/utils.py new file mode 100644 index 000000000..0e6a31fec --- /dev/null +++ b/onadata/apps/logger/utils.py @@ -0,0 +1,37 @@ +def delete_null_user_daily_counters(apps, *args): + """ + Find any DailyXFormCounters without a user, assign them to a user if we can, otherwise delete them + This function is reused between two migrations, logger.0030 and logger.0031. + If/when those migrations get squashed, please delete this function + """ + DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter') # noqa + + counters_without_users = DailyXFormSubmissionCounter.objects.filter(user=None) + + if not counters_without_users.exists(): + return + + # Associate each daily counter with user=None with a user based on its xform + batch = [] + batch_size = 5000 + for counter in ( + counters_without_users + .exclude(xform=None) + .exclude(xform__user=None) + .iterator(chunk_size=batch_size) + ): + counter.user = counter.xform.user + # don't add a user to duplicate counters, so they get deleted when we're done looping + if DailyXFormSubmissionCounter.objects.filter( + date=counter.date, xform=counter.xform + ).exclude(user=None).exists(): + continue + batch.append(counter) + if len(batch) >= batch_size: + DailyXFormSubmissionCounter.objects.bulk_update(batch, ['user_id']) + batch = [] + if batch: + DailyXFormSubmissionCounter.objects.bulk_update(batch, ['user_id']) + + # Delete daily counters without a user to avoid creating invalid monthly counters + DailyXFormSubmissionCounter.objects.filter(user=None).delete()