From b2bb8812e7b0fc35b98057b091e73d431f69b0a6 Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Thu, 12 Oct 2023 09:29:51 -0500 Subject: [PATCH 1/7] delete daily submission counters without a user --- .../logger/migrations/0030_backfill_lost_monthly_counters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index 6f21d7716..432793c6a 100644 --- a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -15,6 +15,9 @@ def populate_missing_monthly_counters(apps, schema_editor): if not DailyXFormSubmissionCounter.objects.all().exists(): return + # Delete daily counters without a user to avoid creating invalid monthly counters + DailyXFormSubmissionCounter.objects.filter(user=None).delete() + previous_migration = MigrationRecorder.Migration.objects.filter( app='logger', name='0029_populate_daily_xform_counters_for_year' ).first() From f0da3c3b54387c0bed9c2da3b3d4327d7795ad21 Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Thu, 12 Oct 2023 09:30:46 -0500 Subject: [PATCH 2/7] make DailyXFormSubmissionCounter.user non-nullable --- ...0031_alter_daily_submission_counter_user.py | 18 ++++++++++++++++++ .../models/daily_xform_submission_counter.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py diff --git a/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py b/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py new file mode 100644 index 000000000..37e216989 --- /dev/null +++ b/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py @@ -0,0 +1,18 @@ +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0030_backfill_lost_monthly_counters'), + ] + + operations = [ + migrations.AlterField( + model_name='dailyxformsubmissioncounter', + name='user', + field=models.ForeignKey('auth.User', related_name='daily_users', null=False, on_delete=models.CASCADE), + ), + ] diff --git a/onadata/apps/logger/models/daily_xform_submission_counter.py b/onadata/apps/logger/models/daily_xform_submission_counter.py index 25abadc3b..a6b962eca 100644 --- a/onadata/apps/logger/models/daily_xform_submission_counter.py +++ b/onadata/apps/logger/models/daily_xform_submission_counter.py @@ -7,7 +7,7 @@ class DailyXFormSubmissionCounter(models.Model): date = models.DateField() - user = models.ForeignKey(User, related_name='daily_counts', null=True, on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='daily_counts', on_delete=models.CASCADE) xform = models.ForeignKey( 'logger.XForm', related_name='daily_counters', null=True, on_delete=models.CASCADE ) From 675a8e98ac3dc80a4bdada0a0e9a8033ed5c3590 Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Thu, 12 Oct 2023 11:18:03 -0500 Subject: [PATCH 3/7] don't delete counters if we can associate them with a user --- .../migrations/0030_backfill_lost_monthly_counters.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index 432793c6a..5b651adf6 100644 --- a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -15,6 +15,17 @@ def populate_missing_monthly_counters(apps, schema_editor): if not DailyXFormSubmissionCounter.objects.all().exists(): return + # Associate each daily counter with user=None with a user based on its xform + for counter in DailyXFormSubmissionCounter.objects.filter(user=None).iterator: + if counter.xform and counter.xform.user: + has_duplicate = DailyXFormSubmissionCounter.objects.filter( + date=counter.date, xform=counter.xform + ).exclude(user=None).exists() + # don't add a user to duplicate counters, so they get deleted in the next step + if not has_duplicate: + counter.user = counter.xform.user + counter.save() + # Delete daily counters without a user to avoid creating invalid monthly counters DailyXFormSubmissionCounter.objects.filter(user=None).delete() From ba90d45cbf81d66e91bc2731547330dc090d0be3 Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Fri, 13 Oct 2023 11:39:36 -0500 Subject: [PATCH 4/7] refactor logger.0030 and logger.0031 --- .../0030_backfill_lost_monthly_counters.py | 20 ++++------- ...031_alter_daily_submission_counter_user.py | 2 +- .../0031_remove_null_user_daily_counters.py | 19 +++++++++++ onadata/apps/logger/migrations/utils.py | 33 +++++++++++++++++++ 4 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py create mode 100644 onadata/apps/logger/migrations/utils.py diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index 5b651adf6..41205c63e 100644 --- a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -6,6 +6,8 @@ from django.db.models.functions import ExtractYear, ExtractMonth from django.utils import timezone +from kobocat.onadata.apps.logger.migrations.utils import delete_null_user_daily_counters + def populate_missing_monthly_counters(apps, schema_editor): @@ -15,20 +17,6 @@ def populate_missing_monthly_counters(apps, schema_editor): if not DailyXFormSubmissionCounter.objects.all().exists(): return - # Associate each daily counter with user=None with a user based on its xform - for counter in DailyXFormSubmissionCounter.objects.filter(user=None).iterator: - if counter.xform and counter.xform.user: - has_duplicate = DailyXFormSubmissionCounter.objects.filter( - date=counter.date, xform=counter.xform - ).exclude(user=None).exists() - # don't add a user to duplicate counters, so they get deleted in the next step - if not has_duplicate: - counter.user = counter.xform.user - counter.save() - - # Delete daily counters without a user to avoid creating invalid monthly counters - DailyXFormSubmissionCounter.objects.filter(user=None).delete() - previous_migration = MigrationRecorder.Migration.objects.filter( app='logger', name='0029_populate_daily_xform_counters_for_year' ).first() @@ -80,6 +68,10 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython( + delete_null_user_daily_counters, + migrations.RunPython.noop, + ), migrations.RunPython( populate_missing_monthly_counters, migrations.RunPython.noop, diff --git a/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py b/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py index 37e216989..0bbd1dde7 100644 --- a/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py +++ b/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0030_backfill_lost_monthly_counters'), + ('logger', '0031_remove_null_user_daily_counters'), ] operations = [ diff --git a/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py b/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py new file mode 100644 index 000000000..2663a5012 --- /dev/null +++ b/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py @@ -0,0 +1,19 @@ +from django.conf import settings +from django.db import migrations + +from kobocat.onadata.apps.logger.migrations.utils import delete_null_user_daily_counters + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0030_backfill_lost_monthly_counters'), + ] + + operations = [ + migrations.RunPython( + delete_null_user_daily_counters, + migrations.RunPython.noop, + ), + ] diff --git a/onadata/apps/logger/migrations/utils.py b/onadata/apps/logger/migrations/utils.py new file mode 100644 index 000000000..f8a28e2b7 --- /dev/null +++ b/onadata/apps/logger/migrations/utils.py @@ -0,0 +1,33 @@ +def delete_null_user_daily_counters(apps, schema_editor): + + DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter') # noqa + + counters_without_users = DailyXFormSubmissionCounter.objects.filter(user=None) + + if not counters_without_users.exists(): + return + + # Associate each daily counter with user=None with a user based on its xform + batch = [] + batch_size = 5000 + for counter in ( + counters_without_users + .exclude(xform=None) + .exclude(xform__user=None) + .iterator() + ): + counter.user = counter.xform.user + # don't add a user to duplicate counters, so they get deleted when we're done looping + if DailyXFormSubmissionCounter.objects.filter( + date=counter.date, xform=counter.xform + ).exclude(user=None).exists(): + continue + batch.append(counter) + if len(batch) >= batch_size: + DailyXFormSubmissionCounter.objects.bulk_update(batch, ['user_id']) + batch = [] + if batch: + DailyXFormSubmissionCounter.objects.bulk_update(batch, ['user_id']) + + # Delete daily counters without a user to avoid creating invalid monthly counters + DailyXFormSubmissionCounter.objects.filter(user=None).delete() From fb4ff4a7fff921b4162bb02ac919fde94927ed8b Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Fri, 13 Oct 2023 11:41:57 -0500 Subject: [PATCH 5/7] rename old logger.0031 to logger.0032 to keep schema alteration atomic --- ...ounter_user.py => 0032_alter_daily_submission_counter_user.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename onadata/apps/logger/migrations/{0031_alter_daily_submission_counter_user.py => 0032_alter_daily_submission_counter_user.py} (100%) diff --git a/onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py b/onadata/apps/logger/migrations/0032_alter_daily_submission_counter_user.py similarity index 100% rename from onadata/apps/logger/migrations/0031_alter_daily_submission_counter_user.py rename to onadata/apps/logger/migrations/0032_alter_daily_submission_counter_user.py From 7abba8da8ce21db4757994b1e98b4c185f493b00 Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Fri, 13 Oct 2023 14:01:37 -0500 Subject: [PATCH 6/7] move delete_null_user_daily_counters to loggers/utils.py --- .../migrations/0030_backfill_lost_monthly_counters.py | 2 +- .../migrations/0031_remove_null_user_daily_counters.py | 2 +- onadata/apps/logger/{migrations => }/utils.py | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) rename onadata/apps/logger/{migrations => }/utils.py (79%) diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index 41205c63e..4c4887788 100644 --- a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -6,7 +6,7 @@ from django.db.models.functions import ExtractYear, ExtractMonth from django.utils import timezone -from kobocat.onadata.apps.logger.migrations.utils import delete_null_user_daily_counters +from onadata.apps.logger.utils import delete_null_user_daily_counters def populate_missing_monthly_counters(apps, schema_editor): diff --git a/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py b/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py index 2663a5012..8be5f80f0 100644 --- a/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py +++ b/onadata/apps/logger/migrations/0031_remove_null_user_daily_counters.py @@ -1,7 +1,7 @@ from django.conf import settings from django.db import migrations -from kobocat.onadata.apps.logger.migrations.utils import delete_null_user_daily_counters +from onadata.apps.logger.utils import delete_null_user_daily_counters class Migration(migrations.Migration): diff --git a/onadata/apps/logger/migrations/utils.py b/onadata/apps/logger/utils.py similarity index 79% rename from onadata/apps/logger/migrations/utils.py rename to onadata/apps/logger/utils.py index f8a28e2b7..7de3af42d 100644 --- a/onadata/apps/logger/migrations/utils.py +++ b/onadata/apps/logger/utils.py @@ -1,5 +1,9 @@ -def delete_null_user_daily_counters(apps, schema_editor): - +def delete_null_user_daily_counters(apps, *args): + """ + Find any DailyXFormCounters without a user, assign them to a user if we can, otherwise delete them + This function is reused between two migrations, logger.0030 and logger.0031. + If/when those migrations get squashed, please delete this function + """ DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter') # noqa counters_without_users = DailyXFormSubmissionCounter.objects.filter(user=None) From a82302e594cff7d4e288a824bd2010c965eba5b7 Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Fri, 13 Oct 2023 14:27:17 -0500 Subject: [PATCH 7/7] use batch_size in iterator --- onadata/apps/logger/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/logger/utils.py b/onadata/apps/logger/utils.py index 7de3af42d..0e6a31fec 100644 --- a/onadata/apps/logger/utils.py +++ b/onadata/apps/logger/utils.py @@ -18,7 +18,7 @@ def delete_null_user_daily_counters(apps, *args): counters_without_users .exclude(xform=None) .exclude(xform__user=None) - .iterator() + .iterator(chunk_size=batch_size) ): counter.user = counter.xform.user # don't add a user to duplicate counters, so they get deleted when we're done looping