Skip to content

Commit

Permalink
chore(issues): Backfill IGNORED groups with missing substatuses (#75702)
Browse files Browse the repository at this point in the history
We need to run a backfill to fix the missing substatuses for IGNORED
groups. This [redash query](https://redash.getsentry.net/queries/6898)
shows we have ~120K ignored groups that have no substatus. Further
digging shows that we _should_ have the info needed to backfill via
Activity and GroupSnooze tables.

Marking this as a post-deploy migration for safety.

Fixes #75684
  • Loading branch information
snigdhas authored Aug 29, 2024
1 parent 2bfd35e commit fa2fcd9
Show file tree
Hide file tree
Showing 3 changed files with 210 additions and 1 deletion.
2 changes: 1 addition & 1 deletion migrations_lockfile.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ hybridcloud: 0016_add_control_cacheversion
nodestore: 0002_nodestore_no_dictfield
remote_subscriptions: 0003_drop_remote_subscription
replays: 0004_index_together
sentry: 0752_fix_substatus_for_unresolved_groups
sentry: 0753_fix_substatus_for_ignored_groups
social_auth: 0002_default_auto_field
uptime: 0007_update_detected_subscription_interval
107 changes: 107 additions & 0 deletions src/sentry/migrations/0753_fix_substatus_for_ignored_groups.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# Generated by Django 5.0.7 on 2024-08-05 17:50

from django.apps.registry import Apps
from django.db import migrations
from django.db.backends.base.schema import BaseDatabaseSchemaEditor

from sentry.new_migrations.migrations import CheckedMigration
from sentry.utils.query import RangeQuerySetWrapperWithProgressBarApprox

# Copying constants defined in the models


class ActivityType:
SET_IGNORED = 3


class GroupHistoryStatus:
ARCHIVED_UNTIL_ESCALATING = 15
ARCHIVED_FOREVER = 16
ARCHIVED_UNTIL_CONDITION_MET = 17


class GroupSubStatus:
UNTIL_ESCALATING = 1
# Group is ignored/archived for a count/user count/duration
UNTIL_CONDITION_MET = 4
# Group is ignored/archived forever
FOREVER = 5


class GroupStatus:
IGNORED = 2


# End copy

ACTIVITY_DATA_FIELDS = {
"ignoreCount",
"ignoreDuration",
"ignoreUntil",
"ignoreUserCount",
"ignoreUserWindow",
"ignoreWindow",
}


def backfill_substatus_for_ignored_groups(
apps: Apps, schema_editor: BaseDatabaseSchemaEditor
) -> None:
Group = apps.get_model("sentry", "Group")
Activity = apps.get_model("sentry", "Activity")
GroupSnooze = apps.get_model("sentry", "GroupSnooze")

activity = Activity.objects.filter(type=ActivityType.SET_IGNORED)
for group in RangeQuerySetWrapperWithProgressBarApprox(
Group.objects.filter(status=GroupStatus.IGNORED, substatus=None),
):
group_activity = activity.filter(group_id=group.id).order_by("-datetime").first()
new_substatus = None
if group_activity:
# If ignoreUntilEscalating is set, we should set the substatus to UNTIL_ESCALATING
if group_activity.data.get("ignoreUntilEscalating", False):
new_substatus = GroupSubStatus.UNTIL_ESCALATING
# If any other field in the activity data is set, we should set the substatus to UNTIL_CONDITION_MET
elif any(group_activity.data.get(field) for field in ACTIVITY_DATA_FIELDS):
new_substatus = GroupSubStatus.UNTIL_CONDITION_MET

# If no activity is found or the activity data is not set, check the group snooze table
if not new_substatus:
snooze = GroupSnooze.objects.filter(group=group)
if snooze.exists():
# If snooze exists, we should set the substatus to UNTIL_CONDITION_MET
new_substatus = GroupSubStatus.UNTIL_CONDITION_MET
else:
# If we have no other information stored about the group's status conditions, the group is ignored forever
new_substatus = GroupSubStatus.FOREVER

group.substatus = new_substatus
group.save(update_fields=["substatus"])


class Migration(CheckedMigration):
# This flag is used to mark that a migration shouldn't be automatically run in production.
# This should only be used for operations where it's safe to run the migration after your
# code has deployed. So this should not be used for most operations that alter the schema
# of a table.
# Here are some things that make sense to mark as post deployment:
# - Large data migrations. Typically we want these to be run manually so that they can be
# monitored and not block the deploy for a long period of time while they run.
# - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to
# run this outside deployments so that we don't block them. Note that while adding an index
# is a schema change, it's completely safe to run the operation after the code has deployed.
# Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment

is_post_deployment = True

dependencies = [
("sentry", "0752_fix_substatus_for_unresolved_groups"),
]

operations = [
migrations.RunPython(
backfill_substatus_for_ignored_groups,
migrations.RunPython.noop,
hints={"tables": ["sentry_groupedmessage"]},
),
]
102 changes: 102 additions & 0 deletions tests/sentry/migrations/test_0753_fix_substatus_for_ignored_groups.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
from sentry.models.activity import Activity
from sentry.models.group import Group, GroupStatus
from sentry.models.groupsnooze import GroupSnooze
from sentry.models.organization import Organization
from sentry.testutils.cases import TestMigrations
from sentry.types.activity import ActivityType
from sentry.types.group import GroupSubStatus


class FixSubstatusForIgnoreedGroupsTest(TestMigrations):
migrate_from = "0752_fix_substatus_for_unresolved_groups"
migrate_to = "0753_fix_substatus_for_ignored_groups"

def setup_before_migration(self, app):
self.organization = Organization.objects.create(name="test", slug="test")
self.project = self.create_project(organization=self.organization)
self.do_not_update = Group.objects.create(
project=self.project,
status=GroupStatus.IGNORED,
substatus=GroupSubStatus.UNTIL_ESCALATING,
)

self.ignored_until_condition_met = Group.objects.create(
project=self.project,
status=GroupStatus.IGNORED,
)
# .update() skips calling the pre_save checks which requires a substatus
self.ignored_until_condition_met.update(substatus=None)
self.ignored_until_condition_met.refresh_from_db()
assert self.ignored_until_condition_met.substatus is None
Activity.objects.create(
group=self.ignored_until_condition_met,
project=self.project,
type=ActivityType.SET_IGNORED.value,
data={"ignoreCount": 10},
)

self.ignored_until_condition_met_no_activity = Group.objects.create(
project=self.project,
status=GroupStatus.IGNORED,
)
self.ignored_until_condition_met_no_activity.update(substatus=None)
assert self.ignored_until_condition_met_no_activity.substatus is None
Activity.objects.create(
group=self.ignored_until_condition_met_no_activity,
project=self.project,
type=ActivityType.SET_IGNORED.value,
data={
"ignoreCount": None,
"ignoreDuration": None,
"ignoreUntil": None,
"ignoreUserCount": None,
"ignoreUserWindow": None,
"ignoreWindow": None,
"ignoreUntilEscalating": None,
},
)
GroupSnooze.objects.create(
group=self.ignored_until_condition_met_no_activity,
count=10,
)

self.ignored_until_escalating = Group.objects.create(
project=self.project,
status=GroupStatus.IGNORED,
)
# .update() skips calling the pre_save checks which requires a substatus
self.ignored_until_escalating.update(substatus=None)
self.ignored_until_escalating.refresh_from_db()
assert self.ignored_until_escalating.substatus is None
Activity.objects.create(
group=self.ignored_until_escalating,
project=self.project,
type=ActivityType.SET_IGNORED.value,
data={"ignoreUntilEscalating": True},
)

self.ignored_forever = Group.objects.create(
project=self.project,
status=GroupStatus.IGNORED,
)
self.ignored_forever.update(substatus=None)
assert self.ignored_forever.substatus is None

def test(self):
self.do_not_update.refresh_from_db()
assert self.do_not_update.substatus == GroupSubStatus.UNTIL_ESCALATING

self.ignored_until_condition_met.refresh_from_db()
assert self.ignored_until_condition_met.substatus == GroupSubStatus.UNTIL_CONDITION_MET

self.ignored_until_condition_met_no_activity.refresh_from_db()
assert (
self.ignored_until_condition_met_no_activity.substatus
== GroupSubStatus.UNTIL_CONDITION_MET
)

self.ignored_until_escalating.refresh_from_db()
assert self.ignored_until_escalating.substatus == GroupSubStatus.UNTIL_ESCALATING

self.ignored_forever.refresh_from_db()
assert self.ignored_forever.substatus == GroupSubStatus.FOREVER

0 comments on commit fa2fcd9

Please sign in to comment.