feat(uptime): Add url_domain and url_suffix columns to the uptime sub…

…scription model (#76366)

This adds `url_domain` and `url_suffix` to the uptime subscription
model. The plan is to store the domainof each url here for use in
limiting total number of checks per domain

We'll use `url_suffix` to understand how many customers are using a
specific host, which can be useful for analytics and potentially for
large downtime detection.

<!-- Describe your PR here. -->

migrations_lockfile.txt

@@ -12,4 +12,4 @@ remote_subscriptions: 0003_drop_remote_subscription
 replays: 0004_index_together
 sentry: 0753_fix_substatus_for_ignored_groups
 social_auth: 0002_default_auto_field
-uptime: 0007_update_detected_subscription_interval
+uptime: 0008_uptime_url_suffix

src/sentry/uptime/migrations/0008_uptime_url_suffix.py

@@ -0,0 +1,90 @@
+# Generated by Django 5.0.8 on 2024-08-16 22:25
+
+from django.db import migrations, models
+
+from sentry.new_migrations.migrations import CheckedMigration
+
+
+class Migration(CheckedMigration):
+    # This flag is used to mark that a migration shouldn't be automatically run in production.
+    # This should only be used for operations where it's safe to run the migration after your
+    # code has deployed. So this should not be used for most operations that alter the schema
+    # of a table.
+    # Here are some things that make sense to mark as post deployment:
+    # - Large data migrations. Typically we want these to be run manually so that they can be
+    #   monitored and not block the deploy for a long period of time while they run.
+    # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to
+    #   run this outside deployments so that we don't block them. Note that while adding an index
+    #   is a schema change, it's completely safe to run the operation after the code has deployed.
+    # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment
+
+    is_post_deployment = False
+
+    dependencies = [
+        ("uptime", "0007_update_detected_subscription_interval"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    """
+                    ALTER TABLE "uptime_uptimesubscription" ADD COLUMN "url_domain" character varying(255) NOT NULL DEFAULT '';
+                    """,
+                    reverse_sql="""
+                ALTER TABLE "uptime_uptimesubscription" DROP COLUMN "url_domain";
+                """,
+                    hints={"tables": ["uptime_uptimesubscription"]},
+                ),
+                migrations.RunSQL(
+                    """
+                    ALTER TABLE "uptime_uptimesubscription" ADD COLUMN "url_domain_suffix" character varying(255) NOT NULL DEFAULT '';
+                    """,
+                    reverse_sql="""
+                ALTER TABLE "uptime_uptimesubscription" DROP COLUMN "url_domain_suffix";
+                """,
+                    hints={"tables": ["uptime_uptimesubscription"]},
+                ),
+                migrations.RunSQL(
+                    """
+                    ALTER TABLE "uptime_uptimesubscription" ADD COLUMN "host_whois_orgname" character varying(255) NOT NULL DEFAULT '';
+                    """,
+                    reverse_sql="""
+                ALTER TABLE "uptime_uptimesubscription" DROP COLUMN "host_whois_orgname";
+                """,
+                    hints={"tables": ["uptime_uptimesubscription"]},
+                ),
+                migrations.RunSQL(
+                    """
+                    ALTER TABLE "uptime_uptimesubscription" ADD COLUMN "host_whois_orgid" character varying(255) NOT NULL DEFAULT '';
+                    """,
+                    reverse_sql="""
+                ALTER TABLE "uptime_uptimesubscription" DROP COLUMN "host_whois_orgid";
+                """,
+                    hints={"tables": ["uptime_uptimesubscription"]},
+                ),
+            ],
+            state_operations=[
+                migrations.AddField(
+                    model_name="uptimesubscription",
+                    name="url_domain",
+                    field=models.CharField(db_index=True, default="", max_length=255),
+                ),
+                migrations.AddField(
+                    model_name="uptimesubscription",
+                    name="url_domain_suffix",
+                    field=models.CharField(db_index=True, default="", max_length=255),
+                ),
+                migrations.AddField(
+                    model_name="uptimesubscription",
+                    name="host_whois_orgname",
+                    field=models.CharField(db_index=True, default="", max_length=255),
+                ),
+                migrations.AddField(
+                    model_name="uptimesubscription",
+                    name="host_whois_orgid",
+                    field=models.CharField(db_index=True, default="", max_length=255),
+                ),
+            ],
+        )
+    ]

src/sentry/uptime/models.py

@@ -24,6 +24,15 @@ class UptimeSubscription(BaseRemoteSubscription, DefaultFieldsModel):
 
     # The url to check
     url = models.CharField(max_length=255)
+    # The domain of the url, extracted via TLDExtract
+    url_domain = models.CharField(max_length=255, db_index=True, default="")
+    # The suffix of the url, extracted via TLDExtract. This can be a public suffix, such as .com, .gov.uk, .com.au, or
+    # a private suffix, such as vercel.dev
+    url_domain_suffix = models.CharField(max_length=255, db_index=True, default="")
+    # Org name of the host of the url
+    host_whois_orgname = models.CharField(max_length=255, db_index=True, default="")
+    # Org id of the host of the url
+    host_whois_orgid = models.CharField(max_length=255, db_index=True, default="")
     # How frequently to run the check in seconds
     interval_seconds = models.IntegerField()
     # How long to wait for a response from the url before we assume a timeout

tests/sentry/migrations/test_0007_update_detected_subscription_interval.py

@@ -1,7 +1,10 @@
+import pytest
+
 from sentry.testutils.cases import TestMigrations
 from sentry.uptime.models import ProjectUptimeSubscriptionMode, UptimeSubscription
 
 
+@pytest.mark.skip("Migration is no longer runnable. Retain until migration is removed.")
 class UpdateAutoDetectedActiveIntervalSecondsTest(TestMigrations):
     migrate_from = "0006_projectuptimesubscription_name_owner"
     migrate_to = "0007_update_detected_subscription_interval"