From 295c12d105aed244c00ebc4cb72b4fb7931f093d Mon Sep 17 00:00:00 2001 From: Alex Zaslavsky Date: Wed, 9 Aug 2023 15:57:23 -0700 Subject: [PATCH] ref(backup): Encapsulate old export config (#54322) There are a number of properties of the export system API that we'd like to eventually change. These include: - Barring users from manually excluding/including specific models on export - Only exporting `sentry...` models To keep the original behavior export system intact until we debut these changes, this commit hides the behind `OldExportConfig`, which allows them to be toggled on (for tests) but left off for the actual CLI tool (for now). Issue: getsentry/team-ospo#171 --- .../backup/datetime-with-unzeroed-millis.json | 8 ------ .../backup/datetime-with-zeroed-millis.json | 8 ------ fixtures/backup/fresh-install.json | 8 ------ fixtures/backup/single-option.json | 8 ------ src/sentry/backup/exports.py | 26 +++++++++++++------ src/sentry/runner/commands/backup.py | 23 ++++++++++++---- src/sentry/testutils/helpers/backups.py | 4 +-- 7 files changed, 38 insertions(+), 47 deletions(-) diff --git a/fixtures/backup/datetime-with-unzeroed-millis.json b/fixtures/backup/datetime-with-unzeroed-millis.json index 16df5ee76dee97..bab9daa1b17d51 100644 --- a/fixtures/backup/datetime-with-unzeroed-millis.json +++ b/fixtures/backup/datetime-with-unzeroed-millis.json @@ -1,12 +1,4 @@ [ - { - "model": "sites.site", - "pk": 1, - "fields": { - "domain": "example.com", - "name": "example.com" - } - }, { "model": "sentry.option", "pk": 1, diff --git a/fixtures/backup/datetime-with-zeroed-millis.json b/fixtures/backup/datetime-with-zeroed-millis.json index a5c6d0a167dc21..144d8cf98b5abc 100644 --- a/fixtures/backup/datetime-with-zeroed-millis.json +++ b/fixtures/backup/datetime-with-zeroed-millis.json @@ -1,12 +1,4 @@ [ - { - "model": "sites.site", - "pk": 1, - "fields": { - "domain": "example.com", - "name": "example.com" - } - }, { "model": "sentry.option", "pk": 1, diff --git a/fixtures/backup/fresh-install.json b/fixtures/backup/fresh-install.json index 32f2c621d2b3ad..99c579b3c0aa09 100644 --- a/fixtures/backup/fresh-install.json +++ b/fixtures/backup/fresh-install.json @@ -1,12 +1,4 @@ [ -{ - "model": "sites.site", - "pk": 1, - "fields": { - "domain": "example.com", - "name": "example.com" - } -}, { "model": "sentry.option", "pk": 1, diff --git a/fixtures/backup/single-option.json b/fixtures/backup/single-option.json index 68535efe8696f6..b83553c9ca47b7 100644 --- a/fixtures/backup/single-option.json +++ b/fixtures/backup/single-option.json @@ -1,12 +1,4 @@ [ - { - "model": "sites.site", - "pk": 1, - "fields": { - "domain": "example.com", - "name": "example.com" - } - }, { "model": "sentry.option", "pk": 1, diff --git a/src/sentry/backup/exports.py b/src/sentry/backup/exports.py index 7be7772a1e6df5..6d11df381e50ac 100644 --- a/src/sentry/backup/exports.py +++ b/src/sentry/backup/exports.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone +from typing import NamedTuple import click from django.core.serializers import serialize @@ -76,6 +77,7 @@ def sort_dependencies(): rel_model = getattr(field.remote_field, "model", None) if rel_model is not None and rel_model != model: deps.append(rel_model) + model_dependencies.append((model, deps)) model_dependencies.reverse() @@ -119,20 +121,28 @@ def sort_dependencies(): return model_list -def exports(dest, indent, exclude, printer=click.echo): - """Exports core metadata for the Sentry installation.""" +class OldExportConfig(NamedTuple): + """While we are migrating to the new backup system, we need to take care not to break the old + and relatively untested workflows. This model allows us to stub in the old configs.""" + + # Do we include models that aren't in `sentry.*` databases, like the native Django ones (sites, + # sessions, etc)? + include_non_sentry_models: bool = False + + # A list of models to exclude from the export - eventually we want to deprecate and remove this + # option. + excluded_models: set[str] = set() + - if exclude is None: - exclude = () - else: - exclude = exclude.lower().split(",") +def exports(dest, old_config: OldExportConfig, indent: int, printer=click.echo): + """Exports core data for the Sentry installation.""" def yield_objects(): # Collate the objects to be serialized. for model in sort_dependencies(): if ( - not getattr(model, "__include_in_export__", True) - or model.__name__.lower() in exclude + not getattr(model, "__include_in_export__", old_config.include_non_sentry_models) + or model.__name__.lower() in old_config.excluded_models or model._meta.proxy ): printer(f">> Skipping model <{model.__name__}>", err=True) diff --git a/src/sentry/runner/commands/backup.py b/src/sentry/runner/commands/backup.py index 7f49e38685a560..c850b076c12df7 100644 --- a/src/sentry/runner/commands/backup.py +++ b/src/sentry/runner/commands/backup.py @@ -2,7 +2,7 @@ import click -from sentry.backup.exports import exports +from sentry.backup.exports import OldExportConfig, exports from sentry.backup.imports import imports from sentry.runner.decorators import configuration @@ -12,7 +12,7 @@ @click.option("--silent", "-q", default=False, is_flag=True, help="Silence all debug output.") @configuration def import_(src, silent): - """CLI command wrapping the `exec_import` functionality.""" + """Imports core data for a Sentry installation.""" imports(src, (lambda *args, **kwargs: None) if silent else click.echo) @@ -26,6 +26,19 @@ def import_(src, silent): @click.option("--exclude", default=None, help="Models to exclude from export.", metavar="MODELS") @configuration def export(dest, silent, indent, exclude): - """Exports core metadata for the Sentry installation.""" - - exports(dest, indent, exclude, (lambda *args, **kwargs: None) if silent else click.echo) + """Exports core data for the Sentry installation.""" + + if exclude is None: + exclude = [] + else: + exclude = exclude.lower().split(",") + + exports( + dest, + OldExportConfig( + include_non_sentry_models=True, + excluded_models=set(exclude), + ), + indent, + (lambda *args, **kwargs: None) if silent else click.echo, + ) diff --git a/src/sentry/testutils/helpers/backups.py b/src/sentry/testutils/helpers/backups.py index 031eddd626d898..ec8cb36c5c2cea 100644 --- a/src/sentry/testutils/helpers/backups.py +++ b/src/sentry/testutils/helpers/backups.py @@ -6,7 +6,7 @@ from django.core.management import call_command from sentry.backup.comparators import ComparatorMap -from sentry.backup.exports import exports +from sentry.backup.exports import OldExportConfig, exports from sentry.backup.findings import ComparatorFindings from sentry.backup.helpers import get_exportable_final_derivations_of, get_final_derivations_of from sentry.backup.imports import imports @@ -39,7 +39,7 @@ def export_to_file(path: Path) -> JSONData: json_file_path = str(path) with open(json_file_path, "w+") as tmp_file: - exports(tmp_file, 2, None, NOOP_PRINTER) + exports(tmp_file, OldExportConfig(), 2, NOOP_PRINTER) with open(json_file_path) as tmp_file: output = json.load(tmp_file)