diff --git a/catalog/dags/database/batched_update/batched_update_dag.py b/catalog/dags/database/batched_update/batched_update_dag.py index 40e0a83f3bf..9443f7e6796 100644 --- a/catalog/dags/database/batched_update/batched_update_dag.py +++ b/catalog/dags/database/batched_update/batched_update_dag.py @@ -131,13 +131,13 @@ pattern="^WHERE", ), "update_query": Param( - default="SET...", + default="SET updated_on = NOW(), ...", type="string", description=( "The part of the SQL `UPDATE` command, beginning with `SET`, that" " will be run for each batch." ), - pattern="^SET", + pattern="^SET updated_on = NOW()", ), "batch_size": Param( default=constants.DEFAULT_BATCH_SIZE, diff --git a/catalog/dags/popularity/sql.py b/catalog/dags/popularity/sql.py index 388bd02f92c..4edd024fb8e 100644 --- a/catalog/dags/popularity/sql.py +++ b/catalog/dags/popularity/sql.py @@ -318,7 +318,8 @@ def format_update_standardized_popularity_query( by a `batched_update` DagRun. """ return ( - f"SET {col.STANDARDIZED_POPULARITY.db_name} =" + f"SET {col.UPDATED_ON.db_name} = NOW()," + f" {col.STANDARDIZED_POPULARITY.db_name} =" f" {sql_info.standardized_popularity_fn}({sql_info.media_table}.{PARTITION}," f" {sql_info.media_table}.{METADATA_COLUMN})" ) diff --git a/catalog/tests/dags/popularity/test_sql.py b/catalog/tests/dags/popularity/test_sql.py index 3c42ab860e0..6ae57652328 100644 --- a/catalog/tests/dags/popularity/test_sql.py +++ b/catalog/tests/dags/popularity/test_sql.py @@ -430,7 +430,7 @@ def test_standardized_popularity_function_calculates( "query_id": "foo_provider_popularity_refresh_20230101", "table_name": "image", "select_query": "WHERE provider='foo_provider' AND updated_on < '2023-01-01 00:00:00'", - "update_query": "SET standardized_popularity = standardized_image_popularity(image.provider, image.meta_data)", + "update_query": "SET updated_on = NOW(), standardized_popularity = standardized_image_popularity(image.provider, image.meta_data)", "batch_size": 10000, "update_timeout": 3600.0, "dry_run": False, @@ -446,7 +446,7 @@ def test_standardized_popularity_function_calculates( "query_id": "my_provider_popularity_refresh_20230101", "table_name": "audio", "select_query": "WHERE provider='my_provider' AND updated_on < '2023-01-01 00:00:00'", - "update_query": "SET standardized_popularity = standardized_audio_popularity(audio.provider, audio.meta_data)", + "update_query": "SET updated_on = NOW(), standardized_popularity = standardized_audio_popularity(audio.provider, audio.meta_data)", "batch_size": 10000, "update_timeout": 3600.0, "dry_run": False, @@ -456,7 +456,7 @@ def test_standardized_popularity_function_calculates( "query_id": "your_provider_popularity_refresh_20230101", "table_name": "audio", "select_query": "WHERE provider='your_provider' AND updated_on < '2023-01-01 00:00:00'", - "update_query": "SET standardized_popularity = standardized_audio_popularity(audio.provider, audio.meta_data)", + "update_query": "SET updated_on = NOW(), standardized_popularity = standardized_audio_popularity(audio.provider, audio.meta_data)", "batch_size": 10000, "update_timeout": 3600.0, "dry_run": False,