Skip to content

Commit 4d4e365

Browse files
scholtzanscholtzan
committed
Cache dryrun results locally
1 parent 5cb73be commit 4d4e365

File tree

5 files changed

+421
-15
lines changed

5 files changed

+421
-15
lines changed

‎.circleci/workflows.yml‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,16 @@ executors:
3535
machine:
3636
image: ubuntu-2404:current
3737

38+
# environment variables for caching
39+
environment: &cache_environment
40+
DRYRUN_CACHE_ENABLED: "true"
41+
DRYRUN_CACHE_DIR: "/tmp/workspace/dryrun_cache"
42+
3843
jobs:
3944
build:
4045
docker: &docker
4146
- image: python:<< pipeline.parameters.python-version >>
47+
environment: *cache_environment
4248
steps:
4349
- checkout
4450
- &restore_venv_cache
@@ -166,7 +172,7 @@ jobs:
166172
- checkout
167173
- *restore_venv_cache
168174
- *build
169-
- &attach_generated_sql
175+
- &attach_workspace_artifacts
170176
attach_workspace:
171177
at: /tmp/workspace
172178
- &copy_staged_sql
@@ -181,6 +187,11 @@ jobs:
181187
name: Run SQL tests
182188
command: |
183189
PATH="venv/bin:$PATH" script/entrypoint -m sql -n 8 -p no:bigquery_etl.pytest_plugin.routine
190+
- &persist_dryrun_cache
191+
persist_to_workspace:
192+
root: /tmp/workspace
193+
paths:
194+
- dryrun_cache
184195
- &copy_debug_sql
185196
run:
186197
name: Copy generated SQL to save for debugging
@@ -214,7 +225,7 @@ jobs:
214225
- checkout
215226
- *restore_venv_cache
216227
- *build
217-
- *attach_generated_sql
228+
- *attach_workspace_artifacts
218229
- *copy_staged_sql
219230
- *authenticate
220231
- run:
@@ -244,6 +255,7 @@ jobs:
244255
echo $PATHS
245256
PATH="venv/bin:$PATH" script/bqetl dryrun --validate-schemas $PATHS
246257
# yamllint enable rule:line-length
258+
- *persist_dryrun_cache
247259
- *copy_debug_sql
248260
- *store_debug_artifacts
249261
- unless:
@@ -262,7 +274,7 @@ jobs:
262274
- checkout
263275
- *restore_venv_cache
264276
- *build
265-
- *attach_generated_sql
277+
- *attach_workspace_artifacts
266278
- &copy_generated_sql
267279
run:
268280
name: Move generated-sql into place
@@ -293,7 +305,7 @@ jobs:
293305
name: Validate workgroup access configuration on main
294306
command: |
295307
PATH="venv/bin:$PATH" script/bqetl metadata validate-workgroups sql/
296-
- *attach_generated_sql
308+
- *attach_workspace_artifacts
297309
- *copy_staged_sql
298310
- *authenticate
299311
- run:
@@ -341,7 +353,7 @@ jobs:
341353
- checkout
342354
- *restore_venv_cache
343355
- *build
344-
- *attach_generated_sql
356+
- *attach_workspace_artifacts
345357
- *authenticate
346358
- &add_private_bigquery_etl_ssh_keys
347359
add_ssh_keys:
@@ -391,7 +403,7 @@ jobs:
391403
name: Pull telemetry-airflow
392404
command: |
393405
git clone https://github.com/mozilla/telemetry-airflow.git ~/telemetry-airflow
394-
- *attach_generated_sql
406+
- *attach_workspace_artifacts
395407
- *copy_generated_sql
396408
- run:
397409
name: Replace telemetry-airflow DAGs with BigQuery ETL DAGs
@@ -441,7 +453,7 @@ jobs:
441453
- checkout
442454
- *restore_venv_cache
443455
- *build
444-
- *attach_generated_sql
456+
- *attach_workspace_artifacts
445457
- *copy_staged_sql
446458
- run:
447459
name: Run routine tests
@@ -468,9 +480,11 @@ jobs:
468480
- checkout
469481
- *restore_venv_cache
470482
- *build
471-
- *attach_generated_sql
483+
- *attach_workspace_artifacts
472484
- *copy_staged_sql
473485
- *authenticate
486+
- *attach_dryrun_cache
487+
- *restore_dryrun_cache_from_workspace
474488
- run:
475489
name: Validate views
476490
command: |
@@ -491,7 +505,7 @@ jobs:
491505
- checkout
492506
- *restore_venv_cache
493507
- *build
494-
- *attach_generated_sql
508+
- *attach_workspace_artifacts
495509
- add_ssh_keys:
496510
fingerprints:
497511
- "22:b9:3c:1b:82:ab:3f:e4:b5:79:70:d1:7b:b9:28:d2"
@@ -520,6 +534,7 @@ jobs:
520534
- *restore_venv_cache
521535
- *build
522536
- *authenticate
537+
- *attach_workspace_artifacts
523538
- run:
524539
name: Generate SQL content
525540
command: |
@@ -614,6 +629,7 @@ jobs:
614629
root: /tmp/workspace
615630
paths:
616631
- generated-sql
632+
- dryrun_cache
617633
- unless:
618634
condition: *validate-sql-or-routines
619635
steps:
@@ -628,7 +644,7 @@ jobs:
628644
- checkout
629645
- *restore_venv_cache
630646
- *build
631-
- *attach_generated_sql
647+
- *attach_workspace_artifacts
632648
- *copy_generated_sql
633649
- add_ssh_keys:
634650
fingerprints:
@@ -664,6 +680,7 @@ jobs:
664680
root: /tmp/workspace
665681
paths:
666682
- staged-generated-sql
683+
- dryrun_cache
667684
- *copy_debug_sql
668685
- *store_debug_artifacts
669686
- unless:
@@ -678,7 +695,7 @@ jobs:
678695
or:
679696
- << pipeline.parameters.deploy >>
680697
steps:
681-
- *attach_generated_sql
698+
- *attach_workspace_artifacts
682699
- add_ssh_keys:
683700
fingerprints:
684701
- "22:b9:3c:1b:82:ab:3f:e4:b5:79:70:d1:7b:b9:28:d2"
@@ -790,7 +807,7 @@ jobs:
790807
- checkout
791808
- *restore_venv_cache
792809
- *build
793-
- *attach_generated_sql
810+
- *attach_workspace_artifacts
794811
- *authenticate
795812
- *add_private_bigquery_etl_ssh_keys
796813
- run:
@@ -824,6 +841,7 @@ jobs:
824841
paths:
825842
- private-generated-sql
826843
- PRIVATE_BIGQUERY_ETL_SHA
844+
- dryrun_cache
827845
- unless:
828846
condition: *validate-sql
829847
steps:
@@ -834,7 +852,7 @@ jobs:
834852
- when:
835853
condition: *deploy
836854
steps:
837-
- *attach_generated_sql
855+
- *attach_workspace_artifacts
838856
- *add_private_bigquery_etl_ssh_keys
839857
- run:
840858
name: Push to private-generated-sql branch
@@ -871,7 +889,7 @@ jobs:
871889
condition: *deploy
872890
steps:
873891
- checkout
874-
- *attach_generated_sql
892+
- *attach_workspace_artifacts
875893
- run:
876894
name: Move generated-sql into place
877895
command: |

‎bigquery_etl/cli/dryrun.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,33 @@
6464
help="GCP project to perform dry run in when --use_cloud_function=False",
6565
default=ConfigLoader.get("default", "project", fallback="moz-fx-data-shared-prod"),
6666
)
67+
@click.option(
68+
"--use-cache/--no-cache",
69+
help="Enable or disable local caching of dry run results. Default is --no-cache.",
70+
default=ConfigLoader.get("dry_run", "cache_enabled", fallback=False),
71+
)
72+
@click.option(
73+
"--cache-ttl-hours",
74+
help="Cache time-to-live in hours. Default is 1 hour.",
75+
type=int,
76+
default=ConfigLoader.get("dry_run", "cache_ttl_hours", fallback=1),
77+
)
78+
@click.option(
79+
"--cache-dir",
80+
help="Directory to store cache files. Default is system temp directory.",
81+
type=click.Path(),
82+
default=None,
83+
)
6784
@billing_project_option()
6885
def dryrun(
6986
paths: List[str],
7087
use_cloud_function: bool,
7188
validate_schemas: bool,
7289
respect_skip: bool,
7390
project: str,
91+
use_cache: bool,
92+
cache_ttl_hours: int,
93+
cache_dir: str,
7494
billing_project: str,
7595
):
7696
"""Perform a dry run."""
@@ -118,6 +138,9 @@ def dryrun(
118138
use_cloud_function,
119139
respect_skip,
120140
validate_schemas,
141+
use_cache,
142+
cache_ttl_hours,
143+
cache_dir,
121144
credentials=credentials,
122145
id_token=id_token,
123146
billing_project=billing_project,
@@ -141,6 +164,9 @@ def _sql_file_valid(
141164
use_cloud_function,
142165
respect_skip,
143166
validate_schemas,
167+
use_cache,
168+
cache_ttl_hours,
169+
cache_dir,
144170
sqlfile,
145171
credentials,
146172
id_token,
@@ -154,6 +180,9 @@ def _sql_file_valid(
154180
respect_skip=respect_skip,
155181
id_token=id_token,
156182
billing_project=billing_project,
183+
cache_enabled=use_cache,
184+
cache_ttl_hours=cache_ttl_hours,
185+
cache_dir=cache_dir,
157186
)
158187
if validate_schemas:
159188
try:

0 commit comments

Comments
 (0)