Ensure done files cleared out in case of retry

georgetown-cset · Jan 23, 2024 · ec804ef · ec804ef
1 parent a4436c8
commit ec804ef
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/utils/run_ids_scripts.sh b/utils/run_ids_scripts.sh
@@ -1,4 +1,5 @@
 cd /mnt/disks/data/run
+gsutil rm gs://airflow-data-exchange/article_linkage/tmp/done_files/ids_are_done
 python3 create_merge_ids.py --match_dir usable_ids --prev_id_mapping_dir prev_id_mapping --merge_file id_mapping.jsonl --current_ids_dir article_pairs
 /snap/bin/gsutil -m cp id_mapping.jsonl gs://airflow-data-exchange/article_linkage/tmp/
 /snap/bin/gsutil -m cp simhash_results/* gs://airflow-data-exchange/article_linkage/simhash_results/

diff --git a/utils/run_simhash_scripts.sh b/utils/run_simhash_scripts.sh
@@ -1,4 +1,5 @@
 cd /mnt/disks/data/run
+gsutil rm gs://airflow-data-exchange/article_linkage/tmp/done_files/simhash_is_done
 python3 run_simhash.py simhash_input simhash_results --simhash_indexes simhash_indexes --new_simhash_indexes new_simhash_indexes
 cp -r article_pairs usable_ids
 cp simhash_results/* article_pairs/