From aecddd4eaa2fa4a1f43fbb0a4bd6a9288962f671 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 5 Mar 2024 14:44:25 +1100 Subject: [PATCH 1/2] Move extracted tarballs to a completed directory --- scripts/untar_gz_files.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/untar_gz_files.py b/scripts/untar_gz_files.py index 034d471b..85c40cad 100644 --- a/scripts/untar_gz_files.py +++ b/scripts/untar_gz_files.py @@ -76,6 +76,16 @@ def main(bucket: str, subdir: str, blob_name: str, outdir: str): subprocess.run(['rm', f'{filepath}'], check=True) logging.info(f'Deleted {file} from disk') + # Move the tarball to the "completed" directory + subprocess.run( + [ + 'gsutil', + 'mv', + f'gs://{bucket}/{subdir}/{os.path.basename(blob_name)}', + f'gs://{bucket}/{subdir}/completed/{os.path.basename(blob_name)}' + ], + check=True + ) logging.info('All tarballs extracted and uploaded. Finishing...') From 8edcfd09e801af1a9443dd1e545f8304ea517d58 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Mon, 11 Mar 2024 16:29:07 +1100 Subject: [PATCH 2/2] Linting --- scripts/untar_gz_files.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/untar_gz_files.py b/scripts/untar_gz_files.py index 85c40cad..e41a001c 100644 --- a/scripts/untar_gz_files.py +++ b/scripts/untar_gz_files.py @@ -79,12 +79,12 @@ def main(bucket: str, subdir: str, blob_name: str, outdir: str): # Move the tarball to the "completed" directory subprocess.run( [ - 'gsutil', - 'mv', - f'gs://{bucket}/{subdir}/{os.path.basename(blob_name)}', - f'gs://{bucket}/{subdir}/completed/{os.path.basename(blob_name)}' - ], - check=True + 'gsutil', + 'mv', + f'gs://{bucket}/{subdir}/{os.path.basename(blob_name)}', + f'gs://{bucket}/{subdir}/completed/{os.path.basename(blob_name)}', + ], + check=True, ) logging.info('All tarballs extracted and uploaded. Finishing...')