From b6c2fab536872e8d991524a89288e2fca2dbb3c4 Mon Sep 17 00:00:00 2001 From: Yiming Yang Date: Sun, 5 Sep 2021 16:19:55 -0400 Subject: [PATCH 1/2] Use ionice for uploading/downloading when available --- strato/backends/_aws.py | 8 +++++--- strato/backends/_gcp.py | 7 +++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/strato/backends/_aws.py b/strato/backends/_aws.py index 3ffd96f..a519323 100644 --- a/strato/backends/_aws.py +++ b/strato/backends/_aws.py @@ -1,4 +1,4 @@ -import os +import os, shutil from subprocess import check_call @@ -8,7 +8,8 @@ def __init__(self): self._call_prefix = ['aws', 's3'] def copy(self, filenames): - call_args = self._call_prefix.copy() + call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + call_args += self._call_prefix call_args.extend(['cp', '--only-show-errors']) source_files = filenames[:-1] @@ -31,7 +32,8 @@ def copy(self, filenames): check_call(subcall_args) def sync(self, source, target): - call_args = self._call_prefix.copy() + call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + call_args += self._call_prefix call_args.extend(['sync', '--delete', '--only-show-errors', source, target]) print(' '.join(call_args)) check_call(call_args) diff --git a/strato/backends/_gcp.py b/strato/backends/_gcp.py index 627aa8d..acd3453 100644 --- a/strato/backends/_gcp.py +++ b/strato/backends/_gcp.py @@ -1,3 +1,4 @@ +import shutil from subprocess import check_call @@ -7,7 +8,8 @@ def __init__(self): self._call_prefix = ['gsutil', '-q', '-o', 'GSUtil:parallel_composite_upload_threshold=150M'] def copy(self, recursive, parallel, filenames): - call_args = self._call_prefix.copy() + call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + call_args += self._call_prefix if parallel: call_args.append('-m') call_args.append('cp') @@ -18,7 +20,8 @@ def copy(self, recursive, parallel, filenames): check_call(call_args) def sync(self, parallel, source, target): - call_args = self._call_prefix.copy() + call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + call_args += self._call_prefix if parallel: call_args.append('-m') call_args.extend(['rsync', '-d', '-r', source, target]) From 32aee53f950450a4e68fee9b5cd198a0f46d9fcf Mon Sep 17 00:00:00 2001 From: Yiming Yang Date: Sun, 5 Sep 2021 16:33:44 -0400 Subject: [PATCH 2/2] Make ionice an option of cp and sync commands --- strato/backends/_aws.py | 8 ++++---- strato/backends/_gcp.py | 8 ++++---- strato/backends/_local.py | 12 +++++++----- strato/commands/cp.py | 11 ++++++----- strato/commands/sync.py | 11 ++++++----- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/strato/backends/_aws.py b/strato/backends/_aws.py index a519323..f42f66f 100644 --- a/strato/backends/_aws.py +++ b/strato/backends/_aws.py @@ -7,8 +7,8 @@ def __init__(self): self._backend = 'aws' self._call_prefix = ['aws', 's3'] - def copy(self, filenames): - call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + def copy(self, ionice, filenames): + call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else [] call_args += self._call_prefix call_args.extend(['cp', '--only-show-errors']) @@ -31,8 +31,8 @@ def copy(self, filenames): print(' '.join(subcall_args)) check_call(subcall_args) - def sync(self, source, target): - call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + def sync(self, ionice, source, target): + call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else [] call_args += self._call_prefix call_args.extend(['sync', '--delete', '--only-show-errors', source, target]) print(' '.join(call_args)) diff --git a/strato/backends/_gcp.py b/strato/backends/_gcp.py index acd3453..5f7f5c9 100644 --- a/strato/backends/_gcp.py +++ b/strato/backends/_gcp.py @@ -7,8 +7,8 @@ def __init__(self): self._backend = 'gcp' self._call_prefix = ['gsutil', '-q', '-o', 'GSUtil:parallel_composite_upload_threshold=150M'] - def copy(self, recursive, parallel, filenames): - call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + def copy(self, recursive, parallel, ionice, filenames): + call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else [] call_args += self._call_prefix if parallel: call_args.append('-m') @@ -19,8 +19,8 @@ def copy(self, recursive, parallel, filenames): print(' '.join(call_args)) check_call(call_args) - def sync(self, parallel, source, target): - call_args = ['ionice', '-c', '2', '-n', '7'] if shutil.which('ionice') != None else [] + def sync(self, parallel, ionice, source, target): + call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else [] call_args += self._call_prefix if parallel: call_args.append('-m') diff --git a/strato/backends/_local.py b/strato/backends/_local.py index 76537fd..80f55c8 100644 --- a/strato/backends/_local.py +++ b/strato/backends/_local.py @@ -1,27 +1,29 @@ -import os +import os, shutil from subprocess import check_call class LocalBackend: def __init__(self): self._backend = 'local' - def copy(self, recursive, filenames): + def copy(self, recursive, ionice, filenames): assert len(filenames) >= 2, "Either source or destination is missing!" target = filenames[-1] call_args1 = ['mkdir', '-p', target] print(' '.join(call_args1)) check_call(call_args1) - call_args = ['cp'] + call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice')) != None else [] + call_args += ['cp'] if recursive: call_args.append('-r') call_args.extend(filenames) print(' '.join(call_args)) check_call(call_args) - def sync(self, source, target): + def sync(self, ionice, source, target): target = os.path.dirname(target) - call_args = ['rsync', '-r', '--delete', source, target] + call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice')) != None else [] + call_args += ['rsync', '-r', '--delete', source, target] print(' '.join(call_args)) check_call(call_args) diff --git a/strato/commands/cp.py b/strato/commands/cp.py index 83b58bb..926036c 100644 --- a/strato/commands/cp.py +++ b/strato/commands/cp.py @@ -1,21 +1,21 @@ import argparse -def copy_files(backend, recursive, parallel, filenames): +def copy_files(backend, recursive, parallel, ionice, filenames): assert backend in ['aws', 'gcp', 'local'], "Backend not supported!" if backend == 'aws': from strato.backends import AWSBackend be = AWSBackend() - be.copy(filenames) + be.copy(ionice, filenames) elif backend == 'gcp': from strato.backends import GCPBackend be = GCPBackend() - be.copy(recursive, parallel, filenames) + be.copy(recursive, parallel, ionice, filenames) else: from strato.backends import LocalBackend be = LocalBackend() - be.copy(recursive, filenames) + be.copy(recursive, ionice, filenames) def main(argsv): @@ -23,7 +23,8 @@ def main(argsv): parser.add_argument('--backend', dest='backend', action='store', required=True, help='Specify which backend to use. Available options: aws, gcp, local.') parser.add_argument('-r', dest='recursive', action='store_true', help="Recursive copy. Not needed for AWS backend.") parser.add_argument('-m', dest='parallel', action='store_true', help="Run operations in parallel. Only available for GCP backend.") + parser.add_argument('--ionice', dest='ionice', action='store_true', help="Run with ionice to avoid monopolizing local disk's I/O. Only available for Linux.") parser.add_argument('files', metavar='filenames', type=str, nargs='+', help='List of file paths.') args = parser.parse_args(argsv) - copy_files(args.backend, args.recursive, args.parallel, args.files) + copy_files(args.backend, args.recursive, args.parallel, args.ionice, args.files) diff --git a/strato/commands/sync.py b/strato/commands/sync.py index f87f423..4170eec 100644 --- a/strato/commands/sync.py +++ b/strato/commands/sync.py @@ -1,28 +1,29 @@ import argparse -def synchronize_folders(backend, parallel, source, target): +def synchronize_folders(backend, parallel, ionice, source, target): assert backend in ['aws', 'gcp', 'local'], "Backend not supported!" if backend == 'aws': from strato.backends import AWSBackend be = AWSBackend() - be.sync(source, target) + be.sync(ionice, source, target) elif backend == 'gcp': from strato.backends import GCPBackend be = GCPBackend() - be.sync(parallel, source, target) + be.sync(parallel, ionice, source, target) else: from strato.backends import LocalBackend be = LocalBackend() - be.sync(source, target) + be.sync(ionice, source, target) def main(argsv): parser = argparse.ArgumentParser(description="Synchronize source and target folders.") parser.add_argument('--backend', dest='backend', action='store', required=True, help='Specify which backend to use. Available options: aws, gcp, local.') parser.add_argument('-m', dest='parallel', action='store_true', help="Run operations in parallel. Only available for GCP backend.") + parser.add_argument('--ionice', dest='ionice', action='store_true', help="Run with ionice to avoid monopolizing local disk's I/O. Only available for Linux.") parser.add_argument('source', metavar='source', type=str, help='Source folder path.') parser.add_argument('target', metavar='target', type=str, help='Target folder path.') args = parser.parse_args(argsv) - synchronize_folders(args.backend, args.parallel, args.source, args.target) + synchronize_folders(args.backend, args.parallel, args.ionice, args.source, args.target)