Merge pull request #3 from lilab-bcb/develop

Add `--ionice` option

strato/backends/_aws.py

@@ -1,4 +1,4 @@
-import os
+import os, shutil
 from subprocess import check_call
 
 
@@ -7,8 +7,9 @@ def __init__(self):
         self._backend = 'aws'
         self._call_prefix = ['aws', 's3']
 
-    def copy(self, filenames):
-        call_args = self._call_prefix.copy()
+    def copy(self, ionice, filenames):
+        call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else []
+        call_args += self._call_prefix
         call_args.extend(['cp', '--only-show-errors'])
 
         source_files = filenames[:-1]
@@ -30,8 +31,9 @@ def copy(self, filenames):
             print(' '.join(subcall_args))
             check_call(subcall_args)
 
-    def sync(self, source, target):
-        call_args = self._call_prefix.copy()
+    def sync(self, ionice, source, target):
+        call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else []
+        call_args += self._call_prefix
         call_args.extend(['sync', '--delete', '--only-show-errors', source, target])
         print(' '.join(call_args))
         check_call(call_args)

strato/backends/_gcp.py

@@ -1,3 +1,4 @@
+import shutil
 from subprocess import check_call
 
 
@@ -6,8 +7,9 @@ def __init__(self):
         self._backend = 'gcp'
         self._call_prefix = ['gsutil', '-q', '-o', 'GSUtil:parallel_composite_upload_threshold=150M']
 
-    def copy(self, recursive, parallel, filenames):
-        call_args = self._call_prefix.copy()
+    def copy(self, recursive, parallel, ionice, filenames):
+        call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else []
+        call_args += self._call_prefix
         if parallel:
             call_args.append('-m')
         call_args.append('cp')
@@ -17,8 +19,9 @@ def copy(self, recursive, parallel, filenames):
         print(' '.join(call_args))
         check_call(call_args)
 
-    def sync(self, parallel, source, target):
-        call_args = self._call_prefix.copy()
+    def sync(self, parallel, ionice, source, target):
+        call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice') != None) else []
+        call_args += self._call_prefix
         if parallel:
             call_args.append('-m')
         call_args.extend(['rsync', '-d', '-r', source, target])

strato/backends/_local.py

@@ -1,27 +1,29 @@
-import os
+import os, shutil
 from subprocess import check_call
 
 class LocalBackend:
     def __init__(self):
         self._backend = 'local'
 
-    def copy(self, recursive, filenames):
+    def copy(self, recursive, ionice, filenames):
         assert len(filenames) >= 2, "Either source or destination is missing!"
         target = filenames[-1]
         call_args1 = ['mkdir', '-p', target]
         print(' '.join(call_args1))
         check_call(call_args1)
 
-        call_args = ['cp']
+        call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice')) != None else []
+        call_args += ['cp']
         if recursive:
             call_args.append('-r')
         call_args.extend(filenames)
         print(' '.join(call_args))
         check_call(call_args)
 
-    def sync(self, source, target):
+    def sync(self, ionice, source, target):
         target = os.path.dirname(target)
-        call_args = ['rsync', '-r', '--delete', source, target]
+        call_args = ['ionice', '-c', '2', '-n', '7'] if ionice and (shutil.which('ionice')) != None else []
+        call_args += ['rsync', '-r', '--delete', source, target]
         print(' '.join(call_args))
         check_call(call_args)
 

strato/commands/cp.py

@@ -1,29 +1,30 @@
 import argparse
 
 
-def copy_files(backend, recursive, parallel, filenames):
+def copy_files(backend, recursive, parallel, ionice, filenames):
     assert backend in ['aws', 'gcp', 'local'], "Backend not supported!"
 
     if backend == 'aws':
         from strato.backends import AWSBackend
         be = AWSBackend()
-        be.copy(filenames)
+        be.copy(ionice, filenames)
     elif backend == 'gcp':
         from strato.backends import GCPBackend
         be = GCPBackend()
-        be.copy(recursive, parallel, filenames)
+        be.copy(recursive, parallel, ionice, filenames)
     else:
         from strato.backends import LocalBackend
         be = LocalBackend()
-        be.copy(recursive, filenames)
+        be.copy(recursive, ionice, filenames)
 
 
 def main(argsv):
     parser = argparse.ArgumentParser(description="Copy files or folders. Notice that for AWS backend, a folder link must end with a slash '/'.")
     parser.add_argument('--backend', dest='backend', action='store', required=True, help='Specify which backend to use. Available options: aws, gcp, local.')
     parser.add_argument('-r', dest='recursive', action='store_true', help="Recursive copy. Not needed for AWS backend.")
     parser.add_argument('-m', dest='parallel', action='store_true', help="Run operations in parallel. Only available for GCP backend.")
+    parser.add_argument('--ionice', dest='ionice', action='store_true', help="Run with ionice to avoid monopolizing local disk's I/O. Only available for Linux.")
     parser.add_argument('files', metavar='filenames', type=str, nargs='+', help='List of file paths.')
 
     args = parser.parse_args(argsv)
-    copy_files(args.backend, args.recursive, args.parallel, args.files)
+    copy_files(args.backend, args.recursive, args.parallel, args.ionice, args.files)

strato/commands/sync.py

@@ -1,28 +1,29 @@
 import argparse
 
 
-def synchronize_folders(backend, parallel, source, target):
+def synchronize_folders(backend, parallel, ionice, source, target):
     assert backend in ['aws', 'gcp', 'local'], "Backend not supported!"
 
     if backend == 'aws':
         from strato.backends import AWSBackend
         be = AWSBackend()
-        be.sync(source, target)
+        be.sync(ionice, source, target)
     elif backend == 'gcp':
         from strato.backends import GCPBackend
         be = GCPBackend()
-        be.sync(parallel, source, target)
+        be.sync(parallel, ionice, source, target)
     else:
         from strato.backends import LocalBackend
         be = LocalBackend()
-        be.sync(source, target)
+        be.sync(ionice, source, target)
 
 def main(argsv):
     parser = argparse.ArgumentParser(description="Synchronize source and target folders.")
     parser.add_argument('--backend', dest='backend', action='store', required=True, help='Specify which backend to use. Available options: aws, gcp, local.')
     parser.add_argument('-m', dest='parallel', action='store_true', help="Run operations in parallel. Only available for GCP backend.")
+    parser.add_argument('--ionice', dest='ionice', action='store_true', help="Run with ionice to avoid monopolizing local disk's I/O. Only available for Linux.")
     parser.add_argument('source', metavar='source', type=str, help='Source folder path.')
     parser.add_argument('target', metavar='target', type=str, help='Target folder path.')
 
     args = parser.parse_args(argsv)
-    synchronize_folders(args.backend, args.parallel, args.source, args.target)
+    synchronize_folders(args.backend, args.parallel, args.ionice, args.source, args.target)