add multipart-chunksize parameter

thelastpickle · Jul 25, 2024 · dc82837 · dc82837
1 parent 969e34f
commit dc82837
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/medusa/config.py b/medusa/config.py
@@ -30,8 +30,9 @@
     'StorageConfig',
     ['bucket_name', 'key_file', 'prefix', 'fqdn', 'host_file_separator', 'storage_provider', 'storage_class',
      'base_path', 'max_backup_age', 'max_backup_count', 'api_profile', 'transfer_max_bandwidth',
-     'concurrent_transfers', 'multi_part_upload_threshold', 'host', 'region', 'port', 'secure', 'ssl_verify',
-     'aws_cli_path', 'kms_id', 'backup_grace_period_in_days', 'use_sudo_for_restore', 'k8s_mode', 'read_timeout']
+     'concurrent_transfers', 'multi_part_upload_threshold', 'multipart_chunksize', 'host', 'region', 'port',
+     'secure', 'ssl_verify', 'aws_cli_path', 'kms_id', 'backup_grace_period_in_days', 'use_sudo_for_restore',
+     'k8s_mode', 'read_timeout']
 )
 
 CassandraConfig = collections.namedtuple(

diff --git a/medusa/storage/s3_base_storage.py b/medusa/storage/s3_base_storage.py
@@ -185,6 +185,7 @@ def _make_connection_arguments(self, config) -> t.Dict[str, str]:
     def _make_transfer_config(self, config):
 
         transfer_max_bandwidth = config.transfer_max_bandwidth or None
+        multipart_chunksize = config.multipart_chunksize or None
 
         # we hard-code this one because the parallelism is for now applied to chunking the files
         transfer_config = {
@@ -194,6 +195,9 @@ def _make_transfer_config(self, config):
         if transfer_max_bandwidth is not None:
             transfer_config['max_bandwidth'] = AbstractStorage._human_size_to_bytes(transfer_max_bandwidth)
 
+        if multipart_chunksize is not None:
+            transfer_config['multipart_chunksize'] = AbstractStorage._human_size_to_bytes(multipart_chunksize)
+
         return TransferConfig(**transfer_config)
 
     @staticmethod