From 529c1bcfe249c241defecbf26450043846d2fdcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Elliott=20B=C3=A9cue?= Date: Tue, 18 Jul 2023 16:46:32 +0200 Subject: [PATCH 1/2] Implement chunk size argument and refactor mbuffer command generation Fixes #203 --- zfs_autobackup/ZfsAutobackup.py | 36 +++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 8b33466..010d51e 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -119,6 +119,8 @@ def get_parser(self): help='Limit data transfer rate in Bytes/sec (e.g. 128K. requires mbuffer.)') group.add_argument('--buffer', metavar='SIZE', default=None, help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)') + parser.add_argument('--chunk-size', metavar="CHUNKSIZE", default=None, + help='Tune chunk size when mbuffer is used. (requires mbuffer.)') group.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append', help='pipe zfs send output through COMMAND (can be used multiple times)') group.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append', @@ -234,11 +236,22 @@ def get_send_pipes(self, logger): """determine the zfs send pipe""" ret = [] + _mbuffer = False + _buffer = "16M" + _cs = "128k" + _rate = False # IO buffer if self.args.buffer: logger("zfs send buffer : {}".format(self.args.buffer)) - ret.extend([ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m" + self.args.buffer]) + _mbuffer = True + _buffer = self.args.buffer + + # IO chunk size + if self.args.chunk_size: + logger("zfs send chunk size : {}".format(self.args.chunk_size)) + _mbuffer = True + _cs = self.args.chunk_size # custom pipes for send_pipe in self.args.send_pipe: @@ -256,7 +269,14 @@ def get_send_pipes(self, logger): # transfer rate if self.args.rate: logger("zfs send transfer rate : {}".format(self.args.rate)) - ret.extend([ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m16M", "-R" + self.args.rate]) + _mbuffer = True + _rate = self.args.rate + + if _mbuffer: + cmd = [ExecuteNode.PIPE, "mbuffer", "-q", "-s{}".format(_cs), "-m{}".format(_buffer)] + if _rate: + cmd.append("-R{}".format(self.args.rate)) + ret.extend(cmd) return ret @@ -278,11 +298,19 @@ def get_recv_pipes(self, logger): logger("zfs recv custom pipe : {}".format(recv_pipe)) # IO buffer - if self.args.buffer: + if self.args.buffer or self.args.chunk_size: + _cs = "128k" + _buffer = "16M" # only add second buffer if its usefull. (e.g. non local transfer or other pipes active) if self.args.ssh_source != None or self.args.ssh_target != None or self.args.recv_pipe or self.args.send_pipe or self.args.compress != None: logger("zfs recv buffer : {}".format(self.args.buffer)) - ret.extend(["mbuffer", "-q", "-s128k", "-m" + self.args.buffer, ExecuteNode.PIPE]) + + if self.args.chunk_size: + _cs = self.args.chunk_size + if self.args.buffer: + _buffer = self.args.buffer + + ret.extend(["mbuffer", "-q", "-s{}".format(_cs), "-m{}".format(_buffer), ExecuteNode.PIPE]) return ret From c458f82e0fdbdbc06b2d796f256c9edea59015a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Elliott=20B=C3=A9cue?= Date: Wed, 4 Oct 2023 00:02:59 +0200 Subject: [PATCH 2/2] Rename chunk-size to buffer-chunk-size --- zfs_autobackup/ZfsAutobackup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 010d51e..2d53327 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -119,7 +119,7 @@ def get_parser(self): help='Limit data transfer rate in Bytes/sec (e.g. 128K. requires mbuffer.)') group.add_argument('--buffer', metavar='SIZE', default=None, help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)') - parser.add_argument('--chunk-size', metavar="CHUNKSIZE", default=None, + parser.add_argument('--buffer-chunk-size', metavar="BUFFERCHUNKSIZE", default=None, help='Tune chunk size when mbuffer is used. (requires mbuffer.)') group.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append', help='pipe zfs send output through COMMAND (can be used multiple times)') @@ -248,10 +248,10 @@ def get_send_pipes(self, logger): _buffer = self.args.buffer # IO chunk size - if self.args.chunk_size: - logger("zfs send chunk size : {}".format(self.args.chunk_size)) + if self.args.buffer_chunk_size: + logger("zfs send chunk size : {}".format(self.args.buffer_chunk_size)) _mbuffer = True - _cs = self.args.chunk_size + _cs = self.args.buffer_chunk_size # custom pipes for send_pipe in self.args.send_pipe: @@ -298,15 +298,15 @@ def get_recv_pipes(self, logger): logger("zfs recv custom pipe : {}".format(recv_pipe)) # IO buffer - if self.args.buffer or self.args.chunk_size: + if self.args.buffer or self.args.buffer_chunk_size: _cs = "128k" _buffer = "16M" # only add second buffer if its usefull. (e.g. non local transfer or other pipes active) if self.args.ssh_source != None or self.args.ssh_target != None or self.args.recv_pipe or self.args.send_pipe or self.args.compress != None: logger("zfs recv buffer : {}".format(self.args.buffer)) - if self.args.chunk_size: - _cs = self.args.chunk_size + if self.args.buffer_chunk_size: + _cs = self.args.buffer_chunk_size if self.args.buffer: _buffer = self.args.buffer