From 35b079a9a949ba4cd59dbf5014c3b3f6677199a2 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 10:52:23 +0200 Subject: [PATCH] WIP: less hackish caching --- zfs_autobackup/ZfsDataset.py | 132 +++++++++++++++++++++++------------ zfs_autobackup/ZfsNode.py | 7 +- 2 files changed, 93 insertions(+), 46 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index afdfd37..97df5f5 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,9 +1,9 @@ + import re from datetime import datetime import sys import time -from .CachedProperty import CachedProperty from .ExecuteNode import ExecuteError @@ -28,9 +28,30 @@ def __init__(self, zfs_node, name, force_exists=None): """ self.zfs_node = zfs_node self.name = name # full name + + #caching + self.__snapshots=None #type: None|list[ZfsDataset] + self.__written_since_ours=None #type: None|int + self.__exists_check=None #type: None|bool + self.__properties=None #type: None|dict[str,str] + self.__recursive_datasets=None #type: None|list[ZfsDataset] + self.__datasets=None #type: None|list[ZfsDataset] + self.invalidate() self.force_exists = force_exists + + def invalidate(self): + """clear caches""" + # CachedProperty.clear(self) + self.force_exists = None + self.__snapshots=None + self.__written_since_ours=None + self.__exists_check=None + self.__properties=None + self.__recursive_datasets=None + self.__datasets=None + def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -72,10 +93,6 @@ def debug(self, txt): """ self.zfs_node.debug("{}: {}".format(self.name, txt)) - def invalidate(self): - """clear caches""" - CachedProperty.clear(self) - self.force_exists = None def split_path(self): """return the path elements as an array""" @@ -269,13 +286,17 @@ def find_next_snapshot(self, snapshot, also_other_snapshots=False): return self.snapshots[index] return None - @CachedProperty + @property def exists_check(self): """check on disk if it exists""" - self.debug("Checking if dataset exists") - return (len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, - valid_exitcodes=[0, 1], - hide_errors=True)) > 0) + + if self.__exists_check is None: + self.debug("Checking if dataset exists") + self.__exists_check=(len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, + valid_exitcodes=[0, 1], + hide_errors=True)) > 0) + + return self.__exists_check @property def exists(self): @@ -344,22 +365,24 @@ def destroy(self, fail_exception=False, deferred=False, verbose=True): else: raise - @CachedProperty + @property def properties(self): """all zfs properties""" - cmd = [ - "zfs", "get", "-H", "-o", "property,value", "-p", "all", self.name - ] + if self.__properties is None: - self.debug("Getting zfs properties") + cmd = [ + "zfs", "get", "-H", "-o", "property,value", "-p", "all", self.name + ] - ret = {} - for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): - if len(pair) == 2: - ret[pair[0]] = pair[1] + self.debug("Getting zfs properties") - return ret + self.__properties = {} + for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): + if len(pair) == 2: + self.__properties[pair[0]] = pair[1] + + return self.__properties def is_changed(self, min_changed_bytes=1): """dataset is changed since ANY latest snapshot ? @@ -447,24 +470,32 @@ def timestamp(self): # print ("ADDED VIRT") # self._virtual_snapshots.append(snapshot) - @CachedProperty + @property def snapshots(self): """get all snapshots of this dataset :rtype: list[ZfsDataset] """ - self.debug("Getting snapshots") + #cached? + if self.__snapshots is None: - cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name - ] - return self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + self.debug("Getting snapshots") + + cmd = [ + "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name + ] + + self.__snapshots=self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + + + return self.__snapshots @property def our_snapshots(self): """get list[snapshots] creates by us of this dataset""" ret = [] + for snapshot in self.snapshots: if snapshot.is_ours(): ret.append(snapshot) @@ -525,18 +556,22 @@ def find_snapshot_index(self, snapshot): return None - @CachedProperty + @property def written_since_ours(self): """get number of bytes written since our last snapshot""" - latest_snapshot = self.our_snapshots[-1] + if self.__written_since_ours is None: + + latest_snapshot = self.our_snapshots[-1] + + self.debug("Getting bytes written since our last snapshot") + cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] - self.debug("Getting bytes written since our last snapshot") - cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] + output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) - output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) + self.__written_since_ours=int(output[0]) - return int(output[0]) + return self.__written_since_ours def is_changed_ours(self, min_changed_bytes=1): """dataset is changed since OUR latest snapshot? @@ -557,23 +592,28 @@ def is_changed_ours(self, min_changed_bytes=1): return True - @CachedProperty + @property def recursive_datasets(self, types="filesystem,volume"): """get all (non-snapshot) datasets recursively under us Args: :type types: str + :rtype: list[ZfsDataset] """ - self.debug("Getting all recursive datasets under us") + if self.__recursive_datasets is None: + + self.debug("Getting all recursive datasets under us") - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name - ]) + names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name + ]) - return self.zfs_node.get_datasets(names[1:], force_exists=True) + self.__recursive_datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) - @CachedProperty + return self.__recursive_datasets + + @property def datasets(self, types="filesystem,volume"): """get all (non-snapshot) datasets directly under us @@ -581,13 +621,17 @@ def datasets(self, types="filesystem,volume"): :type types: str """ - self.debug("Getting all datasets under us") + if self.__datasets is None: + + self.debug("Getting all datasets under us") + + names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name + ]) - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name - ]) + self.__datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) - return self.zfs_node.get_datasets(names[1:], force_exists=True) + return self.__datasets def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, send_pipes, zfs_compressed): diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index d1ebd7f..b94651b 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -209,6 +209,8 @@ def debug(self, txt): def consistent_snapshot(self, datasets, snapshot_name, min_changed_bytes, pre_snapshot_cmds=[], post_snapshot_cmds=[], set_snapshot_properties=[]): """create a consistent (atomic) snapshot of specified datasets, per pool. + Args: + :type datasets: list[ZfsDataset] """ pools = {} @@ -230,8 +232,9 @@ def consistent_snapshot(self, datasets, snapshot_name, min_changed_bytes, pre_sn pools[pool].append(snapshot) # update cache, but try to prevent an unneeded zfs list - if self.readonly or CachedProperty.is_cached(dataset, 'snapshots'): - dataset.snapshots.append(snapshot) # NOTE: this will trigger zfs list if its not cached + # if self.readonly or CachedProperty.is_cached(dataset, 'snapshots'): + #dataset.snapshots.append(snapshot) # NOTE: this will trigger zfs list if its not cached + dataset.invalidate() if not pools: self.verbose("No changes anywhere: not creating snapshots.")