diff --git a/changedetectionio/store.py b/changedetectionio/store.py index c377255716c..38fbeaffad1 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -1,3 +1,5 @@ +import pathlib + from changedetectionio.strtobool import strtobool from flask import ( @@ -61,14 +63,28 @@ def __init__(self, datastore_path="/datastore", include_default_watches=True, ve try: # @todo retest with ", encoding='utf-8'" + + logger.debug(f"Scanning for watch.json in data-directory") + self.__data['watching'] = {} + for item in pathlib.Path(datastore_path).glob("*/watch.json"): + try: + logger.trace(f"Found {item}") + with open(item, 'r') as f: + loaded = json.load(f) + if not loaded.get('uuid'): + logger.error(f"Unable to find UUID in {item}") + self.__data['watching'][loaded.get('uuid')] = loaded + except Exception as e: + logger.critical(f"Error reading watch config file {item} - {str(e)}") + + # Convert each existing watch back to the Watch.model object + for uuid, watch in self.__data['watching'].items(): + self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch) + logger.info(f"Watching: {uuid} {watch['url']}") + with open(self.json_store_path) as json_file: from_disk = json.load(json_file) - # @todo isnt there a way todo this dict.update recursively? - # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore. - if 'watching' in from_disk: - self.__data['watching'].update(from_disk['watching']) - if 'app_guid' in from_disk: self.__data['app_guid'] = from_disk['app_guid'] @@ -82,11 +98,6 @@ def __init__(self, datastore_path="/datastore", include_default_watches=True, ve if 'application' in from_disk['settings']: self.__data['settings']['application'].update(from_disk['settings']['application']) - # Convert each existing watch back to the Watch.model object - for uuid, watch in self.__data['watching'].items(): - self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch) - logger.info(f"Watching: {uuid} {watch['url']}") - # And for Tags also, should be Restock type because it has extra settings for uuid, tag in self.__data['settings']['application']['tags'].items(): self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff') @@ -382,25 +393,41 @@ def visualselector_data_is_ready(self, watch_uuid): def sync_to_json(self): logger.info("Saving JSON..") try: - data = deepcopy(self.__data) + config_data = deepcopy(self.__data) + watch_data = config_data.pop('watching') + except RuntimeError as e: # Try again in 15 seconds - time.sleep(15) + time.sleep(5) logger.error(f"! Data changed when writing to JSON, trying again.. {str(e)}") self.sync_to_json() return else: - + # Write the main config file for general settings try: # Re #286 - First write to a temp file, then confirm it looks OK and rename it # This is a fairly basic strategy to deal with the case that the file is corrupted, # system was out of memory, out of RAM etc with open(self.json_store_path+".tmp", 'w') as json_file: - json.dump(data, json_file, indent=4) + json.dump(config_data, json_file, indent=4) os.replace(self.json_store_path+".tmp", self.json_store_path) except Exception as e: logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}") + for uuid, watch in watch_data.items(): + watch_config_path = os.path.join(self.datastore_path, uuid, "watch.json") + # Write the main config file for general settings + try: + # Re #286 - First write to a temp file, then confirm it looks OK and rename it + # This is a fairly basic strategy to deal with the case that the file is corrupted, + # system was out of memory, out of RAM etc + with open(watch_config_path + ".tmp", 'w') as json_file: + json.dump(watch, json_file, indent=4) + os.replace(watch_config_path + ".tmp", watch_config_path) + except Exception as e: + logger.error(f"Error writing watch config {uuid} to {watch_config_path} JSON!! {str(e)}") + + self.needs_write = False self.needs_write_urgent = False