From 43d57155d700623b3085a3c291784921bb97ca1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jason=20=5B=EC=A0=9C=EC=9D=B4=EC=8A=A8=20=EC=A0=9C?= =?UTF-8?q?=EB=A1=AC=5D?= <20238115+DuckBoss@users.noreply.github.com> Date: Sat, 30 May 2020 16:54:28 -0400 Subject: [PATCH 1/2] Added --clear-targets command, disabled auto-deletion --- FileSync/__main__.py | 9 +++++---- FileSync/main.py | 45 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/FileSync/__main__.py b/FileSync/__main__.py index 1377278..559fceb 100644 --- a/FileSync/__main__.py +++ b/FileSync/__main__.py @@ -3,9 +3,9 @@ from os import getcwd, path from sys import exit from pathlib import Path -from FileSync.resources.strings import * -from FileSync.main import FileChecker -from FileSync.setup_utility import setup_settings +from resources.strings import * +from main import FileChecker +from setup_utility import setup_settings if __name__ == "__main__": @@ -26,6 +26,7 @@ parser.add_argument('--username', dest='sftp_user', default='', help='Sets the username for sftp server communication') parser.add_argument('--password', dest='sftp_pass', default='', help='Sets the password for sftp server communication') parser.add_argument('--setup', dest='setup_feature', action='store_true', default=False, help='Initializes setup mode which provides an interactive settings.ini creation utility') + parser.add_argument('--clear-targets', dest='clear_on_start', action='store_true', default=False, help='Clears destination directories before starting synchronizations') args = parser.parse_args() if args.setup_feature: @@ -45,4 +46,4 @@ if not path.isdir(target) and not args.use_sftp: print(f"Encountered a directory error in the settings.ini file. Please make sure the {P_DEST_DIR} is a valid directory.") exit(-1) - checker = FileChecker(config=config, debug=args.debug_feature, quiet=args.quiet_feature, use_sftp=args.use_sftp, sftp_user=args.sftp_user, sftp_pass=args.sftp_pass, no_live_scan=args.live_scan, batch_size=args.batch_size, hash_algo=args.hash_algorithm, benchmark=args.bench_feature, multi=args.multi_feature, scan_interval=int(args.scan_interval)) + checker = FileChecker(config=config, debug=args.debug_feature, quiet=args.quiet_feature, clear_on_start=args.clear_on_start, use_sftp=args.use_sftp, sftp_user=args.sftp_user, sftp_pass=args.sftp_pass, no_live_scan=args.live_scan, batch_size=args.batch_size, hash_algo=args.hash_algorithm, benchmark=args.bench_feature, multi=args.multi_feature, scan_interval=int(args.scan_interval)) diff --git a/FileSync/main.py b/FileSync/main.py index 5e9e0cd..5596a9b 100644 --- a/FileSync/main.py +++ b/FileSync/main.py @@ -6,7 +6,7 @@ from os import makedirs, walk, remove, listdir from pathlib import Path from time import sleep, time -from FileSync.resources.strings import * +from resources.strings import * # Resolves hash algorithm provided by the end-user. @@ -51,17 +51,20 @@ def delete_file(self, target_src, full_target_src, sftp_client=None): except OSError as e: # Reports file read/write permission errors. if e.errno == errno.EPERM: - print(f"Encountered a file permission error while deleting files/directories:\n{e}") + if self.debug: + print(f"Encountered a file permission error while deleting files/directories:\n{e}") return False else: - print(f"Encountered an error while deleting files/directories:\n{e}") + if self.debug: + print(f"Encountered an error while deleting files/directories:\n{e}") return False else: try: sftp_client.remove(full_target_src.as_posix()) return True except Exception as e: - print(f"Encountered SFTP file deletion error:\n{e}") + if self.debug: + print(f"Encountered SFTP file deletion error:\n{e}") return False # Copies files from a source file path to a destination file path, maintaining sub-folder hierarchy. @@ -82,9 +85,11 @@ def copy_file(self, file_src, target_src, full_target_src, sftp_client=None): shutil.copy(file_src, full_target_src) # Reports file read/write permission errors. elif e.errno == errno.EPERM: - print(f"Encountered a file permission error while copying files/directories:\n{e}") + if self.debug: + print(f"Encountered a file permission error while copying files/directories:\n{e}") else: - print(f"Encountered an error while copying files/directories:\n{e}") + if self.debug: + print(f"Encountered an error while copying files/directories:\n{e}") else: try: sftp_client.mkdir(target_src.as_posix()) @@ -93,13 +98,18 @@ def copy_file(self, file_src, target_src, full_target_src, sftp_client=None): print(f"Directory already exists: {target_src.as_posix()}") try: sftp_client.put(file_src.as_posix(), full_target_src.as_posix()) + if self.debug: + print(f'SRC: {file_src.as_posix()}') + print(f'DST: {full_target_src.as_posix()}') + except Exception as e: - print(f"Encountered SFTP file transfer error:\n{e}") + if self.debug: + print(f"Encountered SFTP file transfer error:\n{e}") # Scans the source directory for changes (by checksum) and syncs files to destination directories. class FileChecker: - def __init__(self, config, multi, no_live_scan, batch_size, hash_algo, benchmark, scan_interval, debug=False, quiet=False, use_sftp=False, sftp_pass='', sftp_user=''): + def __init__(self, config, multi, no_live_scan, batch_size, hash_algo, benchmark, scan_interval, debug=False, quiet=False, clear_on_start=False, use_sftp=False, sftp_pass='', sftp_user=''): self.config = config self.debug = debug self.no_live_scan = no_live_scan @@ -135,6 +145,16 @@ def __init__(self, config, multi, no_live_scan, batch_size, hash_algo, benchmark self.copier = FileBackup(debug=self.debug) self.hash_dict = {} + if clear_on_start: + target_paths = [x.strip() for x in self.config[C_MAIN_SETTINGS][P_DEST_DIR].split(',')] + from os import unlink, path + for x in target_paths: + for root, dirs, files in walk(x): + for f in files: + unlink(path.join(root, f)) + for d in dirs: + shutil.rmtree(path.join(root, d)) + self.live_scan() def live_scan(self): @@ -415,7 +435,8 @@ def scan_directory_single(self) -> bool: change_detected = False ignore_dir_list = ([x.strip() for x in self.config[C_MAIN_SETTINGS][P_DIR_IGNORE].split(',')]) src_dir = self.config[C_MAIN_SETTINGS][P_SRC_DIR] - for (dir_path, dir_names, file_names) in walk(src_dir): + for (dir_path, dir_names, file_names) in walk(src_dir, topdown=False): + print(f'DIR_NAMES: {dir_names}') if dir_path.split('\\')[-1] in ignore_dir_list: if self.debug: print(f"Ignoring directory: {dir_path}") @@ -424,6 +445,10 @@ def scan_directory_single(self) -> bool: if self.debug: print(f"Ignoring directory: {dir_path}") continue + elif any(x in dir_path for x in ignore_dir_list): + if self.debug: + print(f"Ignoring directory: {dir_path}") + continue start_time = time() @@ -431,7 +456,7 @@ def scan_directory_single(self) -> bool: change_detected = self.copy_file_to_dest(dir_path=dir_path, file_names=file_names) # Delete files on destination directories that no longer exist in the source directory. - self.delete_missing_files(dir_path=dir_path, file_names=file_names) + # self.delete_missing_files(dir_path=dir_path, file_names=file_names) # TODO: Re-sync files that are in source directory, but missing in destination directory. # TODO: Add file deletion to multi-core directory scans, and SFTP scans. From 684e8a6415900317dd240871e1a56ce704330a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jason=20=5B=EC=A0=9C=EC=9D=B4=EC=8A=A8=20=EC=A0=9C?= =?UTF-8?q?=EB=A1=AC=5D?= Date: Sat, 30 May 2020 17:07:25 -0400 Subject: [PATCH 2/2] Added --clear-targets launch parameter --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c4af27a..416790a 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Open the settings.ini file provided in the repository and modify the data as nee --multi: Enables multi-core processing (not recommended for small directories). --batch-size : Sets the batch size for multi-core processing, if enabled (recommended - 100+ for large quantities of data) --scan-interval : Sets the time interval in seconds between directory scans (recommended - 2-5s) +--clear-targets: Clears destination directories before starting synchronizations --hash : Sets the hashing algorithm to use for checksums (recommended - sha256). Supported hashing algorithms: [adler32, crc32, md5, sha1, sha224, sha256, sha384, sha512] --use-sftp: Enables SFTP server connectivity (use with --username/--password command)