From 36c72f9c0ea0bb5d6d4424c46df1b1c6190cb25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=BCffel?= Date: Fri, 8 Sep 2023 11:48:24 +0200 Subject: [PATCH] add new WithSubfolder option to specify only a subset of folders within SourcePath --- FileSyncJob/FileSyncJobOptions.cs | 7 +- FileSyncJob/FileSyncJobOptionsBuilder.cs | 6 + FileSyncJob/IFileSyncJobOptions.cs | 2 + SyncProviders/FileIOProvider.cs | 192 ++++++++++++++++++++--- SyncProviders/SmbLibProvider.cs | 68 +++++--- 5 files changed, 227 insertions(+), 48 deletions(-) diff --git a/FileSyncJob/FileSyncJobOptions.cs b/FileSyncJob/FileSyncJobOptions.cs index 1949d61..bb3bdd6 100644 --- a/FileSyncJob/FileSyncJobOptions.cs +++ b/FileSyncJob/FileSyncJobOptions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Logging; using System; +using System.Collections.Generic; using System.Net; namespace FileSyncLibNet.FileSyncJob @@ -12,11 +13,15 @@ public class FileSyncJobOptions : IFileSyncJobOptions public NetworkCredential Credentials { get; set; } public TimeSpan Interval { get; set; } = TimeSpan.Zero; public string SearchPattern { get; set; } = "*.*"; + public List Subfolders { get; set; } public bool Recursive { get; set; } = true; public bool SyncDeleted { get; set; } = false; public FileSyncProvider FileSyncProvider { get; set; } = FileSyncProvider.FileIO; - public FileSyncJobOptions() { } + public FileSyncJobOptions() + { + Subfolders = new List(); + } } } diff --git a/FileSyncJob/FileSyncJobOptionsBuilder.cs b/FileSyncJob/FileSyncJobOptionsBuilder.cs index 405d72e..b1fcba1 100644 --- a/FileSyncJob/FileSyncJobOptionsBuilder.cs +++ b/FileSyncJob/FileSyncJobOptionsBuilder.cs @@ -69,6 +69,11 @@ public IFileSyncJobOptionsBuilderSetProperties WithSearchPattern(string searchPa jobOptions.SearchPattern = searchPattern; return this; } + public IFileSyncJobOptionsBuilderSetProperties WithSubfolder(string subfolder) + { + jobOptions.Subfolders.Add(subfolder); + return this; + } public IFileSyncJobOptionsBuilderSetProperties WithInterval(TimeSpan interval) { jobOptions.Interval = interval; @@ -118,6 +123,7 @@ public interface IFileSyncJobOptionsBuilderSetProperties : IFileSyncJobOptionsBu IFileSyncJobOptionsBuilderSetProperties WithCredentials(string username, string password); IFileSyncJobOptionsBuilderSetProperties WithCredentials(NetworkCredential networkCredential); IFileSyncJobOptionsBuilderSetProperties WithSearchPattern(string searchPattern); + IFileSyncJobOptionsBuilderSetProperties WithSubfolder(string subfolder); IFileSyncJobOptionsBuilderSetProperties WithLogger(ILogger logger); IFileSyncJobOptionsBuilderSetProperties WithLogger(Action loggerAction); } diff --git a/FileSyncJob/IFileSyncJobOptions.cs b/FileSyncJob/IFileSyncJobOptions.cs index 023550d..2613809 100644 --- a/FileSyncJob/IFileSyncJobOptions.cs +++ b/FileSyncJob/IFileSyncJobOptions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Logging; using System; +using System.Collections.Generic; using System.Net; namespace FileSyncLibNet.FileSyncJob @@ -13,6 +14,7 @@ public interface IFileSyncJobOptions ILogger Logger { get; set; } bool Recursive { get; set; } string SearchPattern { get; set; } + List Subfolders { get; set; } string SourcePath { get; set; } bool SyncDeleted { get; set; } } diff --git a/SyncProviders/FileIOProvider.cs b/SyncProviders/FileIOProvider.cs index 86abbdb..f66ab3d 100644 --- a/SyncProviders/FileIOProvider.cs +++ b/SyncProviders/FileIOProvider.cs @@ -1,9 +1,13 @@ using FileSyncLibNet.FileSyncJob; using Microsoft.Extensions.Logging; using System; +using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; using System.IO; +using System.Linq; using System.Net; +using System.Threading; namespace FileSyncLibNet.SyncProviders { @@ -22,38 +26,47 @@ public override void SyncSourceToDest() int copied = 0; int skipped = 0; DirectoryInfo _di = new DirectoryInfo(JobOptions.SourcePath); - var _fi = _di.EnumerateFiles( - searchPattern: JobOptions.SearchPattern, - searchOption: JobOptions.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); //Dateien ins Backup kopieren if (JobOptions.Credentials != null) { - - } - foreach (FileInfo f in _fi) + } + + + foreach (var dir in JobOptions.Subfolders.Count > 0 ? _di.GetDirectories() : new[] {_di}) { - var relativeFilename = f.FullName.Substring(Path.GetFullPath(JobOptions.SourcePath).Length).TrimStart('\\'); - var remotefile = new FileInfo(Path.Combine(JobOptions.DestinationPath, relativeFilename)); - bool copy = !remotefile.Exists || remotefile.Length != f.Length || remotefile.LastWriteTime != f.LastWriteTime; - if (copy) + if (JobOptions.Subfolders.Count > 0 && !JobOptions.Subfolders.Select(x => x.ToLower()).Contains(dir.Name.ToLower())) + continue; + var _fi = dir.EnumerateFiles( + searchPattern: JobOptions.SearchPattern, + searchOption: JobOptions.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); + + + foreach (FileInfo f in _fi) { - try + + var relativeFilename = f.FullName.Substring(Path.GetFullPath(JobOptions.SourcePath).Length).TrimStart('\\'); + var remotefile = new FileInfo(Path.Combine(JobOptions.DestinationPath, relativeFilename)); + bool copy = !remotefile.Exists || remotefile.Length != f.Length || remotefile.LastWriteTime != f.LastWriteTime; + if (copy) { - logger.LogDebug("Copy {A}", relativeFilename); - File.Copy(f.FullName, remotefile.FullName, true); - copied++; + try + { + logger.LogDebug("Copy {A}", relativeFilename); + File.Copy(f.FullName, remotefile.FullName, true); + copied++; + } + catch (Exception exc) + { + logger.LogError(exc, "Exception copying {A}", relativeFilename); + } } - catch(Exception exc) + else { - logger.LogError(exc, "Exception copying {A}", relativeFilename); + skipped++; + logger.LogTrace("Skip {A}", relativeFilename); } } - else - { - skipped++; - logger.LogTrace("Skip {A}", relativeFilename); - } } sw.Stop(); logger.LogInformation("{A} files copied, {B} files skipped in {C}s", copied, skipped, sw.ElapsedMilliseconds / 1000.0); @@ -74,7 +87,142 @@ void CopyFileWithBuffer(FileInfo source, FileInfo destination) //destination.Attributes = source.Attributes; } - + void EnsureAccess() + { + //#region Backup + //if (!string.IsNullOrWhiteSpace(einst.BasicSettings["BackupPath"])) + //{ + // //Dateien sichern: + // bool useCredentials = ((string)einst.BasicSettings["BackupPath"]).StartsWith(@"\\"); + // string backupTarget = einst.BasicSettings["BackupPath"]; + + // if (backupTarget.StartsWith(@"\\")) //Netzwerk + // { + // try + // { + // //Probe network access: + // bool accesible = false; + // bool writeable = false; + // try + // { + // DirectoryInfo di = new DirectoryInfo(backupTarget); + // while (di != null && !di.Exists) + // { + // di = di.Parent; + // } + // if (di != null && di.Exists) + // accesible = true; + // if (accesible) + // { + // Directory.CreateDirectory(backupTarget); + // File.WriteAllText(backupTarget + "\\hri_write_probing", "probing write access"); + // try + // { + // File.Delete(backupTarget + "\\hri_write_probing"); + // } + // catch (Exception exc) + // { + // log.LogWarning(exc, "Exception when deleting 'hri_write_probing' file, assuming writable"); + // } + // writeable = true; + // } + // } + // catch { } + // if (!accesible || !writeable) + // { + // DirectoryInfo di = new DirectoryInfo(backupTarget); + // string netpath = di.Root.FullName; + + // if (accesible) //Change credentials.. + // { + // NetworkShare.CancelExistingConnection(netpath); + // } + + // string username = ((string)Einstellungen.BasicSettings["NetworkCredentials"]).Split(':')[0]; + // string password = ((string)Einstellungen.BasicSettings["NetworkCredentials"]).Split(':')[1]; + // NetworkCredential networkCredential = new NetworkCredential(username, password); + // NetworkShare net = null; + // try { net = new NetworkShare(netpath, networkCredential, true); } + // catch (Win32Exception exception) { log.LogInformation("Backup: Fehler beim herstellen einer Netzwerkverbindung zu " + netpath + ":\r\n" + exception.ToString()); } + // } + // } + // catch (Exception e) + // { + // log.LogError("backuptimer_tick", e); + // log.LogInformation("Backup: Fehler beim Zugriff auf Netzlaufwerk:\r\n" + e.ToString()); + // } + + // } + // //else //Filesystem + // { + // try + // { + // //Beim Netzlaufwerk versuchen es erneut zu verbinden + // string drive = backupTarget.Split(':')[0]; + // //if (!Map(drive + ":")) + // // log.LogInformation("Fehler beim mappen des Laufwerkes " + drive + ": - Bitte Einstellungen überprüfen"); + // bool canWrite = true; + // if (!Directory.Exists(backupTarget)) + // { + // try + // { + // Directory.CreateDirectory(backupTarget); + // } + // catch { log.LogInformation($"Konnte nicht auf Backuppfad {backupTarget} schreiben"); canWrite = false; } + // } + // if (canWrite) + // foreach (string folder in new string[] { "HriFFTLog", "HriShockLog", "HriLog", "HriDebugLog", "HRI" }) + // { + // string _path = Path.Combine(ProductionDataPath, folder); + // if (!Directory.Exists(_path)) + // Directory.CreateDirectory(_path); + // string destPath = Path.Combine(backupTarget, folder); + // if (Directory.Exists(backupTarget)) + // if (!Directory.Exists(destPath)) + // Directory.CreateDirectory(destPath); + // DirectoryInfo _di = new DirectoryInfo(_path); + // FileInfo[] _fi = _di.GetFiles(); + // //Dateien ins Backup kopieren + // foreach (FileInfo f in _fi) + // { + // bool copy = false; + // string remotefilename = Path.Combine(destPath, f.Name); + // if (!File.Exists(remotefilename)) + // { + // copy = true; + // } + // else + // { + // FileInfo remoteFile = new FileInfo(remotefilename); + // if (remoteFile.Length != f.Length) + // copy = true; + // } + // if (copy) + // { + // File.Copy(f.FullName, remotefilename, true); + // Thread.Sleep(15); + // } + // else + // { + // Thread.Sleep(5); + // } + // } + // } + + + // } + // catch (Exception e) + // { + // log.LogError("backuptimer_tick", e); + // log.LogInformation("Fehler beim erstellen des Backups!\r\n" + e.ToString()); + // } + // } + //} + //#endregion + + + } + diff --git a/SyncProviders/SmbLibProvider.cs b/SyncProviders/SmbLibProvider.cs index b65ce0f..0525872 100644 --- a/SyncProviders/SmbLibProvider.cs +++ b/SyncProviders/SmbLibProvider.cs @@ -55,42 +55,60 @@ public SmbLibProvider(IFileSyncJobOptions options) public override void SyncSourceToDest() { - DirectoryInfo _di = new DirectoryInfo(JobOptions.SourcePath); - var _fi = _di.EnumerateFiles( - searchPattern: JobOptions.SearchPattern, - searchOption: JobOptions.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); //Dateien ins Backup kopieren if (JobOptions.Credentials != null) { ConnectToShare(Server, Share, JobOptions.Credentials.Domain, JobOptions.Credentials.UserName, JobOptions.Credentials.Password); } - if (JobOptions.SyncDeleted) + + DirectoryInfo _di = new DirectoryInfo(JobOptions.SourcePath); + + foreach (var dir in JobOptions.Subfolders.Count > 0 ? _di.GetDirectories() : new[] { _di }) { - var remoteFiles = ListFiles(DestinationPath, true); - foreach (var file in remoteFiles) + if (JobOptions.Subfolders.Count > 0 && !JobOptions.Subfolders.Select(x => x.ToLower()).Contains(dir.Name.ToLower())) + continue; + + + + var _fi = dir.EnumerateFiles( + searchPattern: JobOptions.SearchPattern, + searchOption: JobOptions.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); + + if (JobOptions.SyncDeleted) { - var realFilePath = file.Substring(file.IndexOf(DestinationPath) + DestinationPath.Length).Trim('\\').Replace('/', '\\'); - if (!_fi.Any(x => x.FullName.Replace('/', '\\').EndsWith(realFilePath))) - DeleteFile(file); + var remoteFiles = ListFiles(DestinationPath, true); + foreach (var file in remoteFiles) + { + var realFilePath = file.Substring(file.IndexOf(DestinationPath) + DestinationPath.Length).Trim('\\').Replace('/', '\\'); + if (!_fi.Any(x => x.FullName.Replace('/', '\\').EndsWith(realFilePath))) + DeleteFile(file); + } } - } - foreach (FileInfo f in _fi) - { - bool copy = false; - var relativeFilename = f.FullName.Substring(Path.GetFullPath(JobOptions.SourcePath).Length); - var remotefile = Path.Combine(DestinationPath, relativeFilename.TrimStart('\\', '/')).Replace('/', '\\'); - var exists = FileExists(remotefile, out long size); - copy = !exists || size != f.Length; - if (copy) + foreach (FileInfo f in _fi) { - logger.LogDebug("Copy {A}", relativeFilename); - //File.Copy(f.FullName, remotefile.FullName, true); - //CopyFileWithBuffer(f, remotefile); - WriteFile(f.FullName, remotefile); + bool copy = false; + var relativeFilename = f.FullName.Substring(Path.GetFullPath(JobOptions.SourcePath).Length); + var remotefile = Path.Combine(DestinationPath, relativeFilename.TrimStart('\\', '/')).Replace('/', '\\'); + var exists = FileExists(remotefile, out long size); + copy = !exists || size != f.Length; + if (copy) + { + logger.LogDebug("Copy {A}", relativeFilename); + //File.Copy(f.FullName, remotefile.FullName, true); + //CopyFileWithBuffer(f, remotefile); + try + { + WriteFile(f.FullName, remotefile); + } + catch (Exception exc) + { + logger.LogError(exc, "Exception in WriteFile for {A}", relativeFilename); + } + } + else + logger.LogDebug("Skip {A}", relativeFilename); } - else - logger.LogDebug("Skip {A}", relativeFilename); } }