Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add scp provider #12

Merged
merged 4 commits into from
Sep 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 16 additions & 15 deletions FileSyncApp/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,12 @@ public class Program
public static volatile bool keepRunning = true;
public static LoggingLevelSwitch LoggingLevel { get; private set; }
public static ILoggerFactory LoggerFactory { get; private set; }
private static Microsoft.Extensions.Logging.ILogger log;
public static Dictionary<string, IFileJob> Jobs = new Dictionary<string, IFileJob>();
public static void Main(string[] args)
{





LoggingLevel= new LoggingLevelSwitch(Serilog.Events.LogEventLevel.Information);
#if DEBUG
LoggingLevel= new LoggingLevelSwitch(Serilog.Events.LogEventLevel.Verbose);
#endif
ConfigureLogger();
LoggerFactory = Microsoft.Extensions.Logging.LoggerFactory.Create(builder => { builder.AddSerilog(Serilog.Log.Logger); });
log = LoggerFactory.CreateLogger("FileSyncAppMain");
if (null!=args && args.Length > 0)
{
if (args.Contains("debug"))
Expand All @@ -58,7 +50,7 @@ public static void Main(string[] args)

static void RunProgram()
{
Console.WriteLine("FileSyncApp - synchronizing folders and clean them up");
log.LogInformation("FileSyncApp - synchronizing folders and clean them up");
Dictionary<string, IFileJobOptions> jobOptions = new Dictionary<string, IFileJobOptions>();
var jsonSettings = new JsonSerializerSettings
{
Expand All @@ -68,6 +60,7 @@ static void RunProgram()
};
if (!File.Exists("config.json"))
{
log.LogInformation("Config file {A} not found, creating a new one", "config.json");
var cleanJob = FileCleanJobOptionsBuilder.CreateBuilder()
.WithDestinationPath("temp")
.WithInterval(TimeSpan.FromMinutes(21))
Expand Down Expand Up @@ -103,6 +96,7 @@ static void RunProgram()
var json = JsonConvert.SerializeObject(jobOptions, Formatting.Indented, jsonSettings);
File.WriteAllText("config.json", json);
}
log.LogInformation("reading config file {A}", "config.json");
var readJobOptions = JsonConvert.DeserializeObject<Dictionary<string, IFileJobOptions>>(File.ReadAllText("config.json"), jsonSettings);
//List<IFileJob> Jobs = new List<IFileJob>();

Expand All @@ -117,16 +111,22 @@ static void RunProgram()
{
job.Value.StartJob();
}
Console.WriteLine("Press Ctrl+C to exit");
log.LogInformation("Press Ctrl+C to exit");
while (keepRunning)
{
Thread.Sleep(200);
Thread.Sleep(500);
}
}


private static void ConfigureLogger()
public static void ConfigureLogger()
{
if (LoggingLevel != null)
return;
LoggingLevel = new LoggingLevelSwitch(Serilog.Events.LogEventLevel.Information);
#if DEBUG
LoggingLevel = new LoggingLevelSwitch(Serilog.Events.LogEventLevel.Verbose);
#endif
string serilogFileTemplate = "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext:l} {Message:lj}{NewLine}{Exception}";
string serilogConsoleTemplate = "{Timestamp:HH:mm:ss.fff}[{Level:u3}]{SourceContext:l} {Message:lj}{NewLine}{Exception}";
Serilog.Log.Logger = new LoggerConfiguration()
Expand All @@ -137,12 +137,13 @@ private static void ConfigureLogger()
retainedFileCountLimit: 10,
fileSizeLimitBytes: 1024 * 1024 * 100,
rollingInterval: RollingInterval.Day,
rollOnFileSizeLimit: true,
outputTemplate: serilogFileTemplate),
blockWhenFull: true)
.MinimumLevel.ControlledBy(LoggingLevel)
.CreateLogger();


LoggerFactory = Microsoft.Extensions.Logging.LoggerFactory.Create(builder => { builder.AddSerilog(Serilog.Log.Logger); });
}


Expand Down
19 changes: 9 additions & 10 deletions FileSyncAppWin/MainForm.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
using FileSyncLibNet.Commons;

namespace FileSyncAppWin
{
public partial class MainForm : Form
Expand All @@ -12,11 +10,11 @@ public MainForm()
this.FormClosing += (s, e) => { FileSyncApp.Program.keepRunning = false; consoleThread?.Join(10_000); };
consoleThread = new Thread(() =>
{

FileSyncApp.Program.Main(null);
});
FileSyncApp.Program.JobsReady += (s,e)=> {
foreach(var job in FileSyncApp.Program.Jobs)
FileSyncApp.Program.JobsReady += (s, e) =>
{
foreach (var job in FileSyncApp.Program.Jobs)
{
job.Value.JobStarted += (j, text) =>
{
Expand All @@ -36,6 +34,7 @@ public MainForm()

};
consoleThread.Start();

this.Resize += ((s, e) =>
{
this.SuspendLayout();
Expand All @@ -56,20 +55,20 @@ public MainForm()
{
this.BeginInvoke(() => { WindowState = FormWindowState.Normal; ShowInTaskbar = true; });
};
notifyIcon1.BalloonTipClicked += (s, e) => { this.BeginInvoke(() => { WindowState = FormWindowState.Normal; ShowInTaskbar = true; }); };
notifyIcon1.BalloonTipClicked += (s, e) => { this.BeginInvoke(() => { WindowState = FormWindowState.Normal; ShowInTaskbar = true; }); };
//notifyIcon1.BalloonTipShown += (s, e) => { ShowInTaskbar = false; };

//this.WindowState = FormWindowState.Minimized;
}






private void NewLogOutput(string e)
{

this.BeginInvoke(() => { textBox1.Text = string.Join(Environment.NewLine, (new string[] { $"{DateTime.Now.ToString("HH:mm:ss.fff")} {e}" }).Concat(textBox1.Text.Split(Environment.NewLine).Take(1000))); });
this.BeginInvoke(() => { textBox1.Text = string.Join(Environment.NewLine, (new string[] { $"{DateTime.Now.ToString("HH:mm:ss.fff")} {e}" }).Concat(textBox1.Text.Split(Environment.NewLine).Take(1000))); });

}
}
}
12 changes: 11 additions & 1 deletion FileSyncLibNet/FileSyncJob/FileSyncJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ private FileSyncJob(IFileJobOptions fileSyncJobOptions)
case SyncProvider.Robocopy:
syncProvider = new RoboCopyProvider(fileSyncJobOptions);
break;
case SyncProvider.SCP:
syncProvider = new ScpProvider(fileSyncJobOptions);
break;
}
}

Expand Down Expand Up @@ -70,7 +73,14 @@ public void StopJob()
}
private void TimerElapsed(object state)
{
RunJobInterlocked();
try
{
RunJobInterlocked();
}
catch (Exception exc)
{
JobError?.Invoke(this, new FileSyncJobEventArgs(JobName, FileSyncJobStatus.Error, exc));
}
}

private void RunJobInterlocked()
Expand Down
7 changes: 4 additions & 3 deletions FileSyncLibNet/FileSyncLibNet.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@
<Description>A library to easily backup or sync 2 folders either once or in a given interval.</Description>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
<PackageReference Include="RoboSharp" Version="1.3.5" />
<PackageReference Include="SMBLibrary" Version="1.5.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageReference Include="RoboSharp" Version="1.5.3" />
<PackageReference Include="SMBLibrary" Version="1.5.3.5" />
<PackageReference Include="SSH.NET" Version="2024.1.0" />
</ItemGroup>
<ItemGroup>
<Content Include="../README.md">
Expand Down
155 changes: 155 additions & 0 deletions FileSyncLibNet/SyncProviders/ScpProvider.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
using FileSyncLibNet.Commons;
using FileSyncLibNet.FileCleanJob;
using FileSyncLibNet.FileSyncJob;
using Microsoft.Extensions.Logging;
using Renci.SshNet;
using Renci.SshNet.Sftp;
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

namespace FileSyncLibNet.SyncProviders
{
internal class ScpProvider : ProviderBase
{


public ScpProvider(IFileJobOptions options) : base(options)
{

}

private void CreateDirectoryRecursive(SftpClient client, string path)
{
var dirs = path.Split('/');
string currentPath =path.StartsWith("/")?"/":"";
for (int i = 0; i < dirs.Length; i++)
{
currentPath = (currentPath.EndsWith("/")?currentPath:(currentPath+"/")) + dirs[i];
if (!string.IsNullOrEmpty(currentPath) && currentPath != "/")
if(!client.Exists(currentPath))
client.CreateDirectory(currentPath);
}
}

public override void SyncSourceToDest()
{
if (!(JobOptions is IFileSyncJobOptions jobOptions))
throw new ArgumentException("this instance has no information about syncing files, it has type " + JobOptions.GetType().ToString());
var sw = Stopwatch.StartNew();
//Format

var pattern = @"scp://(?:(?<user>[^@]+)@)?(?<host>[^:/]+)(?::(?<port>\d+))?(?<path>/.*)?";
var match = Regex.Match(JobOptions.DestinationPath, pattern);
string path;
SftpClient ftpClient;
if (!match.Success)
{
throw new UriFormatException($"Unable to match scp pattern with given URL {JobOptions.DestinationPath}, use format scp://host:port/path");
}
else
{
var user = match.Groups["user"].Value;
var host = match.Groups["host"].Value;
var port = int.Parse(match.Groups["port"].Success ? match.Groups["port"].Value : "22"); // Default SCP port
path = match.Groups["path"].Value;


ftpClient = new SftpClient(host, port, JobOptions.Credentials.UserName, JobOptions.Credentials.Password);
}
ftpClient.Connect();
CreateDirectoryRecursive(ftpClient, path);

var remoteFiles = ftpClient.ListDirectory(path);

Directory.CreateDirectory(jobOptions.SourcePath);

int copied = 0;
int skipped = 0;
DirectoryInfo _di = new DirectoryInfo(jobOptions.SourcePath);
//Dateien ins Backup kopieren
if (JobOptions.Credentials != null)
{

}


foreach (var dir in JobOptions.Subfolders.Count > 0 ? _di.GetDirectories() : new[] { _di })
{
if (JobOptions.Subfolders.Count > 0 && !JobOptions.Subfolders.Select(x => x.ToLower()).Contains(dir.Name.ToLower()))
continue;
var _fi = dir.EnumerateFiles(
searchPattern: JobOptions.SearchPattern,
searchOption: JobOptions.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);

if (jobOptions.RememberLastSync)
{
var old = _fi.Count();
_fi = _fi.Where(x => x.LastWriteTime > (LastRun - jobOptions.Interval)).ToList();
skipped += old - _fi.Count();
LastRun = DateTimeOffset.Now;
}
foreach (FileInfo f in _fi)
{

var relativeFilename = f.FullName.Substring(Path.GetFullPath(jobOptions.SourcePath).Length).TrimStart('\\');
var remoteFilename = Path.Combine(path, relativeFilename).Replace('\\', '/');
ISftpFile remotefile = null;
if (ftpClient.Exists(remoteFilename))
{
remotefile = ftpClient.Get(remoteFilename);
}
bool exists = remotefile != null;
bool lengthMatch = exists && remotefile.Length == f.Length;
bool timeMatch = exists && Math.Abs((remotefile.LastWriteTimeUtc -f.LastWriteTimeUtc).TotalSeconds)<1;

bool copy = !exists || !lengthMatch || !timeMatch;
if (copy)
{
try
{
logger.LogDebug("Copy {A}", relativeFilename);
CreateDirectoryRecursive(ftpClient, Path.GetDirectoryName(remoteFilename).Replace('\\', '/'));

using (var fileStream = System.IO.File.OpenRead(f.FullName))
{
ftpClient.UploadFile(fileStream, remoteFilename);
}
ftpClient.SetLastWriteTimeUtc(remoteFilename, f.LastAccessTimeUtc);
copied++;
if (jobOptions.DeleteSourceAfterBackup)
{
File.Delete(f.FullName);
}
}
catch (Exception exc)
{
logger.LogError(exc, "Exception copying {A}", relativeFilename);
}
}
else
{
skipped++;
logger.LogTrace("Skip {A}", relativeFilename);
}
}
}
sw.Stop();
logger.LogInformation("{A} files copied, {B} files skipped in {C}s", copied, skipped, sw.ElapsedMilliseconds / 1000.0);
}


public override void DeleteFiles()
{
if (!(JobOptions is IFileCleanJobOptions jobOptions))
throw new ArgumentException("this instance has no information about deleting files, it has type " + JobOptions.GetType().ToString());

throw new NotImplementedException("deleting files via scp currently is not supported");
}



}
}
1 change: 1 addition & 0 deletions FileSyncLibNet/SyncProviders/SyncProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@ public enum SyncProvider
FileIO,
SMBLib,
Robocopy,
SCP,
}
}
Loading