diff --git a/src/Paillave.Etl.AzureStorageAccountFileProvider/BlobExtensions.cs b/src/Paillave.Etl.AzureStorageAccountFileProvider/BlobExtensions.cs index 8c68f5fd..3b9fa4c5 100644 --- a/src/Paillave.Etl.AzureStorageAccountFileProvider/BlobExtensions.cs +++ b/src/Paillave.Etl.AzureStorageAccountFileProvider/BlobExtensions.cs @@ -15,7 +15,7 @@ private static BlobServiceClient GetBlobServiceClient(this AzureBlobOptions opti if (options.ConnectionString != null) return new BlobServiceClient(options.ConnectionString); else if (options.BaseUri != null && (options.DefaultAzureCredential ?? false)) - return new BlobServiceClient(options.BaseUri, new DefaultAzureCredential()); + return new BlobServiceClient(options.BaseUri, new DefaultAzureCredential(Environment.UserInteractive)); else throw new ArgumentException("One of the following must be set: 'ConnectionString' or 'BaseUri'+'Token'!", nameof(options)); } diff --git a/src/Paillave.Etl.AzureStorageAccountFileProvider/Paillave.Etl.AzureStorageAccountFileProvider.csproj b/src/Paillave.Etl.AzureStorageAccountFileProvider/Paillave.Etl.AzureStorageAccountFileProvider.csproj index 892d95ab..53eb54f1 100644 --- a/src/Paillave.Etl.AzureStorageAccountFileProvider/Paillave.Etl.AzureStorageAccountFileProvider.csproj +++ b/src/Paillave.Etl.AzureStorageAccountFileProvider/Paillave.Etl.AzureStorageAccountFileProvider.csproj @@ -12,10 +12,10 @@ - + - + diff --git a/src/Paillave.Etl.Dropbox/String.ex.cs b/src/Paillave.Etl.Dropbox/String.ex.cs deleted file mode 100644 index 67fed79e..00000000 --- a/src/Paillave.Etl.Dropbox/String.ex.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.IO; - -namespace Paillave.Etl.Dropbox -{ - public static class StringEx - { - public static Stream ToStream(this string s) - { - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(s); - writer.Flush(); - stream.Position = 0; - return stream; - } - } -} \ No newline at end of file diff --git a/src/Paillave.Etl.Ftp/FtpFileValue.cs b/src/Paillave.Etl.Ftp/FtpFileValue.cs index cb48e74f..f6d2b72a 100644 --- a/src/Paillave.Etl.Ftp/FtpFileValue.cs +++ b/src/Paillave.Etl.Ftp/FtpFileValue.cs @@ -24,7 +24,7 @@ public FtpFileValue(IFtpConnectionInfo connectionInfo, string folder, string fil protected override void DeleteFile() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, DeleteFileSingleTime); protected void DeleteFileSingleTime() { - var pathToDelete = Path.Combine(_folder, this.Name).Replace('\\', '/'); + var pathToDelete = StringEx.ConcatenatePath(_folder, this.Name).Replace('\\', '/'); using (FtpClient client = _connectionInfo.CreateFtpClient()) client.DeleteFile(pathToDelete); } @@ -34,7 +34,7 @@ private Stream GetContentSingleTime() using (FtpClient client = _connectionInfo.CreateFtpClient()) { MemoryStream ms = new MemoryStream(); - var pathToDownload = Path.Combine(_folder, this.Name).Replace('\\', '/'); + var pathToDownload = StringEx.ConcatenatePath(_folder, this.Name).Replace('\\', '/'); if (!client.DownloadStream(ms, pathToDownload)) throw new System.Exception($"File {pathToDownload} failed to be downloaded"); ms.Seek(0, SeekOrigin.Begin); @@ -44,7 +44,7 @@ private Stream GetContentSingleTime() public override StreamWithResource OpenContent() { FtpClient client = _connectionInfo.CreateFtpClient(); - var pathToDownload = Path.Combine(_folder, this.Name).Replace('\\', '/'); + var pathToDownload = StringEx.ConcatenatePath(_folder, this.Name).Replace('\\', '/'); var targetPath = Path.GetTempFileName(); if (client.DownloadFile(targetPath, pathToDownload) != FtpStatus.Success) throw new System.Exception($"File {pathToDownload} failed to be downloaded"); diff --git a/src/Paillave.Etl.Ftp/FtpFileValueProcessor.cs b/src/Paillave.Etl.Ftp/FtpFileValueProcessor.cs index e780ff31..579bf877 100644 --- a/src/Paillave.Etl.Ftp/FtpFileValueProcessor.cs +++ b/src/Paillave.Etl.Ftp/FtpFileValueProcessor.cs @@ -14,10 +14,10 @@ public FtpFileValueProcessor(string code, string name, string connectionName, Ft public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; protected override void Process(IFileValue fileValue, FtpAdapterConnectionParameters connectionParameters, FtpAdapterProcessorParameters processorParameters, Action push, CancellationToken cancellationToken, IExecutionContext context) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); - // var folder = Path.Combine(connectionParameters.RootFolder ?? "", processorParameters.SubFolder ?? ""); - var filePath = Path.Combine(folder, fileValue.Name); + // var folder = StringEx.ConcatenatePath(connectionParameters.RootFolder ?? "", processorParameters.SubFolder ?? ""); + var filePath = StringEx.ConcatenatePath(folder, fileValue.Name); filePath = SmartFormat.Smart.Format(filePath.Replace(@"\",@"\\"), fileValue.Metadata); using var stream = fileValue.Get(processorParameters.UseStreamCopy); byte[] fileContents; @@ -46,8 +46,8 @@ private void UploadSingleTime(FtpAdapterConnectionParameters connectionParameter protected override void Test(FtpAdapterConnectionParameters connectionParameters, FtpAdapterProcessorParameters processorParameters) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); - var testFilePath = Path.Combine(folder, Guid.NewGuid().ToString()); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); + var testFilePath = StringEx.ConcatenatePath(folder, Guid.NewGuid().ToString()); using (FtpClient client = connectionParameters.CreateFtpClient()) { var stream = new MemoryStream(); diff --git a/src/Paillave.Etl.Ftp/FtpFileValueProvider.cs b/src/Paillave.Etl.Ftp/FtpFileValueProvider.cs index 18ba4a16..ba2d9001 100644 --- a/src/Paillave.Etl.Ftp/FtpFileValueProvider.cs +++ b/src/Paillave.Etl.Ftp/FtpFileValueProvider.cs @@ -31,7 +31,7 @@ protected override void Provide(Action pushFileValue, FtpAdapterConn } private FtpListItem[] GetFileList(FtpAdapterConnectionParameters connectionParameters, FtpAdapterProviderParameters providerParameters) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); using (FtpClient client = connectionParameters.CreateFtpClient()) { return (providerParameters.Recursive ? client.GetListing(folder, FtpListOption.Recursive) : client.GetListing(folder)).Where(i => i.Type == FtpObjectType.File).ToArray(); @@ -39,7 +39,7 @@ private FtpListItem[] GetFileList(FtpAdapterConnectionParameters connectionParam } protected override void Test(FtpAdapterConnectionParameters connectionParameters, FtpAdapterProviderParameters providerParameters) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); using (FtpClient client = connectionParameters.CreateFtpClient()) { if (providerParameters.Recursive) diff --git a/src/Paillave.Etl.S3/S3Bucket.cs b/src/Paillave.Etl.S3/S3Bucket.cs index e2065aee..405c1eb2 100644 --- a/src/Paillave.Etl.S3/S3Bucket.cs +++ b/src/Paillave.Etl.S3/S3Bucket.cs @@ -6,6 +6,7 @@ using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; +using Paillave.Etl.Core; namespace Paillave.Etl.S3; public class S3Bucket : IDisposable @@ -24,7 +25,7 @@ public async Task UploadAsync(string objectName, Stream stream, string? folder = var response = await _client.PutObjectAsync(new PutObjectRequest { BucketName = _bucketName, - Key = string.IsNullOrWhiteSpace(folder) ? objectName : Path.Combine(folder, objectName), + Key = string.IsNullOrWhiteSpace(folder) ? objectName : StringEx.ConcatenatePath(folder, objectName), InputStream = stream // FilePath = filePath, }); @@ -35,7 +36,7 @@ public async Task UploadAsync(string objectName, Stream stream, string? folder = } public Task DownloadAsync(string objectName, string? folder = null) - => DownloadFromKeyAsync(string.IsNullOrWhiteSpace(folder) ? objectName : Path.Combine(folder, objectName)); + => DownloadFromKeyAsync(string.IsNullOrWhiteSpace(folder) ? objectName : StringEx.ConcatenatePath(folder, objectName)); public async Task DownloadFromKeyAsync(string objectKey) { GetObjectResponse response = await _client.GetObjectAsync(new GetObjectRequest diff --git a/src/Paillave.Etl.S3/S3FileValue.cs b/src/Paillave.Etl.S3/S3FileValue.cs index d14f0017..d18b80c1 100644 --- a/src/Paillave.Etl.S3/S3FileValue.cs +++ b/src/Paillave.Etl.S3/S3FileValue.cs @@ -26,7 +26,7 @@ protected void DeleteFileSingleTime() { using (var client = _connectionInfo.CreateBucketConnection()) { - client.DeleteAsync(Path.Combine(_folder, Name)).Wait(); + client.DeleteAsync(StringEx.ConcatenatePath(_folder, Name)).Wait(); } } public override Stream GetContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, GetContentSingleTime); @@ -44,7 +44,7 @@ private Stream GetContentSingleTime() private StreamWithResource OpenContentSingleTime() { var client = _connectionInfo.CreateBucketConnection(); - return new StreamWithResource(client.DownloadFromKeyAsync(Path.Combine(_folder, Name)).Result, client); + return new StreamWithResource(client.DownloadFromKeyAsync(StringEx.ConcatenatePath(_folder, Name)).Result, client); } public override StreamWithResource OpenContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, OpenContentSingleTime); diff --git a/src/Paillave.Etl.S3/S3FileValueProcessor.cs b/src/Paillave.Etl.S3/S3FileValueProcessor.cs index bdd9e8be..bef13df2 100644 --- a/src/Paillave.Etl.S3/S3FileValueProcessor.cs +++ b/src/Paillave.Etl.S3/S3FileValueProcessor.cs @@ -22,9 +22,9 @@ public S3FileValueProcessor(string code, string name, string serviceUrl, string public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; protected override void Process(IFileValue fileValue, S3AdapterConnectionParameters connectionParameters, S3AdapterProcessorParameters processorParameters, Action push, CancellationToken cancellationToken, IExecutionContext context) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); using var stream = fileValue.Get(processorParameters.UseStreamCopy); - ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, stream, Path.Combine(folder, fileValue.Name))); + ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, stream, StringEx.ConcatenatePath(folder, fileValue.Name))); push(fileValue); } private void UploadSingleTime(S3AdapterConnectionParameters connectionParameters, Stream stream, string filePath) @@ -37,14 +37,14 @@ private void UploadSingleTime(S3AdapterConnectionParameters connectionParameters protected override void Test(S3AdapterConnectionParameters connectionParameters, S3AdapterProcessorParameters processorParameters) { var fileName = Guid.NewGuid().ToString(); - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); using (var client = connectionParameters.CreateBucketConnection()) { - client.UploadAsync(Path.Combine(folder, fileName), new MemoryStream()).Wait(); + client.UploadAsync(StringEx.ConcatenatePath(folder, fileName), new MemoryStream()).Wait(); } using (var client = connectionParameters.CreateBucketConnection()) { - client.DeleteAsync(Path.Combine(folder, fileName)).Wait(); + client.DeleteAsync(StringEx.ConcatenatePath(folder, fileName)).Wait(); } } } diff --git a/src/Paillave.Etl.S3/S3FileValueProvider.cs b/src/Paillave.Etl.S3/S3FileValueProvider.cs index ee667d0d..0c599adf 100644 --- a/src/Paillave.Etl.S3/S3FileValueProvider.cs +++ b/src/Paillave.Etl.S3/S3FileValueProvider.cs @@ -28,7 +28,7 @@ protected override void Provide(Action pushFileValue, S3AdapterConne { var searchPattern = string.IsNullOrEmpty(providerParameters.FileNamePattern) ? "*" : providerParameters.FileNamePattern; var matcher = new Matcher().AddInclude(searchPattern); - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); var files = ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => GetFileList(connectionParameters, providerParameters)); foreach (var file in files) @@ -40,7 +40,7 @@ protected override void Provide(Action pushFileValue, S3AdapterConne } private List GetFileList(S3AdapterConnectionParameters connectionParameters, S3AdapterProviderParameters providerParameters) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); using (var client = connectionParameters.CreateBucketConnection()) return client.ListAsync(folder, providerParameters.Recursive).Result; } diff --git a/src/Paillave.Etl.Sftp/SftpConnectionInfo.ex.cs b/src/Paillave.Etl.Sftp/SftpConnectionInfo.ex.cs index 2bc5fd50..18e47fcd 100644 --- a/src/Paillave.Etl.Sftp/SftpConnectionInfo.ex.cs +++ b/src/Paillave.Etl.Sftp/SftpConnectionInfo.ex.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using Renci.SshNet; +using Paillave.Etl.Core; namespace Paillave.Etl.Sftp { diff --git a/src/Paillave.Etl.Sftp/SftpFileValue.cs b/src/Paillave.Etl.Sftp/SftpFileValue.cs index 3e16d4b3..72bc63d2 100644 --- a/src/Paillave.Etl.Sftp/SftpFileValue.cs +++ b/src/Paillave.Etl.Sftp/SftpFileValue.cs @@ -29,7 +29,7 @@ protected void DeleteFileSingleTime() using (var client = new SftpClient(connectionInfo)) { client.Connect(); - client.DeleteFile(Path.Combine(_folder, Name)); + client.DeleteFile(StringEx.ConcatenatePath(_folder, Name)); } } public override Stream GetContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, GetContentSingleTime); @@ -39,7 +39,7 @@ private Stream GetContentSingleTime() using (var client = new SftpClient(connectionInfo)) { client.Connect(); - return new MemoryStream(client.ReadAllBytes(Path.Combine(_folder, Name))); + return new MemoryStream(client.ReadAllBytes(StringEx.ConcatenatePath(_folder, Name))); } } private StreamWithResource OpenContentSingleTime() @@ -47,7 +47,7 @@ private StreamWithResource OpenContentSingleTime() var connectionInfo = _connectionInfo.CreateConnectionInfo(); var client = new SftpClient(connectionInfo); client.Connect(); - return new StreamWithResource(client.OpenRead(Path.Combine(_folder, Name)), client); + return new StreamWithResource(client.OpenRead(StringEx.ConcatenatePath(_folder, Name)), client); } public override StreamWithResource OpenContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, OpenContentSingleTime); diff --git a/src/Paillave.Etl.Sftp/SftpFileValueProcessor.cs b/src/Paillave.Etl.Sftp/SftpFileValueProcessor.cs index 643ef95d..7e37636b 100644 --- a/src/Paillave.Etl.Sftp/SftpFileValueProcessor.cs +++ b/src/Paillave.Etl.Sftp/SftpFileValueProcessor.cs @@ -14,7 +14,7 @@ public SftpFileValueProcessor(string code, string name, string connectionName, S public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; protected override void Process(IFileValue fileValue, SftpAdapterConnectionParameters connectionParameters, SftpAdapterProcessorParameters processorParameters, Action push, CancellationToken cancellationToken, IExecutionContext context) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); using var stream = fileValue.Get(processorParameters.UseStreamCopy); byte[] fileContents; using (MemoryStream ms = new MemoryStream()) @@ -22,7 +22,7 @@ protected override void Process(IFileValue fileValue, SftpAdapterConnectionParam stream.CopyTo(ms); fileContents = ms.ToArray(); } - ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, fileContents, Path.Combine(folder, fileValue.Name))); + ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, fileContents, StringEx.ConcatenatePath(folder, fileValue.Name))); push(fileValue); } private void UploadSingleTime(SftpAdapterConnectionParameters connectionParameters, byte[] fileContents, string filePath) @@ -39,7 +39,7 @@ private void UploadSingleTime(SftpAdapterConnectionParameters connectionParamete protected override void Test(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProcessorParameters processorParameters) { var fileName = Guid.NewGuid().ToString(); - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? ""); var connectionInfo = connectionParameters.CreateConnectionInfo(); using (var client = new SftpClient(connectionInfo)) { @@ -53,12 +53,12 @@ protected override void Test(SftpAdapterConnectionParameters connectionParameter fileContents = ms.ToArray(); } - client.WriteAllBytes(Path.Combine(folder, fileName), fileContents); + client.WriteAllBytes(StringEx.ConcatenatePath(folder, fileName), fileContents); } using (var client = new SftpClient(connectionInfo)) { client.Connect(); - client.DeleteFile(Path.Combine(folder, fileName)); + client.DeleteFile(StringEx.ConcatenatePath(folder, fileName)); } } } diff --git a/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs b/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs index 5e094399..12d2154e 100644 --- a/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs +++ b/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs @@ -1,54 +1,58 @@ using System; -using System.IO; using System.Threading; using Paillave.Etl.Core; using Microsoft.Extensions.FileSystemGlobbing; using Renci.SshNet; using System.Linq; -namespace Paillave.Etl.Sftp +namespace Paillave.Etl.Sftp; + +public class SftpFileValueProvider : FileValueProviderBase { - public class SftpFileValueProvider : FileValueProviderBase + public SftpFileValueProvider(string code, string name, string connectionName, SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) + : base(code, name, connectionName, connectionParameters, providerParameters) { } + public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy; + public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; + protected override void Provide(Action pushFileValue, SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters, CancellationToken cancellationToken, IExecutionContext context) { - public SftpFileValueProvider(string code, string name, string connectionName, SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) - : base(code, name, connectionName, connectionParameters, providerParameters) { } - public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy; - public override ProcessImpact MemoryFootPrint => ProcessImpact.Average; - protected override void Provide(Action pushFileValue, SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters, CancellationToken cancellationToken, IExecutionContext context) - { - var searchPattern = string.IsNullOrEmpty(providerParameters.FileNamePattern) ? "*" : providerParameters.FileNamePattern; - var matcher = new Matcher().AddInclude(searchPattern); - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); + var searchPattern = string.IsNullOrEmpty(providerParameters.FileNamePattern) ? "*" : providerParameters.FileNamePattern; + var matcher = new Matcher().AddInclude(searchPattern); + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); - var files = ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => GetFileList(connectionParameters, providerParameters)); - foreach (var file in files) - { - if (cancellationToken.IsCancellationRequested) break; - if (matcher.Match(file.Name).HasMatches) - pushFileValue(new SftpFileValue(connectionParameters, folder, file.Name, this.Code, this.Name, this.ConnectionName)); - } + var files = ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => GetFileList(connectionParameters, providerParameters)); + foreach (var file in files) + { + if (cancellationToken.IsCancellationRequested) break; + if (matcher.Match(file.Name).HasMatches) + pushFileValue(new SftpFileValue(connectionParameters, folder, file.Name, this.Code, this.Name, this.ConnectionName)); } - private Renci.SshNet.Sftp.ISftpFile[] GetFileList(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) + } + private static string ConcatenatePath(params string[] segments) + { + return string.Join("/", segments.Where(i => !string.IsNullOrWhiteSpace(i))).Replace("//", "/"); + } + private Renci.SshNet.Sftp.ISftpFile[] GetFileList(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) + { + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) + ? (providerParameters.SubFolder ?? "") + : ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); + var connectionInfo = connectionParameters.CreateConnectionInfo(); + using (var client = new SftpClient(connectionInfo)) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); - var connectionInfo = connectionParameters.CreateConnectionInfo(); - using (var client = new SftpClient(connectionInfo)) - { - client.Connect(); - return client.ListDirectory(folder).Where(i => !i.IsDirectory).ToArray(); - } + client.Connect(); + return client.ListDirectory(folder).Where(i => !i.IsDirectory).ToArray(); } - protected override void Test(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) + } + protected override void Test(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) + { + var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); + var searchPattern = string.IsNullOrEmpty(providerParameters.FileNamePattern) ? "*" : providerParameters.FileNamePattern; + var matcher = new Matcher().AddInclude(searchPattern); + var connectionInfo = connectionParameters.CreateConnectionInfo(); + using (var client = new SftpClient(connectionInfo)) { - var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); - var searchPattern = string.IsNullOrEmpty(providerParameters.FileNamePattern) ? "*" : providerParameters.FileNamePattern; - var matcher = new Matcher().AddInclude(searchPattern); - var connectionInfo = connectionParameters.CreateConnectionInfo(); - using (var client = new SftpClient(connectionInfo)) - { - client.Connect(); - client.ListDirectory(folder); - } + client.Connect(); + client.ListDirectory(folder); } } -} \ No newline at end of file +} diff --git a/src/Paillave.Etl.Sftp/String.ex.cs b/src/Paillave.Etl.Sftp/String.ex.cs deleted file mode 100644 index 350b2b42..00000000 --- a/src/Paillave.Etl.Sftp/String.ex.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.IO; - -namespace Paillave.Etl.Sftp -{ - public static class StringEx - { - public static Stream ToStream(this string s) - { - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(s); - writer.Flush(); - stream.Position = 0; - return stream; - } - } -} \ No newline at end of file diff --git a/src/Paillave.Etl.S3/String.ex.cs b/src/Paillave.Etl/Core/String.ex.cs similarity index 56% rename from src/Paillave.Etl.S3/String.ex.cs rename to src/Paillave.Etl/Core/String.ex.cs index 576aad02..4d3994a7 100644 --- a/src/Paillave.Etl.S3/String.ex.cs +++ b/src/Paillave.Etl/Core/String.ex.cs @@ -1,6 +1,8 @@ using System.IO; +using System.Linq; + +namespace Paillave.Etl.Core; -namespace Paillave.Etl.S3; public static class StringEx { public static Stream ToStream(this string s) @@ -12,4 +14,8 @@ public static Stream ToStream(this string s) stream.Position = 0; return stream; } -} \ No newline at end of file + public static string ConcatenatePath(params string[] segments) + { + return string.Join("/", segments.Where(i => !string.IsNullOrWhiteSpace(i))).Replace("//", "/"); + } +} diff --git a/src/SharedSettings.props b/src/SharedSettings.props index 8f76892d..754d1e62 100644 --- a/src/SharedSettings.props +++ b/src/SharedSettings.props @@ -2,7 +2,7 @@ latest enable - 2.2.2-beta + 2.2.4-beta NugetIcon.png README.md Stéphane Royer