Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: implement zip and unzip functionality with new classes and para… #523

Merged
merged 1 commit into from
Jan 16, 2025
Merged

feat: implement zip and unzip functionality with new classes and para… #523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 56 additions & 50 deletions src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,52 +1,58 @@
// using System;
// using System.Linq;
// using System.IO;
// using ICSharpCode.SharpZipLib.Zip;
// using Paillave.Etl.Core;
// using System.Threading;
// using Microsoft.Extensions.FileSystemGlobbing;
using System;
using System.Linq;
using System.IO;
using ICSharpCode.SharpZipLib.Zip;
using Paillave.Etl.Core;
using System.Threading;
using Microsoft.Extensions.FileSystemGlobbing;
using System.Collections.Generic;

// namespace Paillave.Etl.Zip
// {
// public class ZipFileProcessorValuesProvider : ValuesProviderBase<IFileValue, IFileValue>
// {
// private ZipFileProcessorParams _args;
// public ZipFileProcessorValuesProvider(UnzipFileProcessorParams args)
// => _args = args;
// public override ProcessImpact PerformanceImpact => ProcessImpact.Average;
// public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
// public override void PushValues(IFileValue input, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
// {
// var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations;
// if (cancellationToken.IsCancellationRequested) return;
// using var stream = input.Get(_args.UseStreamCopy);
// using var zf = new ZipFile(stream);
// var searchPattern = string.IsNullOrEmpty(_args.FileNamePattern) ? "*" : _args.FileNamePattern;
// var matcher = new Matcher().AddInclude(searchPattern);
namespace Paillave.Etl.Zip;

// if (!String.IsNullOrEmpty(_args.Password))
// zf.Password = _args.Password;
// var fileNames = zf.OfType<ZipEntry>().Where(i => i.IsFile && matcher.Match(Path.GetFileName(i.Name)).HasMatches).Select(i => i.Name).ToHashSet();
// foreach (ZipEntry zipEntry in zf)
// {
// if (cancellationToken.IsCancellationRequested) break;
// if (zipEntry.IsFile && matcher.Match(Path.GetFileName(zipEntry.Name)).HasMatches)
// {
// MemoryStream outputStream = new MemoryStream();
// using (var zipStream = zf.GetInputStream(zipEntry))
// zipStream.CopyTo(outputStream, 4096);
// outputStream.Seek(0, SeekOrigin.Begin);
// push(new UnzippedFileValue<UnzippedFileValueMetadata>(outputStream, zipEntry.Name, new UnzippedFileValueMetadata
// {
// ParentFileName = input.Name,
// ParentFileMetadata = input.Metadata,
// Destinations = destinations,
// ConnectorCode = input.Metadata.ConnectorCode,
// ConnectionName = input.Metadata.ConnectionName,
// ConnectorName = input.Metadata.ConnectorName
// }, input, fileNames, zipEntry.Name));
// }
// }
// }
// }
// }
public class ZipFileProcessorParams
{
public string Password { get; set; }
public bool UseStreamCopy { get; set; } = true;
}
public class ZippedFileValueMetadata : FileValueMetadataBase, IFileValueWithDestinationMetadata
{
public IFileValueMetadata ParentFileMetadata { get; set; }
public Dictionary<string, IEnumerable<Destination>> Destinations { get; set; }
}
public class ZipFileProcessorValuesProvider : ValuesProviderBase<IFileValue, IFileValue>
{
private ZipFileProcessorParams _args;
public ZipFileProcessorValuesProvider(ZipFileProcessorParams args)
=> _args = args;
public override ProcessImpact PerformanceImpact => ProcessImpact.Average;
public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
public override void PushValues(IFileValue input, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
{
var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations;
if (cancellationToken.IsCancellationRequested) return;
using var stream = input.Get(_args.UseStreamCopy);
var ms = new MemoryStream();
var fileName = $"{input.Name}.zip";
using (ZipOutputStream zipStream = new ZipOutputStream(ms))
{
if (!String.IsNullOrEmpty(_args.Password))
zipStream.Password = _args.Password;

var zipEntry = new ZipEntry(fileName)
{
DateTime = DateTime.Now,
IsUnicodeText = true
};

zipStream.PutNextEntry(zipEntry);
stream.CopyTo(zipStream);
zipStream.CloseEntry();
}
ms.Seek(0, SeekOrigin.Begin);
push(new ZippedFileValue<ZippedFileValueMetadata>(ms, input.Name, new ZippedFileValueMetadata
{
ParentFileMetadata = input.Metadata,
Destinations = destinations
}, input));
}
}
80 changes: 48 additions & 32 deletions src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,56 @@
using System.Threading;
using Paillave.Etl.Core;

namespace Paillave.Etl.Zip
namespace Paillave.Etl.Zip;

public class ZipAdapterConnectionParameters
{
public class ZipAdapterConnectionParameters
{
public string Password { get; set; }
}
public class ZipAdapterProcessorParameters
{
public string FileNamePattern { get; set; }
}
public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase<ZipAdapterConnectionParameters, object, ZipAdapterProcessorParameters>
{
public override string Description => "Handle zip files";
public override string Name => "Zip";
protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters)
=> null;
protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters)
=> new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters);
}
public class ZipFileValueProcessor : FileValueProcessorBase<ZipAdapterConnectionParameters, ZipAdapterProcessorParameters>
public string Password { get; set; }
}
public enum ZipDirection
{
Unzip,
Zip
}
public class ZipAdapterProcessorParameters
{
public ZipDirection Direction { get; set; } = ZipDirection.Unzip;
public string FileNamePattern { get; set; }
}
public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase<ZipAdapterConnectionParameters, object, ZipAdapterProcessorParameters>
{
public override string Description => "Handle zip files";
public override string Name => "Zip";
protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters)
=> null;
protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters)
=> new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters);
}
public class ZipFileValueProcessor : FileValueProcessorBase<ZipAdapterConnectionParameters, ZipAdapterProcessorParameters>
{
public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters)
: base(code, name, connectionName, connectionParameters, processorParameters) { }
public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy;
public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
{
public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters)
: base(code, name, connectionName, connectionParameters, processorParameters) { }
public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy;
public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
switch (processorParameters.Direction)
{
new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams
{
FileNamePattern = processorParameters.FileNamePattern,
Password = connectionParameters.Password
}).PushValues(fileValue, push, cancellationToken, context);
case ZipDirection.Unzip:
new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams
{
FileNamePattern = processorParameters.FileNamePattern,
Password = connectionParameters.Password
}).PushValues(fileValue, push, cancellationToken, context);
break;
case ZipDirection.Zip:
new ZipFileProcessorValuesProvider(new ZipFileProcessorParams
{
Password = connectionParameters.Password
}).PushValues(fileValue, push, cancellationToken, context);
break;
}

protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { }
}
}

protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { }
}
34 changes: 34 additions & 0 deletions src/Paillave.Etl.Zip/ZippedFileValue.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
using System.IO;
using Paillave.Etl.Core;

namespace Paillave.Etl.Zip
{
public class ZippedFileValue<TMetadata> : FileValueBase<TMetadata> where TMetadata : IFileValueMetadata
{
private readonly Stream _stream;
private readonly IFileValue _underlyingFileValue;
public override string Name { get; }
public ZippedFileValue(Stream stream, string name, TMetadata metadata, IFileValue underlyingFileValue)
: base(metadata)
=> (_stream, Name, _underlyingFileValue)
= (stream, name, underlyingFileValue);
public override Stream GetContent()
{
var ms = new MemoryStream();
_stream.Seek(0, SeekOrigin.Begin);
_stream.CopyTo(ms);
ms.Seek(0, SeekOrigin.Begin);
return ms;
}
protected override void DeleteFile()
{
_underlyingFileValue.Delete();
}

public override StreamWithResource OpenContent()
{
_stream.Seek(0, SeekOrigin.Begin);
return new StreamWithResource(_stream);
}
}
}
2 changes: 1 addition & 1 deletion src/SharedSettings.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<PropertyGroup>
<LangVersion>latest</LangVersion>
<Nullable>enable</Nullable>
<Version>2.2.5-beta</Version>
<Version>2.2.6-beta</Version>
<PackageIcon>NugetIcon.png</PackageIcon>
<PackageReadmeFile>README.md</PackageReadmeFile>
<Authors>Stéphane Royer</Authors>
Expand Down
Loading