Skip to content

Commit eff79b4

Browse files
paillavepaillave
authored
Merge pull request #523 from paillave/dotnet9-migration
feat: implement zip and unzip functionality with new classes and para…
2 parents 13dc402 + d779259 commit eff79b4

File tree

4 files changed

+139
-83
lines changed

4 files changed

+139
-83
lines changed

‎src/Paillave.Etl.Zip/ZipFileProcessorValuesProvider.cs

Lines changed: 56 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,58 @@
1-
// using System;
2-
// using System.Linq;
3-
// using System.IO;
4-
// using ICSharpCode.SharpZipLib.Zip;
5-
// using Paillave.Etl.Core;
6-
// using System.Threading;
7-
// using Microsoft.Extensions.FileSystemGlobbing;
1+
using System;
2+
using System.Linq;
3+
using System.IO;
4+
using ICSharpCode.SharpZipLib.Zip;
5+
using Paillave.Etl.Core;
6+
using System.Threading;
7+
using Microsoft.Extensions.FileSystemGlobbing;
8+
using System.Collections.Generic;
89

9-
// namespace Paillave.Etl.Zip
10-
// {
11-
// public class ZipFileProcessorValuesProvider : ValuesProviderBase<IFileValue, IFileValue>
12-
// {
13-
// private ZipFileProcessorParams _args;
14-
// public ZipFileProcessorValuesProvider(UnzipFileProcessorParams args)
15-
// => _args = args;
16-
// public override ProcessImpact PerformanceImpact => ProcessImpact.Average;
17-
// public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
18-
// public override void PushValues(IFileValue input, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
19-
// {
20-
// var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations;
21-
// if (cancellationToken.IsCancellationRequested) return;
22-
// using var stream = input.Get(_args.UseStreamCopy);
23-
// using var zf = new ZipFile(stream);
24-
// var searchPattern = string.IsNullOrEmpty(_args.FileNamePattern) ? "*" : _args.FileNamePattern;
25-
// var matcher = new Matcher().AddInclude(searchPattern);
10+
namespace Paillave.Etl.Zip;
2611

27-
// if (!String.IsNullOrEmpty(_args.Password))
28-
// zf.Password = _args.Password;
29-
// var fileNames = zf.OfType<ZipEntry>().Where(i => i.IsFile && matcher.Match(Path.GetFileName(i.Name)).HasMatches).Select(i => i.Name).ToHashSet();
30-
// foreach (ZipEntry zipEntry in zf)
31-
// {
32-
// if (cancellationToken.IsCancellationRequested) break;
33-
// if (zipEntry.IsFile && matcher.Match(Path.GetFileName(zipEntry.Name)).HasMatches)
34-
// {
35-
// MemoryStream outputStream = new MemoryStream();
36-
// using (var zipStream = zf.GetInputStream(zipEntry))
37-
// zipStream.CopyTo(outputStream, 4096);
38-
// outputStream.Seek(0, SeekOrigin.Begin);
39-
// push(new UnzippedFileValue<UnzippedFileValueMetadata>(outputStream, zipEntry.Name, new UnzippedFileValueMetadata
40-
// {
41-
// ParentFileName = input.Name,
42-
// ParentFileMetadata = input.Metadata,
43-
// Destinations = destinations,
44-
// ConnectorCode = input.Metadata.ConnectorCode,
45-
// ConnectionName = input.Metadata.ConnectionName,
46-
// ConnectorName = input.Metadata.ConnectorName
47-
// }, input, fileNames, zipEntry.Name));
48-
// }
49-
// }
50-
// }
51-
// }
52-
// }
12+
public class ZipFileProcessorParams
13+
{
14+
public string Password { get; set; }
15+
public bool UseStreamCopy { get; set; } = true;
16+
}
17+
public class ZippedFileValueMetadata : FileValueMetadataBase, IFileValueWithDestinationMetadata
18+
{
19+
public IFileValueMetadata ParentFileMetadata { get; set; }
20+
public Dictionary<string, IEnumerable<Destination>> Destinations { get; set; }
21+
}
22+
public class ZipFileProcessorValuesProvider : ValuesProviderBase<IFileValue, IFileValue>
23+
{
24+
private ZipFileProcessorParams _args;
25+
public ZipFileProcessorValuesProvider(ZipFileProcessorParams args)
26+
=> _args = args;
27+
public override ProcessImpact PerformanceImpact => ProcessImpact.Average;
28+
public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
29+
public override void PushValues(IFileValue input, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
30+
{
31+
var destinations = (input.Metadata as IFileValueWithDestinationMetadata)?.Destinations;
32+
if (cancellationToken.IsCancellationRequested) return;
33+
using var stream = input.Get(_args.UseStreamCopy);
34+
var ms = new MemoryStream();
35+
var fileName = $"{input.Name}.zip";
36+
using (ZipOutputStream zipStream = new ZipOutputStream(ms))
37+
{
38+
if (!String.IsNullOrEmpty(_args.Password))
39+
zipStream.Password = _args.Password;
40+
41+
var zipEntry = new ZipEntry(fileName)
42+
{
43+
DateTime = DateTime.Now,
44+
IsUnicodeText = true
45+
};
46+
47+
zipStream.PutNextEntry(zipEntry);
48+
stream.CopyTo(zipStream);
49+
zipStream.CloseEntry();
50+
}
51+
ms.Seek(0, SeekOrigin.Begin);
52+
push(new ZippedFileValue<ZippedFileValueMetadata>(ms, input.Name, new ZippedFileValueMetadata
53+
{
54+
ParentFileMetadata = input.Metadata,
55+
Destinations = destinations
56+
}, input));
57+
}
58+
}

‎src/Paillave.Etl.Zip/ZipProviderProcessorAdapter.cs

Lines changed: 48 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,40 +2,56 @@
22
using System.Threading;
33
using Paillave.Etl.Core;
44

5-
namespace Paillave.Etl.Zip
5+
namespace Paillave.Etl.Zip;
6+
7+
public class ZipAdapterConnectionParameters
68
{
7-
public class ZipAdapterConnectionParameters
8-
{
9-
public string Password { get; set; }
10-
}
11-
public class ZipAdapterProcessorParameters
12-
{
13-
public string FileNamePattern { get; set; }
14-
}
15-
public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase<ZipAdapterConnectionParameters, object, ZipAdapterProcessorParameters>
16-
{
17-
public override string Description => "Handle zip files";
18-
public override string Name => "Zip";
19-
protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters)
20-
=> null;
21-
protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters)
22-
=> new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters);
23-
}
24-
public class ZipFileValueProcessor : FileValueProcessorBase<ZipAdapterConnectionParameters, ZipAdapterProcessorParameters>
9+
public string Password { get; set; }
10+
}
11+
public enum ZipDirection
12+
{
13+
Unzip,
14+
Zip
15+
}
16+
public class ZipAdapterProcessorParameters
17+
{
18+
public ZipDirection Direction { get; set; } = ZipDirection.Unzip;
19+
public string FileNamePattern { get; set; }
20+
}
21+
public class ZipProviderProcessorAdapter : ProviderProcessorAdapterBase<ZipAdapterConnectionParameters, object, ZipAdapterProcessorParameters>
22+
{
23+
public override string Description => "Handle zip files";
24+
public override string Name => "Zip";
25+
protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, object inputParameters)
26+
=> null;
27+
protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters outputParameters)
28+
=> new ZipFileValueProcessor(code, name, connectionName, connectionParameters, outputParameters);
29+
}
30+
public class ZipFileValueProcessor : FileValueProcessorBase<ZipAdapterConnectionParameters, ZipAdapterProcessorParameters>
31+
{
32+
public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters)
33+
: base(code, name, connectionName, connectionParameters, processorParameters) { }
34+
public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy;
35+
public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
36+
protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
2537
{
26-
public ZipFileValueProcessor(string code, string name, string connectionName, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters)
27-
: base(code, name, connectionName, connectionParameters, processorParameters) { }
28-
public override ProcessImpact PerformanceImpact => ProcessImpact.Heavy;
29-
public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
30-
protected override void Process(IFileValue fileValue, ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
38+
switch (processorParameters.Direction)
3139
{
32-
new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams
33-
{
34-
FileNamePattern = processorParameters.FileNamePattern,
35-
Password = connectionParameters.Password
36-
}).PushValues(fileValue, push, cancellationToken, context);
40+
case ZipDirection.Unzip:
41+
new UnzipFileProcessorValuesProvider(new UnzipFileProcessorParams
42+
{
43+
FileNamePattern = processorParameters.FileNamePattern,
44+
Password = connectionParameters.Password
45+
}).PushValues(fileValue, push, cancellationToken, context);
46+
break;
47+
case ZipDirection.Zip:
48+
new ZipFileProcessorValuesProvider(new ZipFileProcessorParams
49+
{
50+
Password = connectionParameters.Password
51+
}).PushValues(fileValue, push, cancellationToken, context);
52+
break;
3753
}
38-
39-
protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { }
4054
}
41-
}
55+
56+
protected override void Test(ZipAdapterConnectionParameters connectionParameters, ZipAdapterProcessorParameters processorParameters) { }
57+
}

‎src/Paillave.Etl.Zip/ZippedFileValue.cs

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using System.IO;
2+
using Paillave.Etl.Core;
3+
4+
namespace Paillave.Etl.Zip
5+
{
6+
public class ZippedFileValue<TMetadata> : FileValueBase<TMetadata> where TMetadata : IFileValueMetadata
7+
{
8+
private readonly Stream _stream;
9+
private readonly IFileValue _underlyingFileValue;
10+
public override string Name { get; }
11+
public ZippedFileValue(Stream stream, string name, TMetadata metadata, IFileValue underlyingFileValue)
12+
: base(metadata)
13+
=> (_stream, Name, _underlyingFileValue)
14+
= (stream, name, underlyingFileValue);
15+
public override Stream GetContent()
16+
{
17+
var ms = new MemoryStream();
18+
_stream.Seek(0, SeekOrigin.Begin);
19+
_stream.CopyTo(ms);
20+
ms.Seek(0, SeekOrigin.Begin);
21+
return ms;
22+
}
23+
protected override void DeleteFile()
24+
{
25+
_underlyingFileValue.Delete();
26+
}
27+
28+
public override StreamWithResource OpenContent()
29+
{
30+
_stream.Seek(0, SeekOrigin.Begin);
31+
return new StreamWithResource(_stream);
32+
}
33+
}
34+
}

‎src/SharedSettings.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<PropertyGroup>
33
<LangVersion>latest</LangVersion>
44
<Nullable>enable</Nullable>
5-
<Version>2.2.5-beta</Version>
5+
<Version>2.2.6-beta</Version>
66
<PackageIcon>NugetIcon.png</PackageIcon>
77
<PackageReadmeFile>README.md</PackageReadmeFile>
88
<Authors>Stéphane Royer</Authors>

0 commit comments

Comments
 (0)