Skip to content

Commit b24198e

Browse files
Jack-EdwardsJack-Edwards
committed
Improve the multi-part request optimization algorithm
1 parent 5614e87 commit b24198e

File tree

8 files changed

+57
-32
lines changed

8 files changed

+57
-32
lines changed

‎Crypter.Common.Client/Crypter.Common.Client.csproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,8 @@
1313
<ProjectReference Include="..\Crypter.Crypto.Common\Crypter.Crypto.Common.csproj" />
1414
</ItemGroup>
1515

16+
<ItemGroup>
17+
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0" />
18+
</ItemGroup>
19+
1620
</Project>

‎Crypter.Common.Client/Transfer/Handlers/Base/UploadHandler.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,7 @@ public class UploadHandler : IUserUploadHandler
5353

5454
protected Maybe<byte[]> RecipientPublicKey = Maybe<byte[]>.None;
5555

56-
protected UploadHandler(ICrypterApiClient crypterApiClient, ICryptoProvider cryptoProvider,
57-
ClientTransferSettings clientTransferSettings)
56+
protected UploadHandler(ICrypterApiClient crypterApiClient, ICryptoProvider cryptoProvider, ClientTransferSettings clientTransferSettings)
5857
{
5958
CrypterApiClient = crypterApiClient;
6059
CryptoProvider = cryptoProvider;

‎Crypter.Common.Client/Transfer/Handlers/UploadFileHandler.cs

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
using Crypter.Crypto.Common;
4242
using Crypter.Crypto.Common.StreamEncryption;
4343
using EasyMonads;
44+
using Microsoft.Extensions.Logging;
4445

4546
namespace Crypter.Common.Client.Transfer.Handlers;
4647

@@ -51,11 +52,12 @@ public class UploadFileHandler : UploadHandler
5152
private long _fileSize;
5253
private string? _fileContentType;
5354
private bool _transferInfoSet;
55+
private readonly ILogger<UploadFileHandler> _logger;
5456

55-
public UploadFileHandler(ICrypterApiClient crypterApiClient, ICryptoProvider cryptoProvider,
56-
ClientTransferSettings clientTransferSettings)
57+
public UploadFileHandler(ICrypterApiClient crypterApiClient, ICryptoProvider cryptoProvider, ClientTransferSettings clientTransferSettings, ILogger<UploadFileHandler> logger)
5758
: base(crypterApiClient, cryptoProvider, clientTransferSettings)
5859
{
60+
_logger = logger;
5961
}
6062

6163
internal void SetTransferInfo(Func<Stream> fileStreamOpener, string fileName, long fileSize, string fileContentType, int expirationHours)
@@ -99,7 +101,7 @@ public Task<Either<UploadTransferError, UploadHandlerResponse>> UploadAsync(Acti
99101
{
100102
MaxDegreeOfParallelism = ClientTransferSettings.MaximumMultipartParallelism
101103
};
102-
SemaphoreSlim uploadLock = new SemaphoreSlim(1);
104+
SemaphoreSlim readLock = new SemaphoreSlim(1);
103105
bool fault = false;
104106
int currentPosition = 0;
105107
await Parallel.ForEachAsync(asyncEnumerable, parallelOptions, async (streamOpener, _) =>
@@ -108,7 +110,7 @@ await Parallel.ForEachAsync(asyncEnumerable, parallelOptions, async (streamOpene
108110
{
109111
try
110112
{
111-
await uploadLock.WaitAsync(CancellationToken.None);
113+
await readLock.WaitAsync(CancellationToken.None);
112114
Task<Either<UploadMultipartFileTransferError, Unit>> uploadTask = CrypterApiClient.FileTransfer
113115
.UploadMultipartFileTransferAsync(initializeResult.HashId, currentPosition, streamOpener)
114116
.ContinueWith(x =>
@@ -122,7 +124,7 @@ await Parallel.ForEachAsync(asyncEnumerable, parallelOptions, async (streamOpene
122124
}, CancellationToken.None);
123125
indexedUploadResults.Add(currentPosition, uploadTask);
124126
currentPosition++;
125-
uploadLock.Release();
127+
readLock.Release();
126128
await uploadTask;
127129
}
128130
catch (Exception)
@@ -161,7 +163,7 @@ async IAsyncEnumerable<Func<MemoryStream>> SplitEncryptionStreamAsync(Encryption
161163
{
162164
bool endOfStream = false;
163165
Stopwatch loopStopwatch = new Stopwatch();
164-
short blocksPerRequest = ClientTransferSettings.InitialMultipartReadBlocks;
166+
int blocksPerRequest = ClientTransferSettings.InitialMultipartReadBlocks;
165167
do
166168
{
167169
loopStopwatch.Restart();
@@ -191,15 +193,16 @@ async IAsyncEnumerable<Func<MemoryStream>> SplitEncryptionStreamAsync(Encryption
191193
ArrayPool<byte>.Shared.Return(buffer, clearArray: true);
192194
}
193195
}
194-
195-
if (loopStopwatch.Elapsed < TimeSpan.FromSeconds(1) && blocksPerRequest < ClientTransferSettings.MaximumMultipartReadBlocks)
196-
{
197-
blocksPerRequest += 5;
198-
}
199-
else if (loopStopwatch.Elapsed > TimeSpan.FromSeconds(1) && blocksPerRequest > ClientTransferSettings.InitialMultipartReadBlocks)
200-
{
201-
blocksPerRequest -= 5;
202-
}
196+
197+
double factor = ClientTransferSettings.TargetMultipartUploadMilliseconds / loopStopwatch.Elapsed.TotalMilliseconds;
198+
_logger.LogDebug("Factor: {factor}, TargetMultipartUploadMilliseconds: {target}, ElapsedMilliseconds: {elapsed}", factor, ClientTransferSettings.TargetMultipartUploadMilliseconds, loopStopwatch.Elapsed.TotalMilliseconds);
199+
200+
int optimalBlocksPerRequest = Convert.ToInt32(blocksPerRequest * factor);
201+
_logger.LogDebug("Optimal blocks per request: {optimalValue}", optimalBlocksPerRequest);
202+
203+
int lowerBound = Math.Max(ClientTransferSettings.InitialMultipartReadBlocks, optimalBlocksPerRequest);
204+
blocksPerRequest = Math.Min(ClientTransferSettings.MaximumMultipartReadBlocks, lowerBound);
205+
_logger.LogDebug("Blocks per next request: {blocks}", blocksPerRequest);
203206
} while (!endOfStream);
204207
}
205208
}

‎Crypter.Common.Client/Transfer/Handlers/UploadMessageHandler.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
using Crypter.Crypto.Common;
3737
using Crypter.Crypto.Common.StreamEncryption;
3838
using EasyMonads;
39+
using Microsoft.Extensions.Logging;
3940

4041
namespace Crypter.Common.Client.Transfer.Handlers;
4142

@@ -45,11 +46,12 @@ public class UploadMessageHandler : UploadHandler
4546
private string? _messageSubject;
4647
private int _messageSize;
4748
private bool _transferInfoSet;
49+
private readonly ILogger<UploadMessageHandler> _logger;
4850

49-
public UploadMessageHandler(ICrypterApiClient crypterApiClient, ICryptoProvider cryptoProvider,
50-
ClientTransferSettings clientTransferSettings)
51+
public UploadMessageHandler(ICrypterApiClient crypterApiClient, ICryptoProvider cryptoProvider, ClientTransferSettings clientTransferSettings, ILogger<UploadMessageHandler> logger)
5152
: base(crypterApiClient, cryptoProvider, clientTransferSettings)
5253
{
54+
_logger = logger;
5355
}
5456

5557
internal void SetTransferInfo(string messageSubject, string messageBody, int expirationHours)

‎Crypter.Common.Client/Transfer/Models/ClientTransferSettings.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ public class ClientTransferSettings
5858
/// </summary>
5959
public int MaximumMultipartParallelism { get; init; }
6060

61+
/// <summary>
62+
/// Set the number of seconds the adaptive multipart upload algorithm will target for individual upload requests.
63+
/// </summary>
64+
public int TargetMultipartUploadMilliseconds { get; init; }
65+
6166
/// <summary>
6267
/// Set the number of bytes of plaintext to read at one time when encrypting a file or message.
6368
/// </summary>

‎Crypter.Common.Client/Transfer/TransferHandlerFactory.cs

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (C) 2023 Crypter File Transfer
2+
* Copyright (C) 2024 Crypter File Transfer
33
*
44
* This file is part of the Crypter file transfer project.
55
*
@@ -32,6 +32,7 @@
3232
using Crypter.Common.Client.Transfer.Models;
3333
using Crypter.Common.Enums;
3434
using Crypter.Crypto.Common;
35+
using Microsoft.Extensions.Logging;
3536

3637
namespace Crypter.Common.Client.Transfer;
3738

@@ -41,44 +42,42 @@ public class TransferHandlerFactory
4142
private readonly ICryptoProvider _cryptoProvider;
4243
private readonly IUserSessionService _userSessionService;
4344
private readonly ClientTransferSettings _clientTransferSettings;
45+
private readonly ILoggerFactory _loggerFactory;
4446

4547
public TransferHandlerFactory(ICrypterApiClient crypterApiClient, ICryptoProvider cryptoProvider,
46-
IUserSessionService userSessionService, ClientTransferSettings clientTransferSettings)
48+
IUserSessionService userSessionService, ClientTransferSettings clientTransferSettings, ILoggerFactory loggerFactory)
4749
{
4850
_crypterApiClient = crypterApiClient;
4951
_cryptoProvider = cryptoProvider;
5052
_userSessionService = userSessionService;
5153
_clientTransferSettings = clientTransferSettings;
54+
_loggerFactory = loggerFactory;
5255
}
5356

54-
public UploadFileHandler CreateUploadFileHandler(Func<Stream> fileStreamOpener, string fileName, long fileSize,
55-
string fileContentType, int expirationHours)
57+
public UploadFileHandler CreateUploadFileHandler(Func<Stream> fileStreamOpener, string fileName, long fileSize, string fileContentType, int expirationHours)
5658
{
57-
var handler = new UploadFileHandler(_crypterApiClient, _cryptoProvider, _clientTransferSettings);
59+
UploadFileHandler handler = new UploadFileHandler(_crypterApiClient, _cryptoProvider, _clientTransferSettings, _loggerFactory.CreateLogger<UploadFileHandler>());
5860
handler.SetTransferInfo(fileStreamOpener, fileName, fileSize, fileContentType, expirationHours);
5961
return handler;
6062
}
6163

62-
public UploadMessageHandler CreateUploadMessageHandler(string messageSubject, string messageBody,
63-
int expirationHours)
64+
public UploadMessageHandler CreateUploadMessageHandler(string messageSubject, string messageBody, int expirationHours)
6465
{
65-
var handler = new UploadMessageHandler(_crypterApiClient, _cryptoProvider, _clientTransferSettings);
66+
UploadMessageHandler handler = new UploadMessageHandler(_crypterApiClient, _cryptoProvider, _clientTransferSettings, _loggerFactory.CreateLogger<UploadMessageHandler>());
6667
handler.SetTransferInfo(messageSubject, messageBody, expirationHours);
6768
return handler;
6869
}
6970

7071
public DownloadFileHandler CreateDownloadFileHandler(string hashId, TransferUserType userType)
7172
{
72-
var handler =
73-
new DownloadFileHandler(_crypterApiClient, _cryptoProvider, _userSessionService, _clientTransferSettings);
73+
DownloadFileHandler handler = new DownloadFileHandler(_crypterApiClient, _cryptoProvider, _userSessionService, _clientTransferSettings);
7474
handler.SetTransferInfo(hashId, userType);
7575
return handler;
7676
}
7777

7878
public DownloadMessageHandler CreateDownloadMessageHandler(string hashId, TransferUserType userType)
7979
{
80-
var handler =
81-
new DownloadMessageHandler(_crypterApiClient, _cryptoProvider, _userSessionService, _clientTransferSettings);
80+
DownloadMessageHandler handler = new DownloadMessageHandler(_crypterApiClient, _cryptoProvider, _userSessionService, _clientTransferSettings);
8281
handler.SetTransferInfo(hashId, userType);
8382
return handler;
8483
}

‎Crypter.Web/wwwroot/appsettings.Development.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
{
22
"ApiSettings": {
33
"ApiBaseUrl": "https://localhost/api"
4+
},
5+
"Logging": {
6+
"LogLevel": {
7+
"Default": "Debug",
8+
"Microsoft.AspNetCore": "Information"
9+
}
410
}
511
}

‎Crypter.Web/wwwroot/appsettings.json

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,15 @@
88
"MaximumMultipartUploadSizeMB": 250,
99
"MaximumMultipartReadBlocks": 120,
1010
"InitialMultipartReadBlocks": 10,
11-
"MaximumMultipartParallelism": 1,
11+
"MaximumMultipartParallelism": 2,
12+
"TargetMultipartUploadMilliseconds": 1000,
1213
"MaxReadSize": 32704,
1314
"PadSize": 64
15+
},
16+
"Logging": {
17+
"LogLevel": {
18+
"Default": "Information",
19+
"Microsoft.AspNetCore": "Warning"
20+
}
1421
}
1522
}

0 commit comments

Comments
 (0)