diff --git a/Crypter.API/Controllers/FileTransferController.cs b/Crypter.API/Controllers/FileTransferController.cs index 608c59a35..94f268557 100644 --- a/Crypter.API/Controllers/FileTransferController.cs +++ b/Crypter.API/Controllers/FileTransferController.cs @@ -113,9 +113,9 @@ public async Task InitializeMultipartFileTransferAsync([FromQuery [ProducesResponseType(StatusCodes.Status400BadRequest, Type = typeof(ErrorResponse))] [ProducesResponseType(StatusCodes.Status404NotFound, Type = typeof(ErrorResponse))] public async Task UploadMultipartFileTransferAsync([FromQuery] string id, [FromQuery] int position, - [FromForm] IFormFile? cipertext) + [FromForm] IFormFile? ciphertext) { - SaveMultipartFileTransferCommand command = new SaveMultipartFileTransferCommand(UserId, id, position, cipertext?.OpenReadStream()); + SaveMultipartFileTransferCommand command = new SaveMultipartFileTransferCommand(UserId, id, position, ciphertext?.OpenReadStream()); return await _sender.Send(command) .MatchAsync( left: MakeErrorResponse, diff --git a/Crypter.Common.Client/Transfer/Handlers/Base/UploadHandler.cs b/Crypter.Common.Client/Transfer/Handlers/Base/UploadHandler.cs index 0f0a17a7a..3cbb93297 100644 --- a/Crypter.Common.Client/Transfer/Handlers/Base/UploadHandler.cs +++ b/Crypter.Common.Client/Transfer/Handlers/Base/UploadHandler.cs @@ -28,7 +28,6 @@ using System.IO; using Crypter.Common.Client.Interfaces.HttpClients; using Crypter.Common.Client.Transfer.Models; -using Crypter.Common.Enums; using Crypter.Crypto.Common; using Crypter.Crypto.Common.KeyExchange; using Crypter.Crypto.Common.StreamEncryption; diff --git a/Crypter.Common.Client/Transfer/Handlers/UploadFileHandler.cs b/Crypter.Common.Client/Transfer/Handlers/UploadFileHandler.cs index b95e1aeba..de0c092d9 100644 --- a/Crypter.Common.Client/Transfer/Handlers/UploadFileHandler.cs +++ b/Crypter.Common.Client/Transfer/Handlers/UploadFileHandler.cs @@ -27,7 +27,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading.Tasks; using Crypter.Common.Client.Interfaces.HttpClients; using Crypter.Common.Client.Transfer.Handlers.Base; @@ -66,7 +65,7 @@ internal void SetTransferInfo(Func fileStreamOpener, string fileName, lo _transferInfoSet = true; } - public Task> UploadAsync(Action? updateCallback = null, bool multipart = false) + public Task> UploadAsync(Action? updateCallback = null) { if (!_transferInfoSet) { @@ -74,12 +73,12 @@ public Task> UploadAsync(Acti .From(UploadTransferError.UnknownError) .AsTask(); } - + (Func?, EncryptionStream> encryptionStreamOpener, byte[]? senderPublicKey, byte[] proof) = GetEncryptionInfo(_fileStreamOpener!, _fileSize); UploadFileTransferRequest request = new UploadFileTransferRequest(_fileName!, _fileContentType!, senderPublicKey, KeyExchangeNonce, proof, ExpirationHours); - if (multipart) + if (SenderDefined) { // Initialize return CrypterApiClient.FileTransfer @@ -89,21 +88,22 @@ public Task> UploadAsync(Acti // Upload .BindAsync(async initializeResult => { - Either uploadResult = - Either.Neither; + Either uploadResult = Either.Neither; EncryptionStream encryptionStream = encryptionStreamOpener(updateCallback); - long maximumReadLength = ClientTransferSettings.MaximumMultipartUploadPartSizeMB * - Convert.ToInt64(Math.Pow(10, 6)); - foreach (var iterable in SplitEncryptionStream(encryptionStream, maximumReadLength) - .Select((x, y) => new { StreamOpener = x, Index = y })) + long maximumReadLength = ClientTransferSettings.MaximumMultipartUploadPartSizeMB * Convert.ToInt64(Math.Pow(10, 6)); + + IAsyncEnumerator> enumerable = SplitEncryptionStreamAsync(encryptionStream, maximumReadLength).GetAsyncEnumerator(); + int currentPosition = 0; + while (await enumerable.MoveNextAsync()) { uploadResult = await CrypterApiClient.FileTransfer.UploadMultipartFileTransferAsync( - initializeResult.HashId, - iterable.Index, iterable.StreamOpener); + initializeResult.HashId, currentPosition, enumerable.Current); if (!uploadResult.IsRight) { break; } + + currentPosition++; } return await uploadResult @@ -125,23 +125,19 @@ public Task> UploadAsync(Acti RecipientKeySeed)); } - IEnumerable> SplitEncryptionStream(EncryptionStream encryptionStream, long maximumReadLength) + async IAsyncEnumerable> SplitEncryptionStreamAsync(EncryptionStream encryptionStream, long maximumReadLength) { - int bytesRead = 0; + bool endOfStream; do { byte[] buffer = new byte[maximumReadLength]; - bytesRead = encryptionStream.Read(buffer); - if (bytesRead > 0) + int bytesRead = await encryptionStream.ReadAsync(buffer); + endOfStream = bytesRead == 0; + if (!endOfStream) { - yield return () => - { - MemoryStream memoryStream = new MemoryStream(); - memoryStream.Write(buffer, 0, bytesRead); - return memoryStream; - }; + yield return () => new MemoryStream(buffer, 0, bytesRead); } - } while (bytesRead > 0); + } while (!endOfStream); } } } diff --git a/Crypter.Core/DependencyInjection.cs b/Crypter.Core/DependencyInjection.cs index 518b93e5d..9f73f15c0 100644 --- a/Crypter.Core/DependencyInjection.cs +++ b/Crypter.Core/DependencyInjection.cs @@ -24,7 +24,6 @@ * Contact the current copyright holder to discuss commercial license options. */ -using System; using System.Threading.Tasks; using Crypter.Common.Exceptions; using Crypter.Core.Identity; @@ -37,7 +36,6 @@ using Crypter.DataAccess; using Hangfire; using Hangfire.PostgreSql; -using MediatR; using Microsoft.AspNetCore.Builder; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; @@ -108,6 +106,7 @@ public static IServiceCollection AddCrypterCore(this IServiceCollection services { options.AllocatedGB = transferStorageSettings.AllocatedGB; options.Location = transferStorageSettings.Location; + options.MaximumTransferSizeMB = transferStorageSettings.MaximumTransferSizeMB; }); services.AddHangfire(config => config diff --git a/Crypter.Core/Features/Transfer/Commands/AbandonMultipartFileTransferCommand.cs b/Crypter.Core/Features/Transfer/Commands/AbandonMultipartFileTransferCommand.cs index c7b854a87..c5de74c05 100644 --- a/Crypter.Core/Features/Transfer/Commands/AbandonMultipartFileTransferCommand.cs +++ b/Crypter.Core/Features/Transfer/Commands/AbandonMultipartFileTransferCommand.cs @@ -25,7 +25,6 @@ */ using System; -using System.Data; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -70,12 +69,10 @@ public AbandonMultipartFileTransferCommandHandler( public async Task> Handle(AbandonMultipartFileTransferCommand request, CancellationToken cancellationToken) { - await using IDbContextTransaction transaction = await _dataContext.Database - .BeginTransactionAsync(IsolationLevel.Serializable, CancellationToken.None); - - try + DateTimeOffset utcNow = DateTimeOffset.UtcNow; + IExecutionStrategy executionStrategy = _dataContext.Database.CreateExecutionStrategy(); + return await executionStrategy.ExecuteAsync(async () => { - DateTimeOffset utcNow = DateTimeOffset.UtcNow; Task> responseTask = from additionalData in ValidateRequestAsync(request) from abandonResult in Either.FromRightAsync( @@ -100,14 +97,11 @@ from sideEffects in Either.FromRightAsy async () => { FailedMultipartFileTransferAbandonEvent failedMultipartAbandonEvent = - new FailedMultipartFileTransferAbandonEvent(request.HashId, request.SenderId, AbandonMultipartFileTransferError.UnknownError, utcNow); + new FailedMultipartFileTransferAbandonEvent(request.HashId, request.SenderId, + AbandonMultipartFileTransferError.UnknownError, utcNow); await _publisher.Publish(failedMultipartAbandonEvent, CancellationToken.None); }); - } - finally - { - await transaction.CommitAsync(CancellationToken.None); - } + }); } private async Task> ValidateRequestAsync(AbandonMultipartFileTransferCommand request) diff --git a/Crypter.Core/Features/Transfer/Commands/FinalizeMultipartFileTransferCommand.cs b/Crypter.Core/Features/Transfer/Commands/FinalizeMultipartFileTransferCommand.cs index a5ae95387..8d1b60d69 100644 --- a/Crypter.Core/Features/Transfer/Commands/FinalizeMultipartFileTransferCommand.cs +++ b/Crypter.Core/Features/Transfer/Commands/FinalizeMultipartFileTransferCommand.cs @@ -25,7 +25,6 @@ */ using System; -using System.Data; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -71,15 +70,13 @@ public FinalizeMultipartFileTransferCommandHandler( public async Task> Handle(FinalizeMultipartFileTransferCommand request, CancellationToken cancellationToken) { - await using IDbContextTransaction transaction = await _dataContext.Database - .BeginTransactionAsync(IsolationLevel.Serializable, CancellationToken.None); - - try + DateTimeOffset utcNow = DateTimeOffset.UtcNow; + IExecutionStrategy executionStrategy = _dataContext.Database.CreateExecutionStrategy(); + return await executionStrategy.ExecuteAsync(async () => { - DateTimeOffset utcNow = DateTimeOffset.UtcNow; Task> responseTask = from additionalData in ValidateRequestAsync(request) - from finalizeResult in FinalizeAsync(request, additionalData).ToLeftEitherAsync(Unit.Default) + from finalizeResult in FinalizeAsync(additionalData).ToLeftEitherAsync(Unit.Default) let successfulMultipartFinalizeEvent = new SuccessfulMultipartFileTransferFinalizationEvent( additionalData.InitializedTransferEntity.Id, additionalData.InitializedTransferEntity.RecipientId ?? Maybe.None, @@ -93,20 +90,18 @@ from sideEffects in Either.FromRightAs async error => { FailedMultipartFileTransferFinalizationEvent failedMultipartFinalizationEvent = - new FailedMultipartFileTransferFinalizationEvent(request.HashId, request.SenderId, error, utcNow); + new FailedMultipartFileTransferFinalizationEvent(request.HashId, request.SenderId, error, + utcNow); await _publisher.Publish(failedMultipartFinalizationEvent, CancellationToken.None); }, async () => { FailedMultipartFileTransferFinalizationEvent failedMultipartFinalizationEvent = - new FailedMultipartFileTransferFinalizationEvent(request.HashId, request.SenderId, FinalizeMultipartFileTransferError.UnknownError, utcNow); + new FailedMultipartFileTransferFinalizationEvent(request.HashId, request.SenderId, + FinalizeMultipartFileTransferError.UnknownError, utcNow); await _publisher.Publish(failedMultipartFinalizationEvent, CancellationToken.None); }); - } - finally - { - await transaction.CommitAsync(CancellationToken.None); - } + }); } private async Task> ValidateRequestAsync(FinalizeMultipartFileTransferCommand request) @@ -132,7 +127,7 @@ private async Task> return new ValidRequestData(initializedTransferEntity); } - private async Task> FinalizeAsync(FinalizeMultipartFileTransferCommand request, ValidRequestData additionalData) + private async Task> FinalizeAsync(ValidRequestData additionalData) { bool finalizeSuccess = await _transferRepository.JoinTransferPartsAsync( additionalData.InitializedTransferEntity.Id, diff --git a/Crypter.Core/Features/Transfer/Commands/SaveMultipartFileTransferCommand.cs b/Crypter.Core/Features/Transfer/Commands/SaveMultipartFileTransferCommand.cs index 8dd03a896..87f7ea25c 100644 --- a/Crypter.Core/Features/Transfer/Commands/SaveMultipartFileTransferCommand.cs +++ b/Crypter.Core/Features/Transfer/Commands/SaveMultipartFileTransferCommand.cs @@ -25,7 +25,6 @@ */ using System; -using System.Data; using System.IO; using System.Linq; using System.Threading; @@ -81,12 +80,10 @@ public SaveMultipartFileTransferCommandHandler( public async Task> Handle(SaveMultipartFileTransferCommand request, CancellationToken cancellationToken) { - await using IDbContextTransaction transaction = await _dataContext.Database - .BeginTransactionAsync(IsolationLevel.Serializable, CancellationToken.None); - - try + DateTimeOffset utcNow = DateTimeOffset.UtcNow; + IExecutionStrategy executionStrategy = _dataContext.Database.CreateExecutionStrategy(); + return await executionStrategy.ExecuteAsync(async () => { - DateTimeOffset utcNow = DateTimeOffset.UtcNow; Task> responseTask = from additionalData in ValidateRequestAsync(request) from saveResult in SavePartAsync(request, additionalData).ToLeftEitherAsync(Unit.Default) @@ -111,11 +108,7 @@ from sideEffects in Either.FromRightAsyn new FailedMultipartFileTransferUploadEvent(request.HashId, request.SenderId, UploadMultipartFileTransferError.UnknownError, utcNow); await _publisher.Publish(failedMultipartUploadEvent, CancellationToken.None); }); - } - finally - { - await transaction.CommitAsync(CancellationToken.None); - } + }); } private async Task> ValidateRequestAsync(SaveMultipartFileTransferCommand request) diff --git a/Crypter.Core/Repositories/TransferRepository.cs b/Crypter.Core/Repositories/TransferRepository.cs index 87cd34e9b..564739a17 100644 --- a/Crypter.Core/Repositories/TransferRepository.cs +++ b/Crypter.Core/Repositories/TransferRepository.cs @@ -112,11 +112,16 @@ public long GetTransferPartsSize(Guid id, TransferItemType itemType, TransferUse { string directory = GetTransferPartsDirectory(itemType, userType, id); DirectoryInfo directoryInfo = new DirectoryInfo(directory); - return directoryInfo - .EnumerateFiles() - .Select(x => x.Length) - .DefaultIfEmpty(0) - .Sum(x => Convert.ToInt64(x / Math.Pow(10, 6))); + if (directoryInfo.Exists) + { + return directoryInfo + .EnumerateFiles() + .Select(x => x.Length) + .DefaultIfEmpty(0) + .Sum(x => Convert.ToInt64(x / Math.Pow(10, 6))); + } + + return 0; } public async Task SaveTransferAsync(Guid id, TransferItemType itemType, TransferUserType userType, @@ -185,14 +190,15 @@ public async Task JoinTransferPartsAsync(Guid id, TransferItemType itemTyp List filenames = Directory .EnumerateFiles(partsDirectory) - .Order() + .OrderBy(x => int.Parse(Path.GetFileNameWithoutExtension(x))) .ToList(); - bool sequentialFilenames = !filenames - .Where((name, index) => Path.GetFileNameWithoutExtension(name) != index.ToString()) + bool nonSequentialFilenames = filenames + .Select(x => int.Parse(Path.GetFileNameWithoutExtension(x))) + .Where((name, index) => name != index) .Any(); - if (!sequentialFilenames) + if (nonSequentialFilenames) { return false; } diff --git a/Crypter.DataAccess/DependencyInjection.cs b/Crypter.DataAccess/DependencyInjection.cs index 467fa39d9..a0d18a8fe 100644 --- a/Crypter.DataAccess/DependencyInjection.cs +++ b/Crypter.DataAccess/DependencyInjection.cs @@ -34,6 +34,8 @@ namespace Crypter.DataAccess; public static class DependencyInjection { + private static readonly string[] RetryableErrorCodes = ["57P01"]; + public static IServiceCollection AddDataAccess(this IServiceCollection services, string connectionString) { ServiceProvider serviceProvider = services.BuildServiceProvider(); @@ -43,13 +45,13 @@ public static IServiceCollection AddDataAccess(this IServiceCollection services, { optionsBuilder.UseNpgsql(connectionString, npgsqlOptionsBuilder => { - npgsqlOptionsBuilder.EnableRetryOnFailure(5, TimeSpan.FromSeconds(5), new[] { "57P01" }); + npgsqlOptionsBuilder.EnableRetryOnFailure(5, TimeSpan.FromSeconds(5), RetryableErrorCodes); npgsqlOptionsBuilder.MigrationsHistoryTable(HistoryRepository.DefaultTableName, DataContext.SchemaName); }) .LogTo( - filter: (eventId, level) => eventId.Id == CoreEventId.ExecutionStrategyRetrying, - logger: (eventData) => + filter: (eventId, _) => eventId.Id == CoreEventId.ExecutionStrategyRetrying, + logger: eventData => { ExecutionStrategyEventData? retryEventData = eventData as ExecutionStrategyEventData; IReadOnlyList? exceptions = retryEventData?.ExceptionsEncountered; diff --git a/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor b/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor index a9fe1c38b..e7d6085c4 100644 --- a/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor +++ b/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor @@ -63,16 +63,16 @@ diff --git a/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor.cs b/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor.cs index c2097a8d2..6f9d988b0 100644 --- a/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor.cs +++ b/Crypter.Web/Shared/Transfer/UploadFileTransfer.razor.cs @@ -51,9 +51,14 @@ public partial class UploadFileTransfer : IDisposable protected override void OnInitialized() { - TransmissionType = BrowserFunctions.BrowserSupportsRequestStreaming() - ? TransferTransmissionType.Stream - : TransferTransmissionType.Multipart; + if (UserSessionService.Session.IsSome) + { + TransmissionType = TransferTransmissionType.Multipart; + } + else if (BrowserFunctions.BrowserSupportsRequestStreaming()) + { + TransmissionType = TransferTransmissionType.Stream; + } _maxStreamSizeMB = UploadSettings.MaximumUploadSizeMB * Convert.ToInt64(Math.Pow(10, 6)); _maxBufferSizeMB = UploadSettings.MaximumUploadBufferSizeMB * Convert.ToInt64(Math.Pow(10, 6)); } @@ -108,12 +113,14 @@ protected async Task OnEncryptClicked() SetHandlerUserInfo(fileUploader); +#pragma warning disable CS8524 Action? progressUpdater = TransmissionType switch { TransferTransmissionType.Buffer => null, - TransferTransmissionType.Stream => SetUploadPercentage, - _ => null + TransferTransmissionType.Stream + or TransferTransmissionType.Multipart => SetUploadPercentage }; +#pragma warning restore CS8524 Either uploadResponse = await fileUploader.UploadAsync(progressUpdater); diff --git a/Volumes/API/appsettings.json b/Volumes/API/appsettings.json index 14157f535..ce46b9ed4 100644 --- a/Volumes/API/appsettings.json +++ b/Volumes/API/appsettings.json @@ -17,7 +17,8 @@ "Port": 587 }, "TransferStorageSettings": { - "AllocatedGB": 10 + "AllocatedGB": 10, + "MaximumTransferSizeMB": 250 }, "HangfireSettings": { "Workers": 1