diff --git a/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs b/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs index fbd29d9c69..0f254960ad 100644 --- a/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs @@ -31,12 +31,27 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud")] + [RegularExpression("Azure|GoogleCloud|S3")] public string CloudProvider { get; set; } = string.Empty; - [Required] - [MinLength(1)] public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; + + public S3Config? S3Config { get; set; } +} + +public class S3Config +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index 37d1d0d7fa..c079e705e4 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -2,6 +2,7 @@ + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index f6c9c85826..01d256590d 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -1,6 +1,8 @@ +using System.ComponentModel.DataAnnotations; using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; +using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; using Backbone.Tooling.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -11,6 +13,7 @@ public static class BlobStorageServiceCollectionExtensions { public const string AZURE_CLOUD_PROVIDER = "Azure"; public const string GOOGLE_CLOUD_PROVIDER = "GoogleCloud"; + public const string S3_CLOUD_PROVIDER = "S3"; public static void AddBlobStorage(this IServiceCollection services, Action setupOptions) { @@ -34,6 +37,19 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO googleCloudStorageOptions.BucketName = options.Container; }); break; + case S3_CLOUD_PROVIDER: + services.Configure(opt => + { + opt.ServiceUrl = options.S3Config!.ServiceUrl; + opt.KeyId = options.S3Config!.AccessKey; + opt.Key = options.S3Config!.SecretKey; + opt.BucketName = options.S3Config!.BucketName; + }); + + services.AddScoped(); + + break; + default: { if (options.CloudProvider.IsNullOrEmpty()) @@ -56,9 +72,29 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO public class BlobStorageOptions { + [Required] + [MinLength(1)] + [RegularExpression("Azure|GoogleCloud|S3")] public string CloudProvider { get; set; } = null!; public string Container { get; set; } = null!; - public string? ConnectionInfo { get; set; } + public string? ConnectionInfo { get; set; } = null; + + public S3Config? S3Config { get; set; } +} + +public class S3Config +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs new file mode 100644 index 0000000000..635a2b7259 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs @@ -0,0 +1,217 @@ +using System.Net; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Backbone.BuildingBlocks.Application.Abstractions.Exceptions; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public class S3BlobStorage : IBlobStorage, IDisposable +{ + private readonly IAmazonS3 _s3Client; + private readonly List _changedBlobs; + private readonly IList _removedBlobs; + private readonly string _bucketName; + private readonly ILogger _logger; + + public S3BlobStorage(IOptions config, ILogger logger) + { + var s3Config = new AmazonS3Config + { + ServiceURL = config.Value.ServiceUrl, + ForcePathStyle = true + }; + + _s3Client = new AmazonS3Client(config.Value.KeyId, config.Value.Key, s3Config); + _changedBlobs = new List(); + _removedBlobs = new List(); + _bucketName = config.Value.BucketName; + _logger = logger; + } + + public void Add(string folder, string id, byte[] content) + { + _changedBlobs.Add(new ChangedBlob(folder, id, content)); + } + + public void Remove(string folder, string id) + { + _removedBlobs.Add(new RemovedBlob(folder, id)); + } + + public void Dispose() + { + _changedBlobs.Clear(); + _removedBlobs.Clear(); + } + + public async Task FindAsync(string folder, string id) + { + _logger.LogTrace("Reading blob with key '{blobId}'...", id); + + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{id}" + }; + + using var response = await _s3Client.GetObjectAsync(request); + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + + _logger.LogTrace("Found blob with key '{blobId}'.", id); + return memoryStream.ToArray(); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", id); + throw new NotFoundException("Blob", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error downloading blob with key '{blobId}'.", id); + throw; + } + } + + public Task> FindAllAsync(string folder, string? prefix = null) + { + return Task.FromResult(FindAllBlobsAsync(folder, prefix)); + } + + private async IAsyncEnumerable FindAllBlobsAsync(string folder, string? prefix) + { + _logger.LogTrace("Listing all blobs..."); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = prefix != null ? $"{folder}/{prefix}" : folder + }; + + ListObjectsV2Response response; + do + { + response = await _s3Client.ListObjectsV2Async(request); + + foreach (var obj in response.S3Objects) + { + yield return obj.Key; + } + + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + + _logger.LogTrace("Found all blobs."); + } + + public async Task SaveAsync() + { + await UploadChangedBlobs(); + await DeleteRemovedBlobs(); + } + + private async Task UploadChangedBlobs() + { + _logger.LogTrace("Uploading '{changedBlobsCount}' changed blobs...", _changedBlobs.Count); + + var changedBlobs = new List(_changedBlobs); + + foreach (var blob in changedBlobs) + { + await EnsureKeyDoesNotExist(blob.Folder, blob.Name); + + using var memoryStream = new MemoryStream(blob.Content); + + try + { + _logger.LogTrace("Uploading blob with key '{blobName}'...", blob.Name); + + var request = new TransferUtilityUploadRequest + { + InputStream = memoryStream, + Key = $"{blob.Folder}/{blob.Name}", + BucketName = _bucketName + }; + + var transferUtility = new TransferUtility(_s3Client); + await transferUtility.UploadAsync(request); + + _logger.LogTrace("Upload of blob with key '{blobName}' was successful.", blob.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error uploading blob with key '{blobName}'.", blob.Name); + throw; + } + finally + { + _changedBlobs.Remove(blob); + } + } + } + + private async Task EnsureKeyDoesNotExist(string folder, string key) + { + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{key}" + }; + + await _s3Client.GetObjectAsync(request); + + _logger.LogError("A blob with key '{blobName}' already exists.", key); + throw new BlobAlreadyExistsException(key); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + } + } + + private async Task DeleteRemovedBlobs() + { + _logger.LogTrace("Deleting '{removedBlobsCount}' blobs...", _removedBlobs.Count); + + var blobsToDelete = new List(_removedBlobs); + + foreach (var blob in blobsToDelete) + { + try + { + var request = new DeleteObjectRequest + { + BucketName = _bucketName, + Key = $"{blob.Folder}/{blob.Name}" + }; + + await _s3Client.DeleteObjectAsync(request); + + _removedBlobs.Remove(blob); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", blob.Name); + throw new NotFoundException($"Blob with key '{blob.Name}' was not found.", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting blob with key '{blobName}'.", blob.Name); + throw; + } + } + + _logger.LogTrace("Deletion successful."); + } + + private record ChangedBlob(string Folder, string Name, byte[] Content); + + private record RemovedBlob(string Folder, string Name); +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs new file mode 100644 index 0000000000..97ac0d2fc6 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.DependencyInjection; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public static class S3ServiceCollectionExtensions +{ + public static void AddS3(this IServiceCollection services, + Action setupOptions) + { + var options = new S3Options(); + setupOptions.Invoke(options); + + services.AddS3(options); + } + + public static void AddS3(this IServiceCollection services, S3Options options) + { + services.Configure(opt => + { + opt.ServiceUrl = options.ServiceUrl; + opt.KeyId = options.KeyId; + opt.Key = options.Key; + opt.BucketName = options.BucketName; + }); + + services.AddScoped(); + } +} + +public class S3Options +{ + public string ServiceUrl { get; set; } = string.Empty; + public string KeyId { get; set; } = string.Empty; + public string Key { get; set; } = string.Empty; + public string BucketName { get; set; } = string.Empty; +} diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index 5440e727b5..3bd695ad46 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -23,12 +23,14 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud")] + [RegularExpression("Azure|GoogleCloud|S3")] public string CloudProvider { get; set; } = string.Empty; public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; + + public S3Config? S3Config { get; set; } } public class SqlDatabaseConfiguration @@ -45,5 +47,20 @@ public class SqlDatabaseConfiguration [Required] public bool EnableHealthCheck { get; set; } = true; } + + public class S3Config + { + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; + } } } diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index 87b4943bfa..3677ec7aad 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -35,6 +35,11 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati parsedConfiguration.Infrastructure.BlobStorage.ContainerName.IsNullOrEmpty() ? "files" : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; + + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.AccessKey; + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.BucketName; + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.SecretKey; + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.ServiceUrl; }); if (parsedConfiguration.Infrastructure.SqlDatabase.EnableHealthCheck) diff --git a/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs b/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs index 00ca0777df..bb94a26398 100644 --- a/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs +++ b/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs @@ -20,7 +20,7 @@ public static void AddPersistence(this IServiceCollection services, PersistenceO { services.AddDatabase(options.DbOptions); services.Configure(blobOptions => - blobOptions.RootFolder = options.BlobStorageOptions.Container); + blobOptions.RootFolder = options.BlobStorageOptions.Container!); services.AddBlobStorage(options.BlobStorageOptions); services.AddTransient();