From 9c089073f556cab93cfdb4553a616a0974cb487d Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Fri, 21 Jun 2024 15:04:12 +0200 Subject: [PATCH 01/22] feat: implement ionos s3 storage off of main branch --- .../BuildingBlocks.Infrastructure.csproj | 21 +- .../BlobStorageServiceCollectionExtensions.cs | 43 +++- .../BlobStorage/Ionos/IonosS3BlobStorage.cs | 226 ++++++++++++++++++ .../BlobStorage/Ionos/IonosS3ClientFactory.cs | 24 ++ .../IonosS3ServiceCollectionExtensions.cs | 37 +++ .../src/Files.ConsumerApi/Configuration.cs | 19 +- .../src/Files.ConsumerApi/FilesModule.cs | 5 + .../Extensions/IConfigurationExtensions.cs | 21 +- 8 files changed, 379 insertions(+), 17 deletions(-) create mode 100644 BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs create mode 100644 BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs create mode 100644 BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index ce022f0ff9..1dc048b99a 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -2,20 +2,21 @@ + - - - - - - - + + + + + + + - + - - + + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 1387b01adc..1f1f16c53c 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -1,5 +1,8 @@ +using System.ComponentModel.DataAnnotations; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; +using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; using Backbone.Tooling.Extensions; using Microsoft.Extensions.DependencyInjection; @@ -9,6 +12,7 @@ public static class BlobStorageServiceCollectionExtensions { public const string AZURE_CLOUD_PROVIDER = "Azure"; public const string GOOGLE_CLOUD_PROVIDER = "GoogleCloud"; + public const string IONOS_CLOUD_PROVIDER = "Ionos"; public static void AddBlobStorage(this IServiceCollection services, Action setupOptions) { @@ -31,6 +35,19 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO googleCloudStorageOptions.GcpAuthJson = options.ConnectionInfo; googleCloudStorageOptions.BucketName = options.Container; }); + else if (options.CloudProvider == IONOS_CLOUD_PROVIDER) + { + services.Configure(opt => + { + opt.ServiceUrl = options.IonosS3Config.ServiceUrl; + opt.AccessKey = options.IonosS3Config.AccessKey; + opt.SecretKey = options.IonosS3Config.SecretKey; + opt.BucketName = options.IonosS3Config.BucketName; + }); + + services.AddSingleton(); + services.AddScoped(); + } else if (options.CloudProvider.IsNullOrEmpty()) throw new NotSupportedException("No cloud provider was specified."); else @@ -41,9 +58,29 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO public class BlobStorageOptions { - public string CloudProvider { get; set; } = null!; + [Required] + [MinLength(1)] + [RegularExpression("Azure|GoogleCloud|Ionos")] + public string CloudProvider { get; set; } = string.Empty; + + public string? ConnectionInfo { get; set; } = string.Empty; + public string? Container { get; set; } = string.Empty; + + public IonosS3Config? IonosS3Config { get; set; } = new IonosS3Config(); +} + +public class IonosS3Config +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; - public string Container { get; set; } = null!; + [Required] + public string AccessKey { get; set; } = string.Empty; - public string? ConnectionInfo { get; set; } = null; + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; } + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs new file mode 100644 index 0000000000..b703876f04 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs @@ -0,0 +1,226 @@ +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Microsoft.Extensions.Logging; +using System.Net; +using Backbone.BuildingBlocks.Application.Abstractions.Exceptions; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.Options; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; +public class IonosS3BlobStorage : IBlobStorage, IDisposable +{ + private readonly IAmazonS3 _s3Client; + private readonly List _changedBlobs; + private readonly IList _removedBlobs; + private readonly string _bucketName; + private readonly IonosS3Options _config; + private readonly ILogger _logger; + + public IonosS3BlobStorage(IOptions config, ILogger logger) + { + var s3Config = new AmazonS3Config + { + ServiceURL = config.Value.ServiceUrl, + ForcePathStyle = true + }; + + _s3Client = new AmazonS3Client(config.Value.AccessKey, config.Value.SecretKey, s3Config); + _changedBlobs = new List(); + _removedBlobs = new List(); + _bucketName = config.Value.BucketName; + _logger = logger; + } + + public void Add(string folder, string id, byte[] content) + { + _changedBlobs.Add(new ChangedBlob(folder, id, content)); + } + + public void Remove(string folder, string id) + { + _removedBlobs.Add(new RemovedBlob(folder, id)); + } + + public void Dispose() + { + _changedBlobs.Clear(); + _removedBlobs.Clear(); + } + + public async Task FindAsync(string folder, string id) + { + _logger.LogTrace("Reading blob with key '{blobId}'...", id); + + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{id}" + }; + + using var response = await _s3Client.GetObjectAsync(request); + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + + _logger.LogTrace("Found blob with key '{blobId}'.", id); + return memoryStream.ToArray(); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", id); + throw new NotFoundException("Blob", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error downloading blob with key '{blobId}'.", id); + throw; + } + } + + public Task> FindAllAsync(string folder, string? prefix = null) + { + return Task.FromResult(FindAllBlobsAsync(folder, prefix)); + } + + private async IAsyncEnumerable FindAllBlobsAsync(string folder, string? prefix) + { + _logger.LogTrace("Listing all blobs..."); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = prefix != null ? $"{folder}/{prefix}" : folder + }; + + ListObjectsV2Response response; + do + { + response = await _s3Client.ListObjectsV2Async(request); + + foreach (var obj in response.S3Objects) + { + yield return obj.Key; + } + + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + + _logger.LogTrace("Found all blobs."); + } + + public async Task SaveAsync() + { + await UploadChangedBlobs(); + await DeleteRemovedBlobs(); + } + + private async Task UploadChangedBlobs() + { + _logger.LogTrace("Uploading '{changedBlobsCount}' changed blobs...", _changedBlobs.Count); + + var changedBlobs = new List(_changedBlobs); + + foreach (var blob in changedBlobs) + { + await EnsureKeyDoesNotExist(blob.Folder, blob.Name); + + using var memoryStream = new MemoryStream(blob.Content); + + try + { + _logger.LogTrace("Uploading blob with key '{blobName}'...", blob.Name); + + var request = new TransferUtilityUploadRequest + { + InputStream = memoryStream, + Key = $"{blob.Folder}/{blob.Name}", + BucketName = _bucketName + }; + + var transferUtility = new TransferUtility(_s3Client); + await transferUtility.UploadAsync(request); + + _logger.LogTrace("Upload of blob with key '{blobName}' was successful.", blob.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error uploading blob with key '{blobName}'.", blob.Name); + throw; + } + finally + { + _changedBlobs.Remove(blob); + } + } + } + + private async Task EnsureKeyDoesNotExist(string folder, string key) + { + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{key}" + }; + + await _s3Client.GetObjectAsync(request); + + _logger.LogError("A blob with key '{blobName}' already exists.", key); + throw new BlobAlreadyExistsException(key); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return; + } + } + + private async Task DeleteRemovedBlobs() + { + _logger.LogTrace("Deleting '{removedBlobsCount}' blobs...", _removedBlobs.Count); + + var blobsToDelete = new List(_removedBlobs); + + foreach (var blob in blobsToDelete) + { + try + { + var request = new DeleteObjectRequest + { + BucketName = _bucketName, + Key = $"{blob.Folder}/{blob.Name}" + }; + + await _s3Client.DeleteObjectAsync(request); + + _removedBlobs.Remove(blob); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", blob.Name); + throw new NotFoundException($"Blob with key '{blob.Name}' was not found.", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting blob with key '{blobName}'.", blob.Name); + throw; + } + } + + _logger.LogTrace("Deletion successful."); + } + + private record ChangedBlob(string Folder, string Name, byte[] Content); + + private record RemovedBlob(string Folder, string Name); +} + +public class IonosS3Config +{ + public required string ServiceUrl { get; set; } + public required string AccessKey { get; set; } + public required string SecretKey { get; set; } + public required string BucketName { get; set; } +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs new file mode 100644 index 0000000000..5033100130 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs @@ -0,0 +1,24 @@ +using Amazon.S3; +using Microsoft.Extensions.Options; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; +public class IonosS3ClientFactory +{ + private readonly IonosS3Options _options; + + public IonosS3ClientFactory(IOptions options) + { + _options = options.Value; + } + + public IAmazonS3 CreateClient() + { + var config = new AmazonS3Config + { + ServiceURL = _options.ServiceUrl, + ForcePathStyle = true + }; + + return new AmazonS3Client(_options.AccessKey, _options.SecretKey, config); + } +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs new file mode 100644 index 0000000000..6497d9730a --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.DependencyInjection; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; +public static class IonosS3ServiceCollectionExtensions +{ + public static void AddIonosS3(this IServiceCollection services, + Action setupOptions) + { + var options = new IonosS3Options(); + setupOptions.Invoke(options); + + services.AddIonosS3(options); + } + + public static void AddIonosS3(this IServiceCollection services, IonosS3Options options) + { + services.Configure(opt => + { + opt.ServiceUrl = options.ServiceUrl; + opt.AccessKey = options.AccessKey; + opt.SecretKey = options.SecretKey; + opt.BucketName = options.BucketName; + }); + + services.AddSingleton(); + services.AddScoped(); + } +} + +public class IonosS3Options +{ + public string ServiceUrl { get; set; } = string.Empty; + public string AccessKey { get; set; } = string.Empty; + public string SecretKey { get; set; } = string.Empty; + public string BucketName { get; set; } = string.Empty; +} diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index 9a9feb333b..7f057fa8b3 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -23,12 +23,14 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud")] + [RegularExpression("Azure|GoogleCloud|Ionos")] public string CloudProvider { get; set; } = string.Empty; public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; + + public IonosS3Config IonosS3Config { get; set; } = new IonosS3Config(); } public class SqlDatabaseConfiguration @@ -42,5 +44,20 @@ public class SqlDatabaseConfiguration [MinLength(1)] public string ConnectionString { get; set; } = string.Empty; } + + public class IonosS3Config + { + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; + } } } diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index 775e50d97b..1af6198108 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -35,6 +35,11 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati parsedConfiguration.Infrastructure.BlobStorage.ContainerName.IsNullOrEmpty() ? "files" : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; + + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.AccessKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.BucketName; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.SecretKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.ServiceUrl; }); services.AddSqlDatabaseHealthCheck(Name, parsedConfiguration.Infrastructure.SqlDatabase.Provider, parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString); diff --git a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs index 5f27947003..1f3b4c3a34 100644 --- a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs @@ -31,12 +31,27 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud")] + [RegularExpression("Azure|GoogleCloud|Ionos")] public string CloudProvider { get; set; } = string.Empty; - [Required] - [MinLength(1)] public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; + + public IonosS3Config IonosS3Config { get; set; } = new IonosS3Config(); +} + +public class IonosS3Config +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; } From 50aed7f4d527b0a5a3c773c9b699dda7c2df6657 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Sun, 23 Jun 2024 22:58:17 +0200 Subject: [PATCH 02/22] fix: update nuget versions --- .../BuildingBlocks.Infrastructure.csproj | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index 1dc048b99a..070275bac0 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -1,22 +1,22 @@ - + - - + + - - - - + + + + - + - - + + From 4b3a1b3597c9fb6b2ad3613b411e635de2192213 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 08:29:15 +0200 Subject: [PATCH 03/22] fix: handle null reference errors --- .../BlobStorage/BlobStorageServiceCollectionExtensions.cs | 6 +++--- .../Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 1f1f16c53c..af0c01a836 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -33,13 +33,13 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO services.AddGoogleCloudStorage(googleCloudStorageOptions => { googleCloudStorageOptions.GcpAuthJson = options.ConnectionInfo; - googleCloudStorageOptions.BucketName = options.Container; + googleCloudStorageOptions.BucketName = options.Container!; }); else if (options.CloudProvider == IONOS_CLOUD_PROVIDER) { services.Configure(opt => { - opt.ServiceUrl = options.IonosS3Config.ServiceUrl; + opt.ServiceUrl = options.IonosS3Config!.ServiceUrl; opt.AccessKey = options.IonosS3Config.AccessKey; opt.SecretKey = options.IonosS3Config.SecretKey; opt.BucketName = options.IonosS3Config.BucketName; @@ -66,7 +66,7 @@ public class BlobStorageOptions public string? ConnectionInfo { get; set; } = string.Empty; public string? Container { get; set; } = string.Empty; - public IonosS3Config? IonosS3Config { get; set; } = new IonosS3Config(); + public IonosS3Config? IonosS3Config { get; set; } = new(); } public class IonosS3Config diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs index b703876f04..f0198ff250 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs @@ -14,7 +14,6 @@ public class IonosS3BlobStorage : IBlobStorage, IDisposable private readonly List _changedBlobs; private readonly IList _removedBlobs; private readonly string _bucketName; - private readonly IonosS3Options _config; private readonly ILogger _logger; public IonosS3BlobStorage(IOptions config, ILogger logger) From fa2f57092ee6ced419d3d57d938bdb9537fc9980 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 08:43:22 +0200 Subject: [PATCH 04/22] fix: handle null reference errors and fix formatting --- .../Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs | 6 +++--- .../Persistence/IServiceCollectionExtensions.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs index f0198ff250..e6b8fb25f9 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs @@ -1,10 +1,10 @@ -using Amazon.S3; +using System.Net; +using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Transfer; -using Microsoft.Extensions.Logging; -using System.Net; using Backbone.BuildingBlocks.Application.Abstractions.Exceptions; using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; diff --git a/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs b/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs index 00ca0777df..bb94a26398 100644 --- a/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs +++ b/Modules/Files/src/Files.Infrastructure/Persistence/IServiceCollectionExtensions.cs @@ -20,7 +20,7 @@ public static void AddPersistence(this IServiceCollection services, PersistenceO { services.AddDatabase(options.DbOptions); services.Configure(blobOptions => - blobOptions.RootFolder = options.BlobStorageOptions.Container); + blobOptions.RootFolder = options.BlobStorageOptions.Container!); services.AddBlobStorage(options.BlobStorageOptions); services.AddTransient(); From 5969505658b35657fd6fd08a5145e9ab520a8130 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 09:28:19 +0200 Subject: [PATCH 05/22] fix: handle initialization errors --- .../BlobStorageServiceCollectionExtensions.cs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index af0c01a836..6a28aac591 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -66,21 +66,14 @@ public class BlobStorageOptions public string? ConnectionInfo { get; set; } = string.Empty; public string? Container { get; set; } = string.Empty; - public IonosS3Config? IonosS3Config { get; set; } = new(); + public IonosS3Config? IonosS3Config { get; set; } } public class IonosS3Config { - [Required] - public string ServiceUrl { get; set; } = string.Empty; - - [Required] - public string AccessKey { get; set; } = string.Empty; - - [Required] - public string SecretKey { get; set; } = string.Empty; - - [Required] - public string BucketName { get; set; } = string.Empty; + public required string ServiceUrl { get; set; } + public required string AccessKey { get; set; } + public required string SecretKey { get; set; } + public required string BucketName { get; set; } } From 945ab59789b68699d6ead381dcd2a9adcc05bcf1 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 21:57:55 +0200 Subject: [PATCH 06/22] fix: handle null reference errors --- .../BlobStorage/BlobStorageServiceCollectionExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 6a28aac591..a7cc8d44c0 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -71,9 +71,9 @@ public class BlobStorageOptions public class IonosS3Config { - public required string ServiceUrl { get; set; } - public required string AccessKey { get; set; } - public required string SecretKey { get; set; } - public required string BucketName { get; set; } + public string? ServiceUrl { get; set; } + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? BucketName { get; set; } } From c60c57569bdd9b8c45ea796f420762c348cfdc6c Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 22:07:22 +0200 Subject: [PATCH 07/22] fix: handle null reference errors --- .../BlobStorage/BlobStorageServiceCollectionExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index a7cc8d44c0..321b266438 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -39,10 +39,10 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO { services.Configure(opt => { - opt.ServiceUrl = options.IonosS3Config!.ServiceUrl; - opt.AccessKey = options.IonosS3Config.AccessKey; - opt.SecretKey = options.IonosS3Config.SecretKey; - opt.BucketName = options.IonosS3Config.BucketName; + opt.ServiceUrl = options.IonosS3Config!.ServiceUrl!; + opt.AccessKey = options.IonosS3Config!.AccessKey!; + opt.SecretKey = options.IonosS3Config!.SecretKey!; + opt.BucketName = options.IonosS3Config!.BucketName!; }); services.AddSingleton(); From b4f8c55866dcd7a17a732e94d4f0a430e913865e Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 22:16:06 +0200 Subject: [PATCH 08/22] fix: handle null reference errors --- .../BlobStorage/Ionos/IonosS3BlobStorage.cs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs index e6b8fb25f9..0e366092d3 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs @@ -170,10 +170,7 @@ private async Task EnsureKeyDoesNotExist(string folder, string key) _logger.LogError("A blob with key '{blobName}' already exists.", key); throw new BlobAlreadyExistsException(key); } - catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { - return; - } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } private async Task DeleteRemovedBlobs() @@ -218,8 +215,8 @@ private record RemovedBlob(string Folder, string Name); public class IonosS3Config { - public required string ServiceUrl { get; set; } - public required string AccessKey { get; set; } - public required string SecretKey { get; set; } - public required string BucketName { get; set; } + public string? ServiceUrl { get; set; } + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? BucketName { get; set; } } From 8718401180958c481b7fbbb5b6437b1eb07f4549 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 22:28:11 +0200 Subject: [PATCH 09/22] fix: handle null reference errors --- .../Ionos/IonosS3ServiceCollectionExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs index 6497d9730a..84b00e9c80 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs @@ -30,8 +30,8 @@ public static void AddIonosS3(this IServiceCollection services, IonosS3Options o public class IonosS3Options { - public string ServiceUrl { get; set; } = string.Empty; - public string AccessKey { get; set; } = string.Empty; - public string SecretKey { get; set; } = string.Empty; - public string BucketName { get; set; } = string.Empty; + public string? ServiceUrl { get; set; } + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? BucketName { get; set; } } From 99ccf42d1dc0338d065ebfecdd9e97bdaf2a0726 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 22:40:56 +0200 Subject: [PATCH 10/22] fix: wip --- .../Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs | 8 -------- .../Ionos/IonosS3ServiceCollectionExtensions.cs | 8 ++++---- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs index 0e366092d3..f63fc2ce8f 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs @@ -212,11 +212,3 @@ private record ChangedBlob(string Folder, string Name, byte[] Content); private record RemovedBlob(string Folder, string Name); } - -public class IonosS3Config -{ - public string? ServiceUrl { get; set; } - public string? AccessKey { get; set; } - public string? SecretKey { get; set; } - public string? BucketName { get; set; } -} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs index 84b00e9c80..6497d9730a 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs @@ -30,8 +30,8 @@ public static void AddIonosS3(this IServiceCollection services, IonosS3Options o public class IonosS3Options { - public string? ServiceUrl { get; set; } - public string? AccessKey { get; set; } - public string? SecretKey { get; set; } - public string? BucketName { get; set; } + public string ServiceUrl { get; set; } = string.Empty; + public string AccessKey { get; set; } = string.Empty; + public string SecretKey { get; set; } = string.Empty; + public string BucketName { get; set; } = string.Empty; } From 765f6b7bb58488fbd02e6292fa672f2dd297cccb Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 22:51:51 +0200 Subject: [PATCH 11/22] fix: wip --- .../src/Files.ConsumerApi/Configuration.cs | 17 +++++------------ .../Extensions/IConfigurationExtensions.cs | 17 +++++------------ 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index 7f057fa8b3..d448a59255 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -30,7 +30,7 @@ public class BlobStorageConfiguration public string ContainerName { get; set; } = string.Empty; - public IonosS3Config IonosS3Config { get; set; } = new IonosS3Config(); + public IonosS3Config? IonosS3Config { get; set; } } public class SqlDatabaseConfiguration @@ -47,17 +47,10 @@ public class SqlDatabaseConfiguration public class IonosS3Config { - [Required] - public string ServiceUrl { get; set; } = string.Empty; - - [Required] - public string AccessKey { get; set; } = string.Empty; - - [Required] - public string SecretKey { get; set; } = string.Empty; - - [Required] - public string BucketName { get; set; } = string.Empty; + public string? ServiceUrl { get; set; } + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? BucketName { get; set; } } } } diff --git a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs index 1f3b4c3a34..e00e42d3ef 100644 --- a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs @@ -38,20 +38,13 @@ public class BlobStorageConfiguration public string ContainerName { get; set; } = string.Empty; - public IonosS3Config IonosS3Config { get; set; } = new IonosS3Config(); + public IonosS3Config? IonosS3Config { get; set; } } public class IonosS3Config { - [Required] - public string ServiceUrl { get; set; } = string.Empty; - - [Required] - public string AccessKey { get; set; } = string.Empty; - - [Required] - public string SecretKey { get; set; } = string.Empty; - - [Required] - public string BucketName { get; set; } = string.Empty; + public string? ServiceUrl { get; set; } + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? BucketName { get; set; } } From 09ff4d98d6c6486b7c62fdbc1a6cdabbb4e902b9 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 22:56:16 +0200 Subject: [PATCH 12/22] fix: wip --- .../BlobStorageServiceCollectionExtensions.cs | 16 ++++++++-------- .../Files/src/Files.ConsumerApi/Configuration.cs | 15 +++++++++++---- .../Extensions/IConfigurationExtensions.cs | 15 +++++++++++---- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 321b266438..7908f86aa5 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -39,10 +39,10 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO { services.Configure(opt => { - opt.ServiceUrl = options.IonosS3Config!.ServiceUrl!; - opt.AccessKey = options.IonosS3Config!.AccessKey!; - opt.SecretKey = options.IonosS3Config!.SecretKey!; - opt.BucketName = options.IonosS3Config!.BucketName!; + opt.ServiceUrl = options.IonosS3Config!.ServiceUrl; + opt.AccessKey = options.IonosS3Config!.AccessKey; + opt.SecretKey = options.IonosS3Config!.SecretKey; + opt.BucketName = options.IonosS3Config!.BucketName; }); services.AddSingleton(); @@ -71,9 +71,9 @@ public class BlobStorageOptions public class IonosS3Config { - public string? ServiceUrl { get; set; } - public string? AccessKey { get; set; } - public string? SecretKey { get; set; } - public string? BucketName { get; set; } + public required string ServiceUrl { get; set; } = string.Empty; + public required string AccessKey { get; set; } = string.Empty; + public required string SecretKey { get; set; } = string.Empty; + public required string BucketName { get; set; } = string.Empty; } diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index d448a59255..0d1b3f038b 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -47,10 +47,17 @@ public class SqlDatabaseConfiguration public class IonosS3Config { - public string? ServiceUrl { get; set; } - public string? AccessKey { get; set; } - public string? SecretKey { get; set; } - public string? BucketName { get; set; } + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; } } } diff --git a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs index e00e42d3ef..85719dd083 100644 --- a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs @@ -43,8 +43,15 @@ public class BlobStorageConfiguration public class IonosS3Config { - public string? ServiceUrl { get; set; } - public string? AccessKey { get; set; } - public string? SecretKey { get; set; } - public string? BucketName { get; set; } + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKey { get; set; } = string.Empty; + + [Required] + public string SecretKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; } From 703c10b47f3584c24a30a108e1434da34035184f Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Mon, 24 Jun 2024 23:02:42 +0200 Subject: [PATCH 13/22] fix: wip --- Modules/Files/src/Files.ConsumerApi/FilesModule.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index 1af6198108..a25b3be8c9 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -36,10 +36,10 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati ? "files" : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.AccessKey; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.BucketName; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.SecretKey; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config.ServiceUrl; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.AccessKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.BucketName; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.SecretKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.ServiceUrl; }); services.AddSqlDatabaseHealthCheck(Name, parsedConfiguration.Infrastructure.SqlDatabase.Provider, parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString); From a3200af51115f3653ef322859e276e9300153801 Mon Sep 17 00:00:00 2001 From: Nikola Vetnic Date: Tue, 25 Jun 2024 11:35:46 +0200 Subject: [PATCH 14/22] fix: merge changes from main --- .../BlobStorageServiceCollectionExtensions.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 7908f86aa5..4a24f119e9 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -58,13 +58,10 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO public class BlobStorageOptions { - [Required] - [MinLength(1)] - [RegularExpression("Azure|GoogleCloud|Ionos")] - public string CloudProvider { get; set; } = string.Empty; + public string CloudProvider { get; set; } = null!; - public string? ConnectionInfo { get; set; } = string.Empty; - public string? Container { get; set; } = string.Empty; + public string Container { get; set; } = null!; + public string? ConnectionInfo { get; set; } = null; public IonosS3Config? IonosS3Config { get; set; } } From a90b1f08b8bf7175c935f2acd1bf3621007e53aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Vetni=C4=87?= Date: Tue, 25 Jun 2024 14:26:53 +0200 Subject: [PATCH 15/22] fix: revert --- .../BlobStorageServiceCollectionExtensions.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 4a24f119e9..7908f86aa5 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -58,10 +58,13 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO public class BlobStorageOptions { - public string CloudProvider { get; set; } = null!; + [Required] + [MinLength(1)] + [RegularExpression("Azure|GoogleCloud|Ionos")] + public string CloudProvider { get; set; } = string.Empty; - public string Container { get; set; } = null!; - public string? ConnectionInfo { get; set; } = null; + public string? ConnectionInfo { get; set; } = string.Empty; + public string? Container { get; set; } = string.Empty; public IonosS3Config? IonosS3Config { get; set; } } From 9ad700d20a1a8db736014cd20cd4384c3736ea63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Vetni=C4=87?= Date: Wed, 26 Jun 2024 10:48:09 +0200 Subject: [PATCH 16/22] fix: wip --- .../BlobStorageServiceCollectionExtensions.cs | 28 +++++++++++++------ .../src/Files.ConsumerApi/Configuration.cs | 8 +++++- .../Extensions/IConfigurationExtensions.cs | 8 +++++- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 7908f86aa5..1d385d5a3d 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -39,10 +39,10 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO { services.Configure(opt => { - opt.ServiceUrl = options.IonosS3Config!.ServiceUrl; - opt.AccessKey = options.IonosS3Config!.AccessKey; - opt.SecretKey = options.IonosS3Config!.SecretKey; - opt.BucketName = options.IonosS3Config!.BucketName; + opt.ServiceUrl = options.IonosS3Config.ServiceUrl; + opt.AccessKey = options.IonosS3Config.AccessKey; + opt.SecretKey = options.IonosS3Config.SecretKey; + opt.BucketName = options.IonosS3Config.BucketName; }); services.AddSingleton(); @@ -66,14 +66,24 @@ public class BlobStorageOptions public string? ConnectionInfo { get; set; } = string.Empty; public string? Container { get; set; } = string.Empty; - public IonosS3Config? IonosS3Config { get; set; } + public IonosS3Config IonosS3Config { get; set; } = new() + { + ServiceUrl = string.Empty, + AccessKey = string.Empty, + SecretKey = string.Empty, + BucketName = string.Empty + }; } public class IonosS3Config { - public required string ServiceUrl { get; set; } = string.Empty; - public required string AccessKey { get; set; } = string.Empty; - public required string SecretKey { get; set; } = string.Empty; - public required string BucketName { get; set; } = string.Empty; + [Required] + public string ServiceUrl { get; set; } = string.Empty; + [Required] + public string AccessKey { get; set; } = string.Empty; + [Required] + public string SecretKey { get; set; } = string.Empty; + [Required] + public string BucketName { get; set; } = string.Empty; } diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index 0d1b3f038b..2117b39c70 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -30,7 +30,13 @@ public class BlobStorageConfiguration public string ContainerName { get; set; } = string.Empty; - public IonosS3Config? IonosS3Config { get; set; } + public IonosS3Config IonosS3Config { get; set; } = new() + { + ServiceUrl = string.Empty, + AccessKey = string.Empty, + SecretKey = string.Empty, + BucketName = string.Empty + }; } public class SqlDatabaseConfiguration diff --git a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs index 85719dd083..85d25061e0 100644 --- a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs @@ -38,7 +38,13 @@ public class BlobStorageConfiguration public string ContainerName { get; set; } = string.Empty; - public IonosS3Config? IonosS3Config { get; set; } + public IonosS3Config IonosS3Config { get; set; } = new() + { + ServiceUrl = string.Empty, + AccessKey = string.Empty, + SecretKey = string.Empty, + BucketName = string.Empty + }; } public class IonosS3Config From 76a4a38c8da2747abf8bd24c762e35f0326cb8d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Vetni=C4=87?= Date: Wed, 26 Jun 2024 11:04:40 +0200 Subject: [PATCH 17/22] fix: wip --- .../BlobStorageServiceCollectionExtensions.cs | 13 ++++++------- .../Files/src/Files.ConsumerApi/Configuration.cs | 8 +------- .../Extensions/IConfigurationExtensions.cs | 8 +------- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 1d385d5a3d..a5b997618c 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -39,10 +39,10 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO { services.Configure(opt => { - opt.ServiceUrl = options.IonosS3Config.ServiceUrl; - opt.AccessKey = options.IonosS3Config.AccessKey; - opt.SecretKey = options.IonosS3Config.SecretKey; - opt.BucketName = options.IonosS3Config.BucketName; + opt.ServiceUrl = options.IonosS3Config!.ServiceUrl; + opt.AccessKey = options.IonosS3Config!.AccessKey; + opt.SecretKey = options.IonosS3Config!.SecretKey; + opt.BucketName = options.IonosS3Config!.BucketName; }); services.AddSingleton(); @@ -81,9 +81,8 @@ public class IonosS3Config public string ServiceUrl { get; set; } = string.Empty; [Required] public string AccessKey { get; set; } = string.Empty; - [Required] + [Required] public string SecretKey { get; set; } = string.Empty; - [Required] + [Required] public string BucketName { get; set; } = string.Empty; } - diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index 2117b39c70..0d1b3f038b 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -30,13 +30,7 @@ public class BlobStorageConfiguration public string ContainerName { get; set; } = string.Empty; - public IonosS3Config IonosS3Config { get; set; } = new() - { - ServiceUrl = string.Empty, - AccessKey = string.Empty, - SecretKey = string.Empty, - BucketName = string.Empty - }; + public IonosS3Config? IonosS3Config { get; set; } } public class SqlDatabaseConfiguration diff --git a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs index 85d25061e0..85719dd083 100644 --- a/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Modules/Files/src/Files.Jobs.SanityCheck/Extensions/IConfigurationExtensions.cs @@ -38,13 +38,7 @@ public class BlobStorageConfiguration public string ContainerName { get; set; } = string.Empty; - public IonosS3Config IonosS3Config { get; set; } = new() - { - ServiceUrl = string.Empty, - AccessKey = string.Empty, - SecretKey = string.Empty, - BucketName = string.Empty - }; + public IonosS3Config? IonosS3Config { get; set; } } public class IonosS3Config From 1f730775015c7123c9ac043f12cc177926caaf09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Vetni=C4=87?= Date: Wed, 26 Jun 2024 11:21:22 +0200 Subject: [PATCH 18/22] fix: wip --- Modules/Files/src/Files.ConsumerApi/FilesModule.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index a25b3be8c9..6fe8de0cb2 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -36,10 +36,10 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati ? "files" : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.AccessKey; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.BucketName; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.SecretKey; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.ServiceUrl; + options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.AccessKey; + options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.BucketName; + options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.SecretKey; + options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.ServiceUrl; }); services.AddSqlDatabaseHealthCheck(Name, parsedConfiguration.Infrastructure.SqlDatabase.Provider, parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString); From 67db025d893b6b6eb474df26941cd20ce0f4e9c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Vetni=C4=87?= Date: Wed, 26 Jun 2024 11:28:48 +0200 Subject: [PATCH 19/22] fix: wip --- .../BlobStorage/BlobStorageServiceCollectionExtensions.cs | 8 +------- Modules/Files/src/Files.ConsumerApi/FilesModule.cs | 8 ++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index a5b997618c..f853974c1b 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -66,13 +66,7 @@ public class BlobStorageOptions public string? ConnectionInfo { get; set; } = string.Empty; public string? Container { get; set; } = string.Empty; - public IonosS3Config IonosS3Config { get; set; } = new() - { - ServiceUrl = string.Empty, - AccessKey = string.Empty, - SecretKey = string.Empty, - BucketName = string.Empty - }; + public IonosS3Config? IonosS3Config { get; set; } } public class IonosS3Config diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index 6fe8de0cb2..a25b3be8c9 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -36,10 +36,10 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati ? "files" : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; - options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.AccessKey; - options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.BucketName; - options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.SecretKey; - options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.ServiceUrl; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.AccessKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.BucketName; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.SecretKey; + if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.ServiceUrl; }); services.AddSqlDatabaseHealthCheck(Name, parsedConfiguration.Infrastructure.SqlDatabase.Provider, parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString); From 08de492359249a3c7a3eb1c94570a858c2f79550 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 23 Oct 2024 12:53:43 +0200 Subject: [PATCH 20/22] refactor: eliminate the word "Ionos" from code --- .../Extensions/IConfigurationExtensions.cs | 6 ++--- .../BlobStorageServiceCollectionExtensions.cs | 26 +++++++++---------- .../S3BlobStorage.cs} | 13 ++++++---- .../S3ClientFactory.cs} | 9 ++++--- .../S3ServiceCollectionExtensions.cs} | 23 ++++++++-------- .../src/Files.ConsumerApi/Configuration.cs | 6 ++--- .../src/Files.ConsumerApi/FilesModule.cs | 8 +++--- 7 files changed, 48 insertions(+), 43 deletions(-) rename BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/{Ionos/IonosS3BlobStorage.cs => S3/S3BlobStorage.cs} (96%) rename BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/{Ionos/IonosS3ClientFactory.cs => S3/S3ClientFactory.cs} (73%) rename BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/{Ionos/IonosS3ServiceCollectionExtensions.cs => S3/S3ServiceCollectionExtensions.cs} (56%) diff --git a/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs b/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs index 6ee38be0af..0f254960ad 100644 --- a/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs +++ b/Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs @@ -31,17 +31,17 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud|Ionos")] + [RegularExpression("Azure|GoogleCloud|S3")] public string CloudProvider { get; set; } = string.Empty; public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; - public IonosS3Config? IonosS3Config { get; set; } + public S3Config? S3Config { get; set; } } -public class IonosS3Config +public class S3Config { [Required] public string ServiceUrl { get; set; } = string.Empty; diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 33e36bffe9..d8c2ba12af 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -2,7 +2,7 @@ using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; -using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; +using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; using Backbone.Tooling.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -13,7 +13,7 @@ public static class BlobStorageServiceCollectionExtensions { public const string AZURE_CLOUD_PROVIDER = "Azure"; public const string GOOGLE_CLOUD_PROVIDER = "GoogleCloud"; - public const string IONOS_CLOUD_PROVIDER = "Ionos"; + public const string S3_CLOUD_PROVIDER = "S3"; public static void AddBlobStorage(this IServiceCollection services, Action setupOptions) { @@ -37,17 +37,17 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO googleCloudStorageOptions.BucketName = options.Container; }); break; - case IONOS_CLOUD_PROVIDER: - services.Configure(opt => + case S3_CLOUD_PROVIDER: + services.Configure(opt => { - opt.ServiceUrl = options.IonosS3Config!.ServiceUrl; - opt.AccessKey = options.IonosS3Config!.AccessKey; - opt.SecretKey = options.IonosS3Config!.SecretKey; - opt.BucketName = options.IonosS3Config!.BucketName; + opt.ServiceUrl = options.S3Config!.ServiceUrl; + opt.AccessKey = options.S3Config!.AccessKey; + opt.SecretKey = options.S3Config!.SecretKey; + opt.BucketName = options.S3Config!.BucketName; }); - services.AddSingleton(); - services.AddScoped(); + services.AddSingleton(); + services.AddScoped(); break; @@ -75,17 +75,17 @@ public class BlobStorageOptions { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud|Ionos")] + [RegularExpression("Azure|GoogleCloud|S3")] public string CloudProvider { get; set; } = null!; public string Container { get; set; } = null!; public string? ConnectionInfo { get; set; } = null; - public IonosS3Config? IonosS3Config { get; set; } + public S3Config? S3Config { get; set; } } -public class IonosS3Config +public class S3Config { [Required] public string ServiceUrl { get; set; } = string.Empty; diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs similarity index 96% rename from BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs rename to BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs index f63fc2ce8f..a6d1c8f9d4 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3BlobStorage.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs @@ -7,16 +7,17 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; -public class IonosS3BlobStorage : IBlobStorage, IDisposable +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public class S3BlobStorage : IBlobStorage, IDisposable { private readonly IAmazonS3 _s3Client; private readonly List _changedBlobs; private readonly IList _removedBlobs; private readonly string _bucketName; - private readonly ILogger _logger; + private readonly ILogger _logger; - public IonosS3BlobStorage(IOptions config, ILogger logger) + public S3BlobStorage(IOptions config, ILogger logger) { var s3Config = new AmazonS3Config { @@ -170,7 +171,9 @@ private async Task EnsureKeyDoesNotExist(string folder, string key) _logger.LogError("A blob with key '{blobName}' already exists.", key); throw new BlobAlreadyExistsException(key); } - catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + } } private async Task DeleteRemovedBlobs() diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ClientFactory.cs similarity index 73% rename from BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs rename to BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ClientFactory.cs index 5033100130..0b01a1aef9 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ClientFactory.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ClientFactory.cs @@ -1,12 +1,13 @@ using Amazon.S3; using Microsoft.Extensions.Options; -namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; -public class IonosS3ClientFactory +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public class S3ClientFactory { - private readonly IonosS3Options _options; + private readonly S3Options _options; - public IonosS3ClientFactory(IOptions options) + public S3ClientFactory(IOptions options) { _options = options.Value; } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs similarity index 56% rename from BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs rename to BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs index 6497d9730a..7403d77f48 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/Ionos/IonosS3ServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs @@ -1,21 +1,22 @@ using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Microsoft.Extensions.DependencyInjection; -namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.Ionos; -public static class IonosS3ServiceCollectionExtensions +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public static class S3ServiceCollectionExtensions { - public static void AddIonosS3(this IServiceCollection services, - Action setupOptions) + public static void AddS3(this IServiceCollection services, + Action setupOptions) { - var options = new IonosS3Options(); + var options = new S3Options(); setupOptions.Invoke(options); - services.AddIonosS3(options); + services.AddS3(options); } - public static void AddIonosS3(this IServiceCollection services, IonosS3Options options) + public static void AddS3(this IServiceCollection services, S3Options options) { - services.Configure(opt => + services.Configure(opt => { opt.ServiceUrl = options.ServiceUrl; opt.AccessKey = options.AccessKey; @@ -23,12 +24,12 @@ public static void AddIonosS3(this IServiceCollection services, IonosS3Options o opt.BucketName = options.BucketName; }); - services.AddSingleton(); - services.AddScoped(); + services.AddSingleton(); + services.AddScoped(); } } -public class IonosS3Options +public class S3Options { public string ServiceUrl { get; set; } = string.Empty; public string AccessKey { get; set; } = string.Empty; diff --git a/Modules/Files/src/Files.ConsumerApi/Configuration.cs b/Modules/Files/src/Files.ConsumerApi/Configuration.cs index b493b6af2a..3bd695ad46 100644 --- a/Modules/Files/src/Files.ConsumerApi/Configuration.cs +++ b/Modules/Files/src/Files.ConsumerApi/Configuration.cs @@ -23,14 +23,14 @@ public class BlobStorageConfiguration { [Required] [MinLength(1)] - [RegularExpression("Azure|GoogleCloud|Ionos")] + [RegularExpression("Azure|GoogleCloud|S3")] public string CloudProvider { get; set; } = string.Empty; public string ConnectionInfo { get; set; } = string.Empty; public string ContainerName { get; set; } = string.Empty; - public IonosS3Config? IonosS3Config { get; set; } + public S3Config? S3Config { get; set; } } public class SqlDatabaseConfiguration @@ -48,7 +48,7 @@ public class SqlDatabaseConfiguration public bool EnableHealthCheck { get; set; } = true; } - public class IonosS3Config + public class S3Config { [Required] public string ServiceUrl { get; set; } = string.Empty; diff --git a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs index 10f571b934..3677ec7aad 100644 --- a/Modules/Files/src/Files.ConsumerApi/FilesModule.cs +++ b/Modules/Files/src/Files.ConsumerApi/FilesModule.cs @@ -36,10 +36,10 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati ? "files" : parsedConfiguration.Infrastructure.BlobStorage.ContainerName; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.AccessKey; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.BucketName; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.SecretKey; - if (options.BlobStorageOptions.IonosS3Config != null) options.BlobStorageOptions.IonosS3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.IonosS3Config!.ServiceUrl; + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.AccessKey = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.AccessKey; + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.BucketName = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.BucketName; + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.SecretKey = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.SecretKey; + if (options.BlobStorageOptions.S3Config != null) options.BlobStorageOptions.S3Config.ServiceUrl = parsedConfiguration.Infrastructure.BlobStorage.S3Config!.ServiceUrl; }); if (parsedConfiguration.Infrastructure.SqlDatabase.EnableHealthCheck) From f3a1b5825a109dda63f5645fa93b9c2b1501107d Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 23 Oct 2024 13:32:17 +0200 Subject: [PATCH 21/22] chore: remove unused S3ClientFactory --- .../BlobStorageServiceCollectionExtensions.cs | 1 - .../BlobStorage/S3/S3ClientFactory.cs | 25 ------------------- .../S3/S3ServiceCollectionExtensions.cs | 3 +-- 3 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ClientFactory.cs diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index d8c2ba12af..ccfc8e6b15 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -46,7 +46,6 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO opt.BucketName = options.S3Config!.BucketName; }); - services.AddSingleton(); services.AddScoped(); break; diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ClientFactory.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ClientFactory.cs deleted file mode 100644 index 0b01a1aef9..0000000000 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ClientFactory.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Amazon.S3; -using Microsoft.Extensions.Options; - -namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; - -public class S3ClientFactory -{ - private readonly S3Options _options; - - public S3ClientFactory(IOptions options) - { - _options = options.Value; - } - - public IAmazonS3 CreateClient() - { - var config = new AmazonS3Config - { - ServiceURL = _options.ServiceUrl, - ForcePathStyle = true - }; - - return new AmazonS3Client(_options.AccessKey, _options.SecretKey, config); - } -} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs index 7403d77f48..0f7e6bbbf9 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs @@ -1,4 +1,4 @@ -using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Microsoft.Extensions.DependencyInjection; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; @@ -24,7 +24,6 @@ public static void AddS3(this IServiceCollection services, S3Options options) opt.BucketName = options.BucketName; }); - services.AddSingleton(); services.AddScoped(); } } From 509a9302a4483dc7d421bc91cd756257a8d7b071 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Wed, 23 Oct 2024 13:33:16 +0200 Subject: [PATCH 22/22] refactor: rename config properties --- .../BlobStorageServiceCollectionExtensions.cs | 4 ++-- .../Persistence/BlobStorage/S3/S3BlobStorage.cs | 2 +- .../BlobStorage/S3/S3ServiceCollectionExtensions.cs | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index ccfc8e6b15..01d256590d 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -41,8 +41,8 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO services.Configure(opt => { opt.ServiceUrl = options.S3Config!.ServiceUrl; - opt.AccessKey = options.S3Config!.AccessKey; - opt.SecretKey = options.S3Config!.SecretKey; + opt.KeyId = options.S3Config!.AccessKey; + opt.Key = options.S3Config!.SecretKey; opt.BucketName = options.S3Config!.BucketName; }); diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs index a6d1c8f9d4..635a2b7259 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs @@ -25,7 +25,7 @@ public S3BlobStorage(IOptions config, ILogger logger) ForcePathStyle = true }; - _s3Client = new AmazonS3Client(config.Value.AccessKey, config.Value.SecretKey, s3Config); + _s3Client = new AmazonS3Client(config.Value.KeyId, config.Value.Key, s3Config); _changedBlobs = new List(); _removedBlobs = new List(); _bucketName = config.Value.BucketName; diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs index 0f7e6bbbf9..97ac0d2fc6 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs @@ -1,4 +1,4 @@ -using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Microsoft.Extensions.DependencyInjection; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; @@ -19,8 +19,8 @@ public static void AddS3(this IServiceCollection services, S3Options options) services.Configure(opt => { opt.ServiceUrl = options.ServiceUrl; - opt.AccessKey = options.AccessKey; - opt.SecretKey = options.SecretKey; + opt.KeyId = options.KeyId; + opt.Key = options.Key; opt.BucketName = options.BucketName; }); @@ -31,7 +31,7 @@ public static void AddS3(this IServiceCollection services, S3Options options) public class S3Options { public string ServiceUrl { get; set; } = string.Empty; - public string AccessKey { get; set; } = string.Empty; - public string SecretKey { get; set; } = string.Empty; + public string KeyId { get; set; } = string.Empty; + public string Key { get; set; } = string.Empty; public string BucketName { get; set; } = string.Empty; }