From 099193932120b70387e1adaf8f5a97e2c10d9b0b Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Wed, 25 Sep 2024 16:28:31 +0200 Subject: [PATCH 01/11] feat: Add Azure Blob Storage to Health Check --- Applications/ConsumerApi/src/ConsumerApi.csproj | 1 + Applications/ConsumerApi/src/Program.cs | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/Applications/ConsumerApi/src/ConsumerApi.csproj b/Applications/ConsumerApi/src/ConsumerApi.csproj index fd4611b58d..2cfaa879b6 100644 --- a/Applications/ConsumerApi/src/ConsumerApi.csproj +++ b/Applications/ConsumerApi/src/ConsumerApi.csproj @@ -25,6 +25,7 @@ + diff --git a/Applications/ConsumerApi/src/Program.cs b/Applications/ConsumerApi/src/Program.cs index 46bae3ca0c..f2cf709b78 100644 --- a/Applications/ConsumerApi/src/Program.cs +++ b/Applications/ConsumerApi/src/Program.cs @@ -1,5 +1,6 @@ using System.Reflection; using Autofac.Extensions.DependencyInjection; +using Azure.Storage.Blobs; using Backbone.BuildingBlocks.API; using Backbone.BuildingBlocks.API.Extensions; using Backbone.BuildingBlocks.API.Mvc.Middleware; @@ -29,6 +30,7 @@ using Backbone.Modules.Tokens.ConsumerApi; using Backbone.Modules.Tokens.Infrastructure.Persistence.Database; using Backbone.Tooling.Extensions; +using HealthChecks.Azure.Storage.Blobs; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Extensions.Options; @@ -161,6 +163,14 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config .AddCustomFluentValidation() .AddCustomOpenIddict(parsedConfiguration.Authentication); + if (parsedConfiguration.Modules.Files.Infrastructure.BlobStorage.CloudProvider == "Azure") + { + services.AddHealthChecks().AddAzureBlobStorage( + clientFactory: _ => new BlobServiceClient(parsedConfiguration.Modules.Files.Infrastructure.BlobStorage.ConnectionInfo), + optionsFactory: _ => new AzureBlobStorageHealthCheckOptions { ContainerName = parsedConfiguration.Modules.Files.Infrastructure.BlobStorage.ContainerName } + ); + } + if (parsedConfiguration.SwaggerUi.Enabled) services.AddCustomSwaggerUi(parsedConfiguration.SwaggerUi); From 5a38bdf81ef03c8a0560634014c50bed499a4350 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 30 Sep 2024 15:53:05 +0200 Subject: [PATCH 02/11] chore: Move Azure Blob Storage Health Check to Files Module --- Applications/ConsumerApi/src/ConsumerApi.csproj | 1 - Applications/ConsumerApi/src/Program.cs | 10 ---------- .../BuildingBlocks.Infrastructure.csproj | 1 + .../BlobStorageServiceCollectionExtensions.cs | 6 ++++++ 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Applications/ConsumerApi/src/ConsumerApi.csproj b/Applications/ConsumerApi/src/ConsumerApi.csproj index 7f94bbe475..23a9eda662 100644 --- a/Applications/ConsumerApi/src/ConsumerApi.csproj +++ b/Applications/ConsumerApi/src/ConsumerApi.csproj @@ -25,7 +25,6 @@ - diff --git a/Applications/ConsumerApi/src/Program.cs b/Applications/ConsumerApi/src/Program.cs index f2cf709b78..46bae3ca0c 100644 --- a/Applications/ConsumerApi/src/Program.cs +++ b/Applications/ConsumerApi/src/Program.cs @@ -1,6 +1,5 @@ using System.Reflection; using Autofac.Extensions.DependencyInjection; -using Azure.Storage.Blobs; using Backbone.BuildingBlocks.API; using Backbone.BuildingBlocks.API.Extensions; using Backbone.BuildingBlocks.API.Mvc.Middleware; @@ -30,7 +29,6 @@ using Backbone.Modules.Tokens.ConsumerApi; using Backbone.Modules.Tokens.Infrastructure.Persistence.Database; using Backbone.Tooling.Extensions; -using HealthChecks.Azure.Storage.Blobs; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Extensions.Options; @@ -163,14 +161,6 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config .AddCustomFluentValidation() .AddCustomOpenIddict(parsedConfiguration.Authentication); - if (parsedConfiguration.Modules.Files.Infrastructure.BlobStorage.CloudProvider == "Azure") - { - services.AddHealthChecks().AddAzureBlobStorage( - clientFactory: _ => new BlobServiceClient(parsedConfiguration.Modules.Files.Infrastructure.BlobStorage.ConnectionInfo), - optionsFactory: _ => new AzureBlobStorageHealthCheckOptions { ContainerName = parsedConfiguration.Modules.Files.Infrastructure.BlobStorage.ContainerName } - ); - } - if (parsedConfiguration.SwaggerUi.Enabled) services.AddCustomSwaggerUi(parsedConfiguration.SwaggerUi); diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index 3ae2d3ae60..f347608873 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -19,6 +19,7 @@ + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index e6ece37294..fb4f1e6e51 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -1,6 +1,8 @@ +using Azure.Storage.Blobs; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; using Backbone.Tooling.Extensions; +using HealthChecks.Azure.Storage.Blobs; using Microsoft.Extensions.DependencyInjection; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage; @@ -24,6 +26,10 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO { case AZURE_CLOUD_PROVIDER: services.AddAzureStorageAccount(azureStorageAccountOptions => { azureStorageAccountOptions.ConnectionString = options.ConnectionInfo!; }); + services.AddHealthChecks().AddAzureBlobStorage( + clientFactory: _ => new BlobServiceClient(options.ConnectionInfo), + optionsFactory: _ => new AzureBlobStorageHealthCheckOptions { ContainerName = options.Container } + ); break; case GOOGLE_CLOUD_PROVIDER: services.AddGoogleCloudStorage(googleCloudStorageOptions => From 98346c56a01288ea63c441eb4feac7bff35bf70b Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 7 Oct 2024 13:14:00 +0200 Subject: [PATCH 03/11] feat: Add Google Cloud Storage Health Check --- .../BlobStorageServiceCollectionExtensions.cs | 1 + .../GoogleCloudStorageHealthCheck.cs | 51 +++++++++++++++++++ ...CloudStorageServiceCollectionExtensions.cs | 2 + 3 files changed, 54 insertions(+) create mode 100644 BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index fb4f1e6e51..2f30ca44c2 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -37,6 +37,7 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO googleCloudStorageOptions.GcpAuthJson = options.ConnectionInfo; googleCloudStorageOptions.BucketName = options.Container; }); + services.AddHealthChecks().AddCheck("GoogleCloud"); break; default: { diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs new file mode 100644 index 0000000000..2c84396018 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs @@ -0,0 +1,51 @@ +using System.Text; +using Backbone.Tooling.Extensions; +using Google.Cloud.Storage.V1; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; + +public class GoogleCloudStorageHealthCheck : IHealthCheck +{ + public GoogleCloudStorageHealthCheck(StorageClient storage, GoogleCloudStorageOptions options) + { + _storage = storage; + _options = options; + } + + private readonly StorageClient _storage; + private readonly GoogleCloudStorageOptions _options; + + private static bool? _isHealthy; + private const string FILE_TEXT = "healthcheck"; + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + if (!_isHealthy.HasValue) + { + try + { + //Upload random Blob with predefined text + var filename = Guid.NewGuid().ToString(); + await _storage.UploadObjectAsync(_options.BucketName, filename, "text/plain", new MemoryStream(FILE_TEXT.GetBytes()), null, cancellationToken); + + //Download the Blob again and check the text + var downloadStream = new MemoryStream(); + await _storage.DownloadObjectAsync(_options.BucketName, filename, downloadStream, null, cancellationToken); + var downloadedString = Encoding.UTF8.GetString(downloadStream.ToArray()); + if (downloadedString != FILE_TEXT) _isHealthy = false; + + //Delete the Blob + await _storage.DeleteObjectAsync(_options.BucketName, filename, null, cancellationToken); + + _isHealthy = true; + } + catch (Exception) + { + _isHealthy = false; + } + } + + return _isHealthy.Value ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy(); + } +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs index cb4f12835b..ac3e3120d6 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs @@ -30,6 +30,8 @@ public static void AddGoogleCloudStorage(this IServiceCollection services, Googl return storageClient; }); + services.AddSingleton(_ => options); + services.AddScoped(sp => { var storageClient = sp.GetService(); From 398f7e4cabd33b3031fca39f2a9a2867cbd21736 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 7 Oct 2024 16:44:38 +0200 Subject: [PATCH 04/11] test: Change Dependency on Azure Storage emulator --- .ci/compose.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/compose.test.yml b/.ci/compose.test.yml index a4234d74b3..a436bc6067 100644 --- a/.ci/compose.test.yml +++ b/.ci/compose.test.yml @@ -13,7 +13,7 @@ services: rabbitmq: condition: service_started azure-storage-emulator: - condition: service_started + condition: service_healthy configs: - source: Config target: app/appsettings.override.json From b11f477bb979bd988265a32976f5e5eeb29f1f80 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Mon, 7 Oct 2024 16:54:41 +0200 Subject: [PATCH 05/11] chore: Revert dependency change --- .ci/compose.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/compose.test.yml b/.ci/compose.test.yml index a436bc6067..a4234d74b3 100644 --- a/.ci/compose.test.yml +++ b/.ci/compose.test.yml @@ -13,7 +13,7 @@ services: rabbitmq: condition: service_started azure-storage-emulator: - condition: service_healthy + condition: service_started configs: - source: Config target: app/appsettings.override.json From cdc5699f93c08142da4f40da1b7de9a2f0ed42d9 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 8 Oct 2024 12:53:06 +0200 Subject: [PATCH 06/11] refactor: Move the health checks into their respective AddStorage methods --- .../AzureStorageAccountServiceCollectionExtensions.cs | 8 ++++++++ .../BlobStorage/BlobStorageServiceCollectionExtensions.cs | 7 ------- .../GoogleCloudStorageServiceCollectionExtensions.cs | 6 ++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs index da3acfb0d9..0d10f9d8bb 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs @@ -1,5 +1,8 @@ +using Azure.Storage.Blobs; using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using HealthChecks.Azure.Storage.Blobs; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; @@ -20,6 +23,11 @@ public static void AddAzureStorageAccount(this IServiceCollection services, Azur services.Configure(opt => opt.ConnectionString = options.ConnectionString); services.AddSingleton(); services.AddScoped(); + services.AddSingleton(_ => new BlobServiceClient(options.ConnectionString)); + + services.AddHealthChecks().AddAzureBlobStorage( + optionsFactory: sp => new AzureBlobStorageHealthCheckOptions { ContainerName = sp.GetRequiredService>().Value.Container } + ); } } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 2f30ca44c2..e6ece37294 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -1,8 +1,6 @@ -using Azure.Storage.Blobs; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; using Backbone.Tooling.Extensions; -using HealthChecks.Azure.Storage.Blobs; using Microsoft.Extensions.DependencyInjection; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage; @@ -26,10 +24,6 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO { case AZURE_CLOUD_PROVIDER: services.AddAzureStorageAccount(azureStorageAccountOptions => { azureStorageAccountOptions.ConnectionString = options.ConnectionInfo!; }); - services.AddHealthChecks().AddAzureBlobStorage( - clientFactory: _ => new BlobServiceClient(options.ConnectionInfo), - optionsFactory: _ => new AzureBlobStorageHealthCheckOptions { ContainerName = options.Container } - ); break; case GOOGLE_CLOUD_PROVIDER: services.AddGoogleCloudStorage(googleCloudStorageOptions => @@ -37,7 +31,6 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO googleCloudStorageOptions.GcpAuthJson = options.ConnectionInfo; googleCloudStorageOptions.BucketName = options.Container; }); - services.AddHealthChecks().AddCheck("GoogleCloud"); break; default: { diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs index ac3e3120d6..f8605bf9e4 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs @@ -3,6 +3,7 @@ using Google.Apis.Auth.OAuth2; using Google.Cloud.Storage.V1; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; @@ -30,8 +31,6 @@ public static void AddGoogleCloudStorage(this IServiceCollection services, Googl return storageClient; }); - services.AddSingleton(_ => options); - services.AddScoped(sp => { var storageClient = sp.GetService(); @@ -49,6 +48,9 @@ public static void AddGoogleCloudStorage(this IServiceCollection services, Googl return new GoogleCloudStorage(storageClient, logger); }); + + services.AddHealthChecks() + .Add(new HealthCheckRegistration("Google Cloud", sp => new GoogleCloudStorageHealthCheck(sp.GetRequiredService(), options), HealthStatus.Unhealthy, null)); } } From 7a6301465079aa038e1354a9c4c5804614822b9d Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Tue, 8 Oct 2024 12:54:41 +0200 Subject: [PATCH 07/11] refactor: Rewrite and simplify Google Cloud Health Check --- .../GoogleCloudStorageHealthCheck.cs | 80 ++++++++++++------- 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs index 2c84396018..36b6e7af8d 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs @@ -7,45 +7,71 @@ namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleC public class GoogleCloudStorageHealthCheck : IHealthCheck { + private const string FILE_TEXT = "healthcheck"; + private static bool? _isHealthy; + + private readonly StorageClient _storage; + private readonly GoogleCloudStorageOptions _options; + public GoogleCloudStorageHealthCheck(StorageClient storage, GoogleCloudStorageOptions options) { _storage = storage; _options = options; } - private readonly StorageClient _storage; - private readonly GoogleCloudStorageOptions _options; - - private static bool? _isHealthy; - private const string FILE_TEXT = "healthcheck"; - public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { if (!_isHealthy.HasValue) { - try - { - //Upload random Blob with predefined text - var filename = Guid.NewGuid().ToString(); - await _storage.UploadObjectAsync(_options.BucketName, filename, "text/plain", new MemoryStream(FILE_TEXT.GetBytes()), null, cancellationToken); - - //Download the Blob again and check the text - var downloadStream = new MemoryStream(); - await _storage.DownloadObjectAsync(_options.BucketName, filename, downloadStream, null, cancellationToken); - var downloadedString = Encoding.UTF8.GetString(downloadStream.ToArray()); - if (downloadedString != FILE_TEXT) _isHealthy = false; - - //Delete the Blob - await _storage.DeleteObjectAsync(_options.BucketName, filename, null, cancellationToken); - - _isHealthy = true; - } - catch (Exception) - { - _isHealthy = false; - } + var filename = Guid.NewGuid().ToString(); + var isUploadPossible = await IsUploadPossible(filename, cancellationToken); + var isDownloadPossible = await IsDownloadPossible(filename, cancellationToken); + var isDeletionPossible = await IsDeletionPossible(filename, cancellationToken); + + _isHealthy = isUploadPossible && isDownloadPossible && isDeletionPossible; } return _isHealthy.Value ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy(); } + + private async Task IsUploadPossible(string filename, CancellationToken cancellationToken = default) + { + try + { + await _storage.UploadObjectAsync(_options.BucketName, filename, "text/plain", new MemoryStream(FILE_TEXT.GetBytes()), null, cancellationToken); + return true; + } + catch (Exception) + { + return false; + } + } + + private async Task IsDownloadPossible(string filename, CancellationToken cancellationToken = default) + { + try + { + var downloadStream = new MemoryStream(); + await _storage.DownloadObjectAsync(_options.BucketName, filename, downloadStream, null, cancellationToken); + var downloadedString = Encoding.UTF8.GetString(downloadStream.ToArray()); + return downloadedString == FILE_TEXT; + } + catch (Exception) + { + return false; + } + } + + private async Task IsDeletionPossible(string filename, CancellationToken cancellationToken = default) + { + try + { + await _storage.DeleteObjectAsync(_options.BucketName, filename, null, cancellationToken); + return true; + } + catch (Exception) + { + return false; + } + } } From 5c9aa02d6c28f0f210b4e00e33143b0c6e9ed1b0 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 11 Oct 2024 12:46:16 +0200 Subject: [PATCH 08/11] refactor: Rewrite Google Cloud Storage Health Check as common Health check --- ...althCheck.cs => BlobStorageHealthCheck.cs} | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) rename BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/{GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs => BlobStorageHealthCheck.cs} (53%) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs similarity index 53% rename from BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs rename to BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs index 36b6e7af8d..a2efb637f1 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageHealthCheck.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs @@ -1,44 +1,49 @@ using System.Text; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.Tooling.Extensions; -using Google.Cloud.Storage.V1; using Microsoft.Extensions.Diagnostics.HealthChecks; -namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage; -public class GoogleCloudStorageHealthCheck : IHealthCheck +public class BlobStorageHealthCheck : IHealthCheck { private const string FILE_TEXT = "healthcheck"; + private const int MAX_NUMBER_OF_TRIES = 5; + private static bool? _isHealthy; + private static int _numberOfTries = 0; - private readonly StorageClient _storage; - private readonly GoogleCloudStorageOptions _options; + private readonly IBlobStorage _storage; + private readonly string _bucketName; - public GoogleCloudStorageHealthCheck(StorageClient storage, GoogleCloudStorageOptions options) + public BlobStorageHealthCheck(IBlobStorage storage, string bucketName) { _storage = storage; - _options = options; + _bucketName = bucketName; } public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { - if (!_isHealthy.HasValue) + if (!_isHealthy.HasValue || (_isHealthy == false && _numberOfTries < MAX_NUMBER_OF_TRIES)) { var filename = Guid.NewGuid().ToString(); - var isUploadPossible = await IsUploadPossible(filename, cancellationToken); - var isDownloadPossible = await IsDownloadPossible(filename, cancellationToken); - var isDeletionPossible = await IsDeletionPossible(filename, cancellationToken); + var isUploadPossible = await IsUploadPossible(filename); + var isDownloadPossible = await IsDownloadPossible(filename); + var isDeletionPossible = await IsDeletionPossible(filename); _isHealthy = isUploadPossible && isDownloadPossible && isDeletionPossible; + _numberOfTries++; } return _isHealthy.Value ? HealthCheckResult.Healthy() : HealthCheckResult.Unhealthy(); } - private async Task IsUploadPossible(string filename, CancellationToken cancellationToken = default) + private async Task IsUploadPossible(string filename) { try { - await _storage.UploadObjectAsync(_options.BucketName, filename, "text/plain", new MemoryStream(FILE_TEXT.GetBytes()), null, cancellationToken); + _storage.Add(_bucketName, filename, FILE_TEXT.GetBytes()); + await _storage.SaveAsync(); return true; } catch (Exception) @@ -47,13 +52,12 @@ private async Task IsUploadPossible(string filename, CancellationToken can } } - private async Task IsDownloadPossible(string filename, CancellationToken cancellationToken = default) + private async Task IsDownloadPossible(string filename) { try { - var downloadStream = new MemoryStream(); - await _storage.DownloadObjectAsync(_options.BucketName, filename, downloadStream, null, cancellationToken); - var downloadedString = Encoding.UTF8.GetString(downloadStream.ToArray()); + var downloadBytes = await _storage.FindAsync(_bucketName, filename); + var downloadedString = Encoding.UTF8.GetString(downloadBytes); return downloadedString == FILE_TEXT; } catch (Exception) @@ -62,11 +66,12 @@ private async Task IsDownloadPossible(string filename, CancellationToken c } } - private async Task IsDeletionPossible(string filename, CancellationToken cancellationToken = default) + private async Task IsDeletionPossible(string filename) { try { - await _storage.DeleteObjectAsync(_options.BucketName, filename, null, cancellationToken); + _storage.Remove(_bucketName, filename); + await _storage.SaveAsync(); return true; } catch (Exception) From b559bb2fef7e3c2d5d69b41226f62addcd1118ec Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 11 Oct 2024 12:47:15 +0200 Subject: [PATCH 09/11] refactor: Use common health check --- ...ureStorageAccountServiceCollectionExtensions.cs | 14 ++++++++------ .../BlobStorageServiceCollectionExtensions.cs | 6 +++++- ...oogleCloudStorageServiceCollectionExtensions.cs | 9 +++++++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs index 0d10f9d8bb..cdba64017d 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs @@ -1,8 +1,6 @@ -using Azure.Storage.Blobs; using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; -using HealthChecks.Azure.Storage.Blobs; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; +using Microsoft.Extensions.Diagnostics.HealthChecks; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; @@ -23,10 +21,13 @@ public static void AddAzureStorageAccount(this IServiceCollection services, Azur services.Configure(opt => opt.ConnectionString = options.ConnectionString); services.AddSingleton(); services.AddScoped(); - services.AddSingleton(_ => new BlobServiceClient(options.ConnectionString)); - services.AddHealthChecks().AddAzureBlobStorage( - optionsFactory: sp => new AzureBlobStorageHealthCheckOptions { ContainerName = sp.GetRequiredService>().Value.Container } + services.AddHealthChecks().Add( + new HealthCheckRegistration( + "blob_storage", + sp => new BlobStorageHealthCheck(sp.GetRequiredService(), options.BucketName), + HealthStatus.Unhealthy, null + ) ); } } @@ -34,4 +35,5 @@ public static void AddAzureStorageAccount(this IServiceCollection services, Azur public class AzureStorageAccountOptions { public string ConnectionString { get; set; } = string.Empty; + public string BucketName { get; set; } = string.Empty; } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index e6ece37294..23ef7c24f2 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -23,7 +23,11 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO switch (options.CloudProvider) { case AZURE_CLOUD_PROVIDER: - services.AddAzureStorageAccount(azureStorageAccountOptions => { azureStorageAccountOptions.ConnectionString = options.ConnectionInfo!; }); + services.AddAzureStorageAccount(azureStorageAccountOptions => + { + azureStorageAccountOptions.ConnectionString = options.ConnectionInfo!; + azureStorageAccountOptions.BucketName = options.Container; + }); break; case GOOGLE_CLOUD_PROVIDER: services.AddGoogleCloudStorage(googleCloudStorageOptions => diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs index f8605bf9e4..12c31eeda7 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs @@ -49,8 +49,13 @@ public static void AddGoogleCloudStorage(this IServiceCollection services, Googl return new GoogleCloudStorage(storageClient, logger); }); - services.AddHealthChecks() - .Add(new HealthCheckRegistration("Google Cloud", sp => new GoogleCloudStorageHealthCheck(sp.GetRequiredService(), options), HealthStatus.Unhealthy, null)); + services.AddHealthChecks().Add( + new HealthCheckRegistration( + "blob_storage", + sp => new BlobStorageHealthCheck(sp.GetRequiredService(), options.BucketName), + HealthStatus.Unhealthy, null + ) + ); } } From 2385c648ae0de8283edf93879d85dbc4feeb89c6 Mon Sep 17 00:00:00 2001 From: Mika Herrmann Date: Fri, 11 Oct 2024 13:05:15 +0200 Subject: [PATCH 10/11] refactor: Register the common Health check in the common `AddBlobStorage` method --- ...eStorageAccountServiceCollectionExtensions.cs | 10 ---------- .../BlobStorageServiceCollectionExtensions.cs | 16 +++++++++++----- ...gleCloudStorageServiceCollectionExtensions.cs | 9 --------- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs index cdba64017d..da3acfb0d9 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/AzureStorageAccount/AzureStorageAccountServiceCollectionExtensions.cs @@ -1,6 +1,5 @@ using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; @@ -21,19 +20,10 @@ public static void AddAzureStorageAccount(this IServiceCollection services, Azur services.Configure(opt => opt.ConnectionString = options.ConnectionString); services.AddSingleton(); services.AddScoped(); - - services.AddHealthChecks().Add( - new HealthCheckRegistration( - "blob_storage", - sp => new BlobStorageHealthCheck(sp.GetRequiredService(), options.BucketName), - HealthStatus.Unhealthy, null - ) - ); } } public class AzureStorageAccountOptions { public string ConnectionString { get; set; } = string.Empty; - public string BucketName { get; set; } = string.Empty; } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index 23ef7c24f2..f6c9c85826 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -1,7 +1,9 @@ +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; using Backbone.Tooling.Extensions; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage; @@ -23,11 +25,7 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO switch (options.CloudProvider) { case AZURE_CLOUD_PROVIDER: - services.AddAzureStorageAccount(azureStorageAccountOptions => - { - azureStorageAccountOptions.ConnectionString = options.ConnectionInfo!; - azureStorageAccountOptions.BucketName = options.Container; - }); + services.AddAzureStorageAccount(azureStorageAccountOptions => { azureStorageAccountOptions.ConnectionString = options.ConnectionInfo!; }); break; case GOOGLE_CLOUD_PROVIDER: services.AddGoogleCloudStorage(googleCloudStorageOptions => @@ -45,6 +43,14 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO $"{options.CloudProvider} is not a currently supported cloud provider."); } } + + services.AddHealthChecks().Add( + new HealthCheckRegistration( + "blob_storage", + sp => new BlobStorageHealthCheck(sp.GetRequiredService(), options.Container), + HealthStatus.Unhealthy, null + ) + ); } } diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs index 12c31eeda7..cb4f12835b 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/GoogleCloudStorage/GoogleCloudStorageServiceCollectionExtensions.cs @@ -3,7 +3,6 @@ using Google.Apis.Auth.OAuth2; using Google.Cloud.Storage.V1; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; @@ -48,14 +47,6 @@ public static void AddGoogleCloudStorage(this IServiceCollection services, Googl return new GoogleCloudStorage(storageClient, logger); }); - - services.AddHealthChecks().Add( - new HealthCheckRegistration( - "blob_storage", - sp => new BlobStorageHealthCheck(sp.GetRequiredService(), options.BucketName), - HealthStatus.Unhealthy, null - ) - ); } } From 39875c14bc8951c99b9064da412eae7919f1f7d6 Mon Sep 17 00:00:00 2001 From: Timo Notheisen Date: Mon, 14 Oct 2024 13:46:51 +0200 Subject: [PATCH 11/11] chore: replace Azure blob storage library with a more generic one --- .../BuildingBlocks.Infrastructure.csproj | 2 +- .../Persistence/BlobStorage/BlobStorageHealthCheck.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index d5ba547d98..cfddde7c78 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -11,6 +11,7 @@ + @@ -19,7 +20,6 @@ - diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs index a2efb637f1..b6cf815560 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageHealthCheck.cs @@ -11,7 +11,7 @@ public class BlobStorageHealthCheck : IHealthCheck private const int MAX_NUMBER_OF_TRIES = 5; private static bool? _isHealthy; - private static int _numberOfTries = 0; + private static int _numberOfTries; private readonly IBlobStorage _storage; private readonly string _bucketName;