Support S3 buckets as blob storage

Applications/FilesSanityCheck/src/FilesSanityCheck/Extensions/IConfigurationExtensions.cs

@@ -31,12 +31,27 @@ public class BlobStorageConfiguration
 {
     [Required]
     [MinLength(1)]
-    [RegularExpression("Azure|GoogleCloud")]
+    [RegularExpression("Azure|GoogleCloud|S3")]
     public string CloudProvider { get; set; } = string.Empty;
 
-    [Required]
-    [MinLength(1)]
     public string ConnectionInfo { get; set; } = string.Empty;
 
     public string ContainerName { get; set; } = string.Empty;
+
+    public S3Config? S3Config { get; set; }
+}
+
+public class S3Config
+{
+    [Required]
+    public string ServiceUrl { get; set; } = string.Empty;
+
+    [Required]
+    public string AccessKey { get; set; } = string.Empty;
+
+    [Required]
+    public string SecretKey { get; set; } = string.Empty;
+
+    [Required]
+    public string BucketName { get; set; } = string.Empty;
 }

BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj

@@ -2,6 +2,7 @@
 
     <ItemGroup>
         <PackageReference Include="Autofac" Version="8.1.1" />
+        <PackageReference Include="AWSSDK.S3" Version="3.7.309.4" />
         <PackageReference Include="Azure.Messaging.ServiceBus" Version="7.18.2" />
         <PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
         <PackageReference Include="Google.Cloud.PubSub.V1" Version="3.19.0" />

BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs

@@ -1,6 +1,8 @@
+using System.ComponentModel.DataAnnotations;
 using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage;
 using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount;
 using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage;
+using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3;
 using Backbone.Tooling.Extensions;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
@@ -11,6 +13,7 @@ public static class BlobStorageServiceCollectionExtensions
 {
     public const string AZURE_CLOUD_PROVIDER = "Azure";
     public const string GOOGLE_CLOUD_PROVIDER = "GoogleCloud";
+    public const string S3_CLOUD_PROVIDER = "S3";
 
     public static void AddBlobStorage(this IServiceCollection services, Action<BlobStorageOptions> setupOptions)
     {
@@ -34,6 +37,19 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO
                     googleCloudStorageOptions.BucketName = options.Container;
                 });
                 break;
+            case S3_CLOUD_PROVIDER:
+                services.Configure<S3Options>(opt =>
+                {
+                    opt.ServiceUrl = options.S3Config!.ServiceUrl;
+                    opt.KeyId = options.S3Config!.AccessKey;
+                    opt.Key = options.S3Config!.SecretKey;
+                    opt.BucketName = options.S3Config!.BucketName;
+                });
+
+                services.AddScoped<IBlobStorage, S3BlobStorage>();
+
+                break;
+
             default:
             {
                 if (options.CloudProvider.IsNullOrEmpty())
@@ -56,9 +72,29 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO
 
 public class BlobStorageOptions
 {
+    [Required]
+    [MinLength(1)]
+    [RegularExpression("Azure|GoogleCloud|S3")]
     public string CloudProvider { get; set; } = null!;
 
     public string Container { get; set; } = null!;
 
-    public string? ConnectionInfo { get; set; }
+    public string? ConnectionInfo { get; set; } = null;
+
+    public S3Config? S3Config { get; set; }
+}
+
+public class S3Config
+{
+    [Required]
+    public string ServiceUrl { get; set; } = string.Empty;
+
+    [Required]
+    public string AccessKey { get; set; } = string.Empty;
+
+    [Required]
+    public string SecretKey { get; set; } = string.Empty;
+
+    [Required]
+    public string BucketName { get; set; } = string.Empty;
 }

BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs

@@ -0,0 +1,217 @@
+using System.Net;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Amazon.S3.Transfer;
+using Backbone.BuildingBlocks.Application.Abstractions.Exceptions;
+using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3;
+
+public class S3BlobStorage : IBlobStorage, IDisposable
+{
+    private readonly IAmazonS3 _s3Client;
+    private readonly List<ChangedBlob> _changedBlobs;
+    private readonly IList<RemovedBlob> _removedBlobs;
+    private readonly string _bucketName;
+    private readonly ILogger<S3BlobStorage> _logger;
+
+    public S3BlobStorage(IOptions<S3Options> config, ILogger<S3BlobStorage> logger)
+    {
+        var s3Config = new AmazonS3Config
+        {
+            ServiceURL = config.Value.ServiceUrl,
+            ForcePathStyle = true
+        };
+
+        _s3Client = new AmazonS3Client(config.Value.KeyId, config.Value.Key, s3Config);
+        _changedBlobs = new List<ChangedBlob>();
+        _removedBlobs = new List<RemovedBlob>();
+        _bucketName = config.Value.BucketName;
+        _logger = logger;
+    }
+
+    public void Add(string folder, string id, byte[] content)
+    {
+        _changedBlobs.Add(new ChangedBlob(folder, id, content));
+    }
+
+    public void Remove(string folder, string id)
+    {
+        _removedBlobs.Add(new RemovedBlob(folder, id));
+    }
+
+    public void Dispose()
+    {
+        _changedBlobs.Clear();
+        _removedBlobs.Clear();
+    }
+
+    public async Task<byte[]> FindAsync(string folder, string id)
+    {
+        _logger.LogTrace("Reading blob with key '{blobId}'...", id);
+
+        try
+        {
+            var request = new GetObjectRequest
+            {
+                BucketName = _bucketName,
+                Key = $"{folder}/{id}"
+            };
+
+            using var response = await _s3Client.GetObjectAsync(request);
+            using var memoryStream = new MemoryStream();
+            await response.ResponseStream.CopyToAsync(memoryStream);
+
+            _logger.LogTrace("Found blob with key '{blobId}'.", id);
+            return memoryStream.ToArray();
+        }
+        catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+        {
+            _logger.LogError("A blob with key '{blobId}' was not found.", id);
+            throw new NotFoundException("Blob", ex);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error downloading blob with key '{blobId}'.", id);
+            throw;
+        }
+    }
+
+    public Task<IAsyncEnumerable<string>> FindAllAsync(string folder, string? prefix = null)
+    {
+        return Task.FromResult(FindAllBlobsAsync(folder, prefix));
+    }
+
+    private async IAsyncEnumerable<string> FindAllBlobsAsync(string folder, string? prefix)
+    {
+        _logger.LogTrace("Listing all blobs...");
+
+        var request = new ListObjectsV2Request
+        {
+            BucketName = _bucketName,
+            Prefix = prefix != null ? $"{folder}/{prefix}" : folder
+        };
+
+        ListObjectsV2Response response;
+        do
+        {
+            response = await _s3Client.ListObjectsV2Async(request);
+
+            foreach (var obj in response.S3Objects)
+            {
+                yield return obj.Key;
+            }
+
+            request.ContinuationToken = response.NextContinuationToken;
+        } while (response.IsTruncated);
+
+        _logger.LogTrace("Found all blobs.");
+    }
+
+    public async Task SaveAsync()
+    {
+        await UploadChangedBlobs();
+        await DeleteRemovedBlobs();
+    }
+
+    private async Task UploadChangedBlobs()
+    {
+        _logger.LogTrace("Uploading '{changedBlobsCount}' changed blobs...", _changedBlobs.Count);
+
+        var changedBlobs = new List<ChangedBlob>(_changedBlobs);
+
+        foreach (var blob in changedBlobs)
+        {
+            await EnsureKeyDoesNotExist(blob.Folder, blob.Name);
+
+            using var memoryStream = new MemoryStream(blob.Content);
+
+            try
+            {
+                _logger.LogTrace("Uploading blob with key '{blobName}'...", blob.Name);
+
+                var request = new TransferUtilityUploadRequest
+                {
+                    InputStream = memoryStream,
+                    Key = $"{blob.Folder}/{blob.Name}",
+                    BucketName = _bucketName
+                };
+
+                var transferUtility = new TransferUtility(_s3Client);
+                await transferUtility.UploadAsync(request);
+
+                _logger.LogTrace("Upload of blob with key '{blobName}' was successful.", blob.Name);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error uploading blob with key '{blobName}'.", blob.Name);
+                throw;
+            }
+            finally
+            {
+                _changedBlobs.Remove(blob);
+            }
+        }
+    }
+
+    private async Task EnsureKeyDoesNotExist(string folder, string key)
+    {
+        try
+        {
+            var request = new GetObjectRequest
+            {
+                BucketName = _bucketName,
+                Key = $"{folder}/{key}"
+            };
+
+            await _s3Client.GetObjectAsync(request);
+
+            _logger.LogError("A blob with key '{blobName}' already exists.", key);
+            throw new BlobAlreadyExistsException(key);
+        }
+        catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+        {
+        }
+    }
+
+    private async Task DeleteRemovedBlobs()
+    {
+        _logger.LogTrace("Deleting '{removedBlobsCount}' blobs...", _removedBlobs.Count);
+
+        var blobsToDelete = new List<RemovedBlob>(_removedBlobs);
+
+        foreach (var blob in blobsToDelete)
+        {
+            try
+            {
+                var request = new DeleteObjectRequest
+                {
+                    BucketName = _bucketName,
+                    Key = $"{blob.Folder}/{blob.Name}"
+                };
+
+                await _s3Client.DeleteObjectAsync(request);
+
+                _removedBlobs.Remove(blob);
+            }
+            catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                _logger.LogError("A blob with key '{blobId}' was not found.", blob.Name);
+                throw new NotFoundException($"Blob with key '{blob.Name}' was not found.", ex);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error deleting blob with key '{blobName}'.", blob.Name);
+                throw;
+            }
+        }
+
+        _logger.LogTrace("Deletion successful.");
+    }
+
+    private record ChangedBlob(string Folder, string Name, byte[] Content);
+
+    private record RemovedBlob(string Folder, string Name);
+}

BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs

@@ -0,0 +1,37 @@
+using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3;
+
+public static class S3ServiceCollectionExtensions
+{
+    public static void AddS3(this IServiceCollection services,
+        Action<S3Options> setupOptions)
+    {
+        var options = new S3Options();
+        setupOptions.Invoke(options);
+
+        services.AddS3(options);
+    }
+
+    public static void AddS3(this IServiceCollection services, S3Options options)
+    {
+        services.Configure<S3Options>(opt =>
+        {
+            opt.ServiceUrl = options.ServiceUrl;
+            opt.KeyId = options.KeyId;
+            opt.Key = options.Key;
+            opt.BucketName = options.BucketName;
+        });
+
+        services.AddScoped<IBlobStorage, S3BlobStorage>();
+    }
+}
+
+public class S3Options
+{
+    public string ServiceUrl { get; set; } = string.Empty;
+    public string KeyId { get; set; } = string.Empty;
+    public string Key { get; set; } = string.Empty;
+    public string BucketName { get; set; } = string.Empty;
+}

Modules/Files/src/Files.ConsumerApi/Configuration.cs

@@ -23,12 +23,14 @@ public class BlobStorageConfiguration
         {
             [Required]
             [MinLength(1)]
-            [RegularExpression("Azure|GoogleCloud")]
+            [RegularExpression("Azure|GoogleCloud|S3")]
             public string CloudProvider { get; set; } = string.Empty;
 
             public string ConnectionInfo { get; set; } = string.Empty;
 
             public string ContainerName { get; set; } = string.Empty;
+
+            public S3Config? S3Config { get; set; }
         }
 
         public class SqlDatabaseConfiguration
@@ -45,5 +47,20 @@ public class SqlDatabaseConfiguration
             [Required]
             public bool EnableHealthCheck { get; set; } = true;
         }
+
+        public class S3Config
+        {
+            [Required]
+            public string ServiceUrl { get; set; } = string.Empty;
+
+            [Required]
+            public string AccessKey { get; set; } = string.Empty;
+
+            [Required]
+            public string SecretKey { get; set; } = string.Empty;
+
+            [Required]
+            public string BucketName { get; set; } = string.Empty;
+        }
     }
 }