diff --git a/Geodatenbezug.Test/ProcessingTest.cs b/Geodatenbezug.Test/ProcessingTest.cs index 77f5492..cf657ba 100644 --- a/Geodatenbezug.Test/ProcessingTest.cs +++ b/Geodatenbezug.Test/ProcessingTest.cs @@ -1,4 +1,5 @@ -using Geodatenbezug.Models; +using System.Globalization; +using Geodatenbezug.Models; using Microsoft.Extensions.Logging; using Moq; @@ -65,12 +66,17 @@ public async Task GetTopicsToUpdate() }, ]); - loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (SH) wurde am {datestring_delta4:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet"); - loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (ZG) wurde am {datestring_delta23:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet"); - loggerMock.Setup(LogLevel.Information, $"Thema Rebbaukataster (SH) wurde seit {datestring_delta30:yyyy-MM-dd HH:mm:ss} nicht aktualisiert"); + loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (SH) wurde am {datestring_delta4.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet"); + loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (ZG) wurde am {datestring_delta23.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet"); + loggerMock.Setup(LogLevel.Information, $"Thema Rebbaukataster (SH) wurde seit {datestring_delta30.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} nicht aktualisiert"); loggerMock.Setup(LogLevel.Information, "Thema Rebbaukataster (ZG) ist nicht verfügbar"); loggerMock.Setup(LogLevel.Information, "2 Themen werden prozessiert"); + azureStorageMock.SetupSequence(storage => storage.GetLastProcessed(It.IsAny())) + .ReturnsAsync(datestring_delta23) + .ReturnsAsync((DateTime?)null) + .ReturnsAsync(datestring_delta23); + var topicsToProcess = await new Processor(geodiensteApiMock.Object, azureStorageMock.Object, loggerMock.Object).GetTopicsToProcess(); Assert.AreEqual(2, topicsToProcess.Count); Assert.AreEqual(BaseTopic.lwb_perimeter_ln_sf, topicsToProcess[0].BaseTopic); diff --git a/Geodatenbezug/AzureStorage.cs b/Geodatenbezug/AzureStorage.cs index 0d0ceb7..4e6799c 100644 --- a/Geodatenbezug/AzureStorage.cs +++ b/Geodatenbezug/AzureStorage.cs @@ -1,6 +1,8 @@ using Azure.Storage; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Azure.Storage.Sas; +using Geodatenbezug.Models; using Microsoft.Extensions.Logging; namespace Geodatenbezug; @@ -13,18 +15,60 @@ public class AzureStorage(ILogger logger) : IAzureStorage #pragma warning restore SA1009 // Closing parenthesis should be spaced correctly { private const string StorageContainerName = "processed-topics"; + private string connectionString = string.Empty; + + /// + /// Connection string to the Azure Storage. + /// + protected string ConnectionString + { + get + { + if (string.IsNullOrEmpty(connectionString)) + { + var value = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); + if (string.IsNullOrEmpty(value)) + { + throw new InvalidOperationException("AzureWebJobsStorage is not set"); + } + + connectionString = value; + } + + return connectionString; + } + } /// - public async Task UploadFileAsync(string storageFilePath, string localFilePath) + public async Task GetLastProcessed(Topic topic) { - logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch"); - var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); - if (string.IsNullOrEmpty(connectionString)) + logger.LogInformation($"Frage letzte Prozessierung des Themas {topic.TopicTitle} ({topic.Canton}) ab"); + var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName); + var creationDates = new List(); + + await foreach (BlobItem blobItem in containerClient.GetBlobsAsync()) { - throw new InvalidOperationException("AzureWebJobsStorage is not set"); + if (blobItem.Name.Contains(topic.Canton.ToString(), StringComparison.OrdinalIgnoreCase) + && blobItem.Name.Contains(topic.BaseTopic.ToString(), StringComparison.OrdinalIgnoreCase)) + { + creationDates.Add(blobItem.Properties.CreatedOn.GetValueOrDefault().DateTime); + } } - var containerClient = new BlobServiceClient(connectionString).GetBlobContainerClient(StorageContainerName); + if (creationDates.Count > 0) + { + return creationDates.Max(); + } + + return null; + } + + /// + public async Task UploadFileAsync(string storageFilePath, string localFilePath) + { + logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch..."); + + var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName); var blobClient = containerClient.GetBlobClient(storageFilePath); using var localFileStream = File.OpenRead(localFilePath); @@ -42,8 +86,8 @@ public async Task UploadFileAsync(string storageFilePath, string localFi }; sasBuilder.SetPermissions(BlobSasPermissions.Read); - var accountName = connectionString.ExtractValueByKey("AccountName"); - var accountKey = connectionString.ExtractValueByKey("AccountKey"); + var accountName = ConnectionString.ExtractValueByKey("AccountName"); + var accountKey = ConnectionString.ExtractValueByKey("AccountKey"); string sasToken = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(accountName, accountKey)).ToString(); return $"{blobClient.Uri}?{sasToken}"; } diff --git a/Geodatenbezug/IAzureStorage.cs b/Geodatenbezug/IAzureStorage.cs index 8a08856..152d161 100644 --- a/Geodatenbezug/IAzureStorage.cs +++ b/Geodatenbezug/IAzureStorage.cs @@ -1,10 +1,17 @@ -namespace Geodatenbezug; +using Geodatenbezug.Models; + +namespace Geodatenbezug; /// /// Interface for Azure Storage. /// public interface IAzureStorage { + /// + /// Gets the date of the last processed file for the given topic. + /// + Task GetLastProcessed(Topic topic); + /// /// Uploads the file to the specified container and return the download link. /// diff --git a/Geodatenbezug/Processor.cs b/Geodatenbezug/Processor.cs index 1d6ffb3..19683c6 100644 --- a/Geodatenbezug/Processor.cs +++ b/Geodatenbezug/Processor.cs @@ -16,30 +16,29 @@ public class Processor(IGeodiensteApi geodiensteApi, IAzureStorage azureStorage, public async Task> GetTopicsToProcess() { var topics = await geodiensteApi.RequestTopicInfoAsync().ConfigureAwait(false); - var currentTime = DateTime.Now; - var topicsToProcess = topics.FindAll(topic => + var topicsToProcess = new List(); + foreach (var topic in topics) { if (topic.UpdatedAt.HasValue) { - var updatedAtString = topic.UpdatedAt.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); - var timeDifference = currentTime - topic.UpdatedAt.Value; - if (timeDifference.Days < 1) + var updatedAtString = topic.UpdatedAt.Value.ToString("G", CultureInfo.GetCultureInfo("de-CH")); + + var lastProcessed = await azureStorage.GetLastProcessed(topic).ConfigureAwait(false); + if (lastProcessed == null || lastProcessed < topic.UpdatedAt.Value) { logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) wurde am {updatedAtString} aktualisiert und wird verarbeitet"); - return true; + topicsToProcess.Add(topic); } else { logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) wurde seit {updatedAtString} nicht aktualisiert"); - return false; } } else { logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) ist nicht verfügbar"); - return false; } - }); + } var topicsProcessedMessage = topicsToProcess.Count != 1 ? "Themen werden" : "Thema wird"; logger.LogInformation($"{topicsToProcess.Count} {topicsProcessedMessage} prozessiert");