From 7b75cd12f5ee487df0ae5a51415dc79a986739b1 Mon Sep 17 00:00:00 2001 From: tschumpr Date: Wed, 1 May 2024 14:32:15 +0200 Subject: [PATCH 1/5] Check for last update instead of fixed time --- Geodatenbezug.Test/ProcessingTest.cs | 14 +++++--- Geodatenbezug/AzureStorage.cs | 53 +++++++++++++++++++++++----- Geodatenbezug/IAzureStorage.cs | 9 ++++- Geodatenbezug/Processor.cs | 17 +++++---- 4 files changed, 71 insertions(+), 22 deletions(-) diff --git a/Geodatenbezug.Test/ProcessingTest.cs b/Geodatenbezug.Test/ProcessingTest.cs index 77f5492..cf657ba 100644 --- a/Geodatenbezug.Test/ProcessingTest.cs +++ b/Geodatenbezug.Test/ProcessingTest.cs @@ -1,4 +1,5 @@ -using Geodatenbezug.Models; +using System.Globalization; +using Geodatenbezug.Models; using Microsoft.Extensions.Logging; using Moq; @@ -65,12 +66,17 @@ public async Task GetTopicsToUpdate() }, ]); - loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (SH) wurde am {datestring_delta4:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet"); - loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (ZG) wurde am {datestring_delta23:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet"); - loggerMock.Setup(LogLevel.Information, $"Thema Rebbaukataster (SH) wurde seit {datestring_delta30:yyyy-MM-dd HH:mm:ss} nicht aktualisiert"); + loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (SH) wurde am {datestring_delta4.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet"); + loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (ZG) wurde am {datestring_delta23.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet"); + loggerMock.Setup(LogLevel.Information, $"Thema Rebbaukataster (SH) wurde seit {datestring_delta30.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} nicht aktualisiert"); loggerMock.Setup(LogLevel.Information, "Thema Rebbaukataster (ZG) ist nicht verfügbar"); loggerMock.Setup(LogLevel.Information, "2 Themen werden prozessiert"); + azureStorageMock.SetupSequence(storage => storage.GetLastProcessed(It.IsAny())) + .ReturnsAsync(datestring_delta23) + .ReturnsAsync((DateTime?)null) + .ReturnsAsync(datestring_delta23); + var topicsToProcess = await new Processor(geodiensteApiMock.Object, azureStorageMock.Object, loggerMock.Object).GetTopicsToProcess(); Assert.AreEqual(2, topicsToProcess.Count); Assert.AreEqual(BaseTopic.lwb_perimeter_ln_sf, topicsToProcess[0].BaseTopic); diff --git a/Geodatenbezug/AzureStorage.cs b/Geodatenbezug/AzureStorage.cs index b89ffc6..fd5d96b 100644 --- a/Geodatenbezug/AzureStorage.cs +++ b/Geodatenbezug/AzureStorage.cs @@ -1,6 +1,8 @@ using Azure.Storage; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Azure.Storage.Sas; +using Geodatenbezug.Models; using Microsoft.Extensions.Logging; namespace Geodatenbezug; @@ -13,18 +15,53 @@ public class AzureStorage(ILogger logger) : IAzureStorage #pragma warning restore SA1009 // Closing parenthesis should be spaced correctly { private const string StorageContainerName = "processed-topics"; + private string connectionString = string.Empty; + + /// + /// Connection string to the Azure Storage. + /// + protected string ConnectionString + { + get + { + if (string.IsNullOrEmpty(connectionString)) + { + var value = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); + if (string.IsNullOrEmpty(value)) + { + throw new InvalidOperationException("AzureWebJobsStorage is not set"); + } + + connectionString = value; + } + + return connectionString; + } + } /// - public async Task UploadFileAsync(string storageFilePath, string localFilePath) + public async Task GetLastProcessed(Topic topic) { - logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch..."); - var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); - if (string.IsNullOrEmpty(connectionString)) + var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName); + + await foreach (BlobItem blobItem in containerClient.GetBlobsAsync()) { - throw new InvalidOperationException("AzureWebJobsStorage is not set"); + if (blobItem.Name.Contains(topic.Canton.ToString(), StringComparison.InvariantCulture) + && blobItem.Name.Contains(topic.BaseTopic.ToString(), StringComparison.InvariantCulture)) + { + return blobItem.Properties.CreatedOn.GetValueOrDefault().DateTime; + } } - var containerClient = new BlobServiceClient(connectionString).GetBlobContainerClient(StorageContainerName); + return null; + } + + /// + public async Task UploadFileAsync(string storageFilePath, string localFilePath) + { + logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch..."); + + var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName); var blobClient = containerClient.GetBlobClient(storageFilePath); using var localFileStream = File.OpenRead(localFilePath); @@ -42,8 +79,8 @@ public async Task UploadFileAsync(string storageFilePath, string localFi }; sasBuilder.SetPermissions(BlobSasPermissions.Read); - var accountName = connectionString.ExtractValueByKey("AccountName"); - var accountKey = connectionString.ExtractValueByKey("AccountKey"); + var accountName = ConnectionString.ExtractValueByKey("AccountName"); + var accountKey = ConnectionString.ExtractValueByKey("AccountKey"); string sasToken = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(accountName, accountKey)).ToString(); return $"{blobClient.Uri}?{sasToken}"; } diff --git a/Geodatenbezug/IAzureStorage.cs b/Geodatenbezug/IAzureStorage.cs index 8a08856..152d161 100644 --- a/Geodatenbezug/IAzureStorage.cs +++ b/Geodatenbezug/IAzureStorage.cs @@ -1,10 +1,17 @@ -namespace Geodatenbezug; +using Geodatenbezug.Models; + +namespace Geodatenbezug; /// /// Interface for Azure Storage. /// public interface IAzureStorage { + /// + /// Gets the date of the last processed file for the given topic. + /// + Task GetLastProcessed(Topic topic); + /// /// Uploads the file to the specified container and return the download link. /// diff --git a/Geodatenbezug/Processor.cs b/Geodatenbezug/Processor.cs index 1d6ffb3..78e565b 100644 --- a/Geodatenbezug/Processor.cs +++ b/Geodatenbezug/Processor.cs @@ -16,30 +16,29 @@ public class Processor(IGeodiensteApi geodiensteApi, IAzureStorage azureStorage, public async Task> GetTopicsToProcess() { var topics = await geodiensteApi.RequestTopicInfoAsync().ConfigureAwait(false); - var currentTime = DateTime.Now; - var topicsToProcess = topics.FindAll(topic => + var topicsToProcess = new List(); + foreach (var topic in topics) { if (topic.UpdatedAt.HasValue) { - var updatedAtString = topic.UpdatedAt.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); - var timeDifference = currentTime - topic.UpdatedAt.Value; - if (timeDifference.Days < 1) + var updatedAtString = topic.UpdatedAt.Value.ToString("G", CultureInfo.InvariantCulture); + + var lastProcessed = await azureStorage.GetLastProcessed(topic).ConfigureAwait(false); + if (lastProcessed == null || lastProcessed < topic.UpdatedAt.Value) { logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) wurde am {updatedAtString} aktualisiert und wird verarbeitet"); - return true; + topicsToProcess.Add(topic); } else { logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) wurde seit {updatedAtString} nicht aktualisiert"); - return false; } } else { logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) ist nicht verfügbar"); - return false; } - }); + } var topicsProcessedMessage = topicsToProcess.Count != 1 ? "Themen werden" : "Thema wird"; logger.LogInformation($"{topicsToProcess.Count} {topicsProcessedMessage} prozessiert"); From fb4150ee760078fdb00f8c33deb58c07ce0b83d9 Mon Sep 17 00:00:00 2001 From: tschumpr Date: Wed, 1 May 2024 14:32:50 +0200 Subject: [PATCH 2/5] Fix datetime string format --- Geodatenbezug/Processor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Geodatenbezug/Processor.cs b/Geodatenbezug/Processor.cs index 78e565b..19683c6 100644 --- a/Geodatenbezug/Processor.cs +++ b/Geodatenbezug/Processor.cs @@ -21,7 +21,7 @@ public async Task> GetTopicsToProcess() { if (topic.UpdatedAt.HasValue) { - var updatedAtString = topic.UpdatedAt.Value.ToString("G", CultureInfo.InvariantCulture); + var updatedAtString = topic.UpdatedAt.Value.ToString("G", CultureInfo.GetCultureInfo("de-CH")); var lastProcessed = await azureStorage.GetLastProcessed(topic).ConfigureAwait(false); if (lastProcessed == null || lastProcessed < topic.UpdatedAt.Value) From 89ae66fd288f1f6786618a164cad5c76163948f8 Mon Sep 17 00:00:00 2001 From: tschumpr Date: Wed, 1 May 2024 14:41:42 +0200 Subject: [PATCH 3/5] Return latest date --- Geodatenbezug/AzureStorage.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Geodatenbezug/AzureStorage.cs b/Geodatenbezug/AzureStorage.cs index fd5d96b..9d04dfb 100644 --- a/Geodatenbezug/AzureStorage.cs +++ b/Geodatenbezug/AzureStorage.cs @@ -43,16 +43,22 @@ protected string ConnectionString public async Task GetLastProcessed(Topic topic) { var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName); + var creationDates = new List(); await foreach (BlobItem blobItem in containerClient.GetBlobsAsync()) { if (blobItem.Name.Contains(topic.Canton.ToString(), StringComparison.InvariantCulture) && blobItem.Name.Contains(topic.BaseTopic.ToString(), StringComparison.InvariantCulture)) { - return blobItem.Properties.CreatedOn.GetValueOrDefault().DateTime; + creationDates.Add(blobItem.Properties.CreatedOn.GetValueOrDefault().DateTime); } } + if (creationDates.Count > 0) + { + return creationDates.Max(); + } + return null; } From cefe73c4d60098212ac69c263baabfe7c8866315 Mon Sep 17 00:00:00 2001 From: tschumpr Date: Thu, 2 May 2024 09:53:26 +0200 Subject: [PATCH 4/5] Fix merge conflict --- Geodatenbezug/AzureStorage.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Geodatenbezug/AzureStorage.cs b/Geodatenbezug/AzureStorage.cs index 342966a..acab2da 100644 --- a/Geodatenbezug/AzureStorage.cs +++ b/Geodatenbezug/AzureStorage.cs @@ -42,8 +42,7 @@ protected string ConnectionString /// public async Task GetLastProcessed(Topic topic) { - logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch"); - + logger.LogInformation($"Frage letzte Prozessierung des Themas {topic.TopicTitle} ({topic.Canton}) ab"); var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName); var creationDates = new List(); From c3f858f1f43b11aa8af0753f8885ce55e2c5c6e4 Mon Sep 17 00:00:00 2001 From: tschumpr Date: Thu, 2 May 2024 13:34:52 +0200 Subject: [PATCH 5/5] Use strict stringcomparison for comparing enums --- Geodatenbezug/AzureStorage.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Geodatenbezug/AzureStorage.cs b/Geodatenbezug/AzureStorage.cs index acab2da..4e6799c 100644 --- a/Geodatenbezug/AzureStorage.cs +++ b/Geodatenbezug/AzureStorage.cs @@ -48,8 +48,8 @@ protected string ConnectionString await foreach (BlobItem blobItem in containerClient.GetBlobsAsync()) { - if (blobItem.Name.Contains(topic.Canton.ToString(), StringComparison.InvariantCulture) - && blobItem.Name.Contains(topic.BaseTopic.ToString(), StringComparison.InvariantCulture)) + if (blobItem.Name.Contains(topic.Canton.ToString(), StringComparison.OrdinalIgnoreCase) + && blobItem.Name.Contains(topic.BaseTopic.ToString(), StringComparison.OrdinalIgnoreCase)) { creationDates.Add(blobItem.Properties.CreatedOn.GetValueOrDefault().DateTime); }