Get last results from storage (#54)

blw-ofag-ufag · May 2, 2024 · c7e0c96 · c7e0c96
2 parents 3e43f31 + 2f305b6
commit c7e0c96
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 22 deletions.
diff --git a/Geodatenbezug.Test/ProcessingTest.cs b/Geodatenbezug.Test/ProcessingTest.cs
@@ -1,4 +1,5 @@
-using Geodatenbezug.Models;
+using System.Globalization;
+using Geodatenbezug.Models;
 using Microsoft.Extensions.Logging;
 using Moq;
 
@@ -65,12 +66,17 @@ public async Task GetTopicsToUpdate()
                 },
             ]);
 
-        loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (SH) wurde am {datestring_delta4:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet");
-        loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (ZG) wurde am {datestring_delta23:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet");
-        loggerMock.Setup(LogLevel.Information, $"Thema Rebbaukataster (SH) wurde seit {datestring_delta30:yyyy-MM-dd HH:mm:ss} nicht aktualisiert");
+        loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (SH) wurde am {datestring_delta4.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet");
+        loggerMock.Setup(LogLevel.Information, $"Thema Perimeter LN- und Sömmerungsflächen (ZG) wurde am {datestring_delta23.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet");
+        loggerMock.Setup(LogLevel.Information, $"Thema Rebbaukataster (SH) wurde seit {datestring_delta30.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} nicht aktualisiert");
         loggerMock.Setup(LogLevel.Information, "Thema Rebbaukataster (ZG) ist nicht verfügbar");
         loggerMock.Setup(LogLevel.Information, "2 Themen werden prozessiert");
 
+        azureStorageMock.SetupSequence(storage => storage.GetLastProcessed(It.IsAny<Topic>()))
+            .ReturnsAsync(datestring_delta23)
+            .ReturnsAsync((DateTime?)null)
+            .ReturnsAsync(datestring_delta23);
+
         var topicsToProcess = await new Processor(geodiensteApiMock.Object, azureStorageMock.Object, loggerMock.Object).GetTopicsToProcess();
         Assert.AreEqual(2, topicsToProcess.Count);
         Assert.AreEqual(BaseTopic.lwb_perimeter_ln_sf, topicsToProcess[0].BaseTopic);

diff --git a/Geodatenbezug/AzureStorage.cs b/Geodatenbezug/AzureStorage.cs
@@ -1,6 +1,8 @@
 using Azure.Storage;
 using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
 using Azure.Storage.Sas;
+using Geodatenbezug.Models;
 using Microsoft.Extensions.Logging;
 
 namespace Geodatenbezug;
@@ -13,18 +15,60 @@ public class AzureStorage(ILogger<AzureStorage> logger) : IAzureStorage
 #pragma warning restore SA1009 // Closing parenthesis should be spaced correctly
 {
     private const string StorageContainerName = "processed-topics";
+    private string connectionString = string.Empty;
+
+    /// <summary>
+    /// Connection string to the Azure Storage.
+    /// </summary>
+    protected string ConnectionString
+    {
+        get
+        {
+            if (string.IsNullOrEmpty(connectionString))
+            {
+                var value = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
+                if (string.IsNullOrEmpty(value))
+                {
+                    throw new InvalidOperationException("AzureWebJobsStorage is not set");
+                }
+
+                connectionString = value;
+            }
+
+            return connectionString;
+        }
+    }
 
     /// <inheritdoc />
-    public async Task<string> UploadFileAsync(string storageFilePath, string localFilePath)
+    public async Task<DateTime?> GetLastProcessed(Topic topic)
     {
-        logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch");
-        var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
-        if (string.IsNullOrEmpty(connectionString))
+        logger.LogInformation($"Frage letzte Prozessierung des Themas {topic.TopicTitle} ({topic.Canton}) ab");
+        var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName);
+        var creationDates = new List<DateTime>();
+
+        await foreach (BlobItem blobItem in containerClient.GetBlobsAsync())
         {
-            throw new InvalidOperationException("AzureWebJobsStorage is not set");
+            if (blobItem.Name.Contains(topic.Canton.ToString(), StringComparison.OrdinalIgnoreCase)
+                && blobItem.Name.Contains(topic.BaseTopic.ToString(), StringComparison.OrdinalIgnoreCase))
+            {
+                creationDates.Add(blobItem.Properties.CreatedOn.GetValueOrDefault().DateTime);
+            }
         }
 
-        var containerClient = new BlobServiceClient(connectionString).GetBlobContainerClient(StorageContainerName);
+        if (creationDates.Count > 0)
+        {
+            return creationDates.Max();
+        }
+
+        return null;
+    }
+
+    /// <inheritdoc />
+    public async Task<string> UploadFileAsync(string storageFilePath, string localFilePath)
+    {
+        logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch...");
+
+        var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName);
         var blobClient = containerClient.GetBlobClient(storageFilePath);
 
         using var localFileStream = File.OpenRead(localFilePath);
@@ -42,8 +86,8 @@ public async Task<string> UploadFileAsync(string storageFilePath, string localFi
         };
         sasBuilder.SetPermissions(BlobSasPermissions.Read);
 
-        var accountName = connectionString.ExtractValueByKey("AccountName");
-        var accountKey = connectionString.ExtractValueByKey("AccountKey");
+        var accountName = ConnectionString.ExtractValueByKey("AccountName");
+        var accountKey = ConnectionString.ExtractValueByKey("AccountKey");
         string sasToken = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(accountName, accountKey)).ToString();
         return $"{blobClient.Uri}?{sasToken}";
     }

diff --git a/Geodatenbezug/IAzureStorage.cs b/Geodatenbezug/IAzureStorage.cs
@@ -1,10 +1,17 @@
-namespace Geodatenbezug;
+using Geodatenbezug.Models;
+
+namespace Geodatenbezug;
 
 /// <summary>
 /// Interface for Azure Storage.
 /// </summary>
 public interface IAzureStorage
 {
+    /// <summary>
+    /// Gets the date of the last processed file for the given topic.
+    /// </summary>
+    Task<DateTime?> GetLastProcessed(Topic topic);
+
     /// <summary>
     /// Uploads the file to the specified container and return the download link.
     /// </summary>

diff --git a/Geodatenbezug/Processor.cs b/Geodatenbezug/Processor.cs
@@ -16,30 +16,29 @@ public class Processor(IGeodiensteApi geodiensteApi, IAzureStorage azureStorage,
     public async Task<List<Topic>> GetTopicsToProcess()
     {
         var topics = await geodiensteApi.RequestTopicInfoAsync().ConfigureAwait(false);
-        var currentTime = DateTime.Now;
-        var topicsToProcess = topics.FindAll(topic =>
+        var topicsToProcess = new List<Topic>();
+        foreach (var topic in topics)
         {
             if (topic.UpdatedAt.HasValue)
             {
-                var updatedAtString = topic.UpdatedAt.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
-                var timeDifference = currentTime - topic.UpdatedAt.Value;
-                if (timeDifference.Days < 1)
+                var updatedAtString = topic.UpdatedAt.Value.ToString("G", CultureInfo.GetCultureInfo("de-CH"));
+
+                var lastProcessed = await azureStorage.GetLastProcessed(topic).ConfigureAwait(false);
+                if (lastProcessed == null || lastProcessed < topic.UpdatedAt.Value)
                 {
                     logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) wurde am {updatedAtString} aktualisiert und wird verarbeitet");
-                    return true;
+                    topicsToProcess.Add(topic);
                 }
                 else
                 {
                     logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) wurde seit {updatedAtString} nicht aktualisiert");
-                    return false;
                 }
             }
             else
             {
                 logger.LogInformation($"Thema {topic.TopicTitle} ({topic.Canton}) ist nicht verfügbar");
-                return false;
             }
-        });
+        }
 
         var topicsProcessedMessage = topicsToProcess.Count != 1 ? "Themen werden" : "Thema wird";
         logger.LogInformation($"{topicsToProcess.Count} {topicsProcessedMessage} prozessiert");