Merge remote-tracking branch 'origin/main' into update-logs

blw-ofag-ufag · May 2, 2024 · a66ce95 · a66ce95
2 parents 6455c6a + 739817c
commit a66ce95
Show file tree

Hide file tree

Showing 17 changed files with 518 additions and 46 deletions.
diff --git a/Geodatenbezug.Test/MailServiceTest.cs b/Geodatenbezug.Test/MailServiceTest.cs
@@ -0,0 +1,177 @@
+using System.Net;
+using Geodatenbezug.Models;
+using Microsoft.Extensions.Logging;
+using Moq;
+
+namespace Geodatenbezug;
+
+[TestClass]
+public class MailServiceTest
+{
+    private Mock<ILogger<MailService>> loggerMock;
+    private MailService mailService;
+
+    [TestInitialize]
+    public void Initialize()
+    {
+        loggerMock = new Mock<ILogger<MailService>>(MockBehavior.Strict);
+        mailService = new MailService(loggerMock.Object);
+    }
+
+    [TestCleanup]
+    public void Cleanup()
+    {
+        loggerMock.VerifyAll();
+    }
+
+    [TestMethod]
+    public void BuildMailMessage()
+    {
+        var results = new List<ProcessingResult>
+        {
+            new ()
+            {
+                Code = HttpStatusCode.OK,
+                Reason = "Success",
+                Info = "Data processed successfully",
+                TopicTitle = "Perimeter LN- und Sömmerungsflächen",
+                Canton = Canton.SH,
+                UpdatedAt = new DateTime(2023, 11, 05, 15, 33, 22),
+                DownloadUrl = "https://a-test.ch/link.zip",
+            },
+        };
+
+        using var mailMessage = mailService.BuildMailMessage(results);
+        Assert.AreEqual("notifications@geowerkstatt.ch", mailMessage.From.Address);
+        Assert.AreEqual("geodaten@blw.admin.ch", mailMessage.To.First().Address);
+        Assert.AreEqual(0, mailMessage.CC.Count);
+        Assert.AreEqual("Geodatenbezug Prozessierungsresultate", mailMessage.Subject);
+    }
+
+    [TestMethod]
+    public void BuildMailMessageWithNotOkProcessingResult()
+    {
+        var results = new List<ProcessingResult>
+        {
+            new ()
+            {
+                Code = HttpStatusCode.OK,
+                Reason = "Success",
+                Info = "Data processed successfully",
+                TopicTitle = "Perimeter LN- und Sömmerungsflächen",
+                Canton = Canton.SH,
+                UpdatedAt = new DateTime(2023, 11, 05, 15, 33, 22),
+                DownloadUrl = "https://a-test.ch/link.zip",
+            },
+            new ()
+            {
+                Code = HttpStatusCode.NotFound,
+                Reason = "Kein Wert für Key SH gefunden",
+                TopicTitle = "Perimeter Terrassenreben",
+                Canton = Canton.SH,
+                UpdatedAt = new DateTime(2023, 11, 05, 15, 33, 22),
+            },
+        };
+
+        using var mailMessage = mailService.BuildMailMessage(results);
+        Assert.AreEqual("notifications@geowerkstatt.ch", mailMessage.From.Address);
+        Assert.AreEqual("geodaten@blw.admin.ch", mailMessage.To.First().Address);
+        Assert.AreEqual(1, mailMessage.CC.Count);
+        Assert.AreEqual("support@geowerkstatt.ch", mailMessage.CC.First().Address);
+        Assert.AreEqual("Geodatenbezug Prozessierungsresultate", mailMessage.Subject);
+    }
+
+    [TestMethod]
+    public void BuildMailBody()
+    {
+        var expectedMailBody = @"<p>Guten Tag</p>
+<p>Die folgenden Themen wurden erfolgreich prozessiert:</p>
+<table>
+<tr>
+<th style=""text-align: left; padding: 10px;"">Thema</th>
+<th style=""text-align: left; padding: 10px;"">Kanton</th>
+<th style=""text-align: left; padding: 10px;"">Aktualisiert am</th>
+<th style=""text-align: left; padding: 10px;""></th>
+</tr>
+<tr>
+<td style=""text-align: left; padding: 10px;"">Perimeter LN- und Sömmerungsflächen</td>
+<td style=""text-align: left; padding: 10px;"">SH</td>
+<td style=""text-align: left; padding: 10px;"">05.11.2023 15:33:22</td>
+<td style=""text-align: left; padding: 10px;""><a href=""https://a-test.ch/link.zip"">Herunterladen</a></td>
+</tr>
+<tr>
+<td style=""text-align: left; padding: 10px;"">Nutzungsflaechen</td>
+<td style=""text-align: left; padding: 10px;"">SH</td>
+<td style=""text-align: left; padding: 10px;""></td>
+<td style=""text-align: left; padding: 10px;""><a href=""https://a-test.ch/link.zip"">Herunterladen</a></td>
+</tr>
+</table>
+<br />
+<p>Bei folgenden Themen traten während der Prozessierung Fehler auf:</p>
+<table>
+<tr>
+<th style=""text-align: left; padding: 10px;"">Thema</th>
+<th style=""text-align: left; padding: 10px;"">Kanton</th>
+<th style=""text-align: left; padding: 10px;"">Aktualisiert am</th>
+<th style=""text-align: left; padding: 10px;"">Fehler</th>
+</tr>
+<tr>
+<td style=""text-align: left; padding: 10px;"">Rebbaukaster</td>
+<td style=""text-align: left; padding: 10px;"">SH</td>
+<td style=""text-align: left; padding: 10px;"">05.11.2023 15:33:22</td>
+<td style=""text-align: left; padding: 10px;"">Not Found - Data export information not found. Invalid token?</td>
+</tr>
+<tr>
+<td style=""text-align: left; padding: 10px;"">Perimeter Terrassenreben</td>
+<td style=""text-align: left; padding: 10px;"">SH</td>
+<td style=""text-align: left; padding: 10px;"">05.11.2023 15:33:22</td>
+<td style=""text-align: left; padding: 10px;"">Kein Wert für Key SH gefunden</td>
+</tr>
+</table>
+";
+
+        var results = new List<ProcessingResult>
+        {
+            new ()
+            {
+                Code = HttpStatusCode.OK,
+                Reason = "Success",
+                Info = "Data processed successfully",
+                TopicTitle = "Perimeter LN- und Sömmerungsflächen",
+                Canton = Canton.SH,
+                UpdatedAt = new DateTime(2023, 11, 05, 15, 33, 22),
+                DownloadUrl = "https://a-test.ch/link.zip",
+            },
+            new ()
+            {
+                Code = HttpStatusCode.NotFound,
+                Reason = "Not Found",
+                Info = GeodiensteExportError.InvalidToken,
+                TopicTitle = "Rebbaukaster",
+                Canton = Canton.SH,
+                UpdatedAt = new DateTime(2023, 11, 05, 15, 33, 22),
+            },
+            new ()
+            {
+                Code = HttpStatusCode.NotFound,
+                Reason = "Kein Wert für Key SH gefunden",
+                TopicTitle = "Perimeter Terrassenreben",
+                Canton = Canton.SH,
+                UpdatedAt = new DateTime(2023, 11, 05, 15, 33, 22),
+            },
+            new ()
+            {
+                Code = HttpStatusCode.OK,
+                Reason = "Success",
+                Info = "Data processed successfully",
+                TopicTitle = "Nutzungsflaechen",
+                Canton = Canton.SH,
+                UpdatedAt = null,
+                DownloadUrl = "https://a-test.ch/link.zip",
+            },
+        };
+
+        var body = mailService.BuildMailBody(results);
+        Assert.AreEqual(expectedMailBody, body);
+    }
+}
diff --git a/Geodatenbezug.Test/ProcessingTest.cs b/Geodatenbezug.Test/ProcessingTest.cs
@@ -1,4 +1,5 @@
-using Geodatenbezug.Models;
+using System.Globalization;
+using Geodatenbezug.Models;
 using Microsoft.Extensions.Logging;
 using Moq;
 
@@ -10,13 +11,17 @@ public class ProcessingTest
     private Mock<ILogger<Processor>> loggerMock;
     private Mock<IGeodiensteApi> geodiensteApiMock;
     private Mock<IAzureStorage> azureStorageMock;
+    private Mock<IMailService> mailServiceMock;
+    private Processor processor;
 
     [TestInitialize]
     public void Initialize()
     {
         loggerMock = new Mock<ILogger<Processor>>(MockBehavior.Strict);
         geodiensteApiMock = new Mock<IGeodiensteApi>(MockBehavior.Strict);
         azureStorageMock = new Mock<IAzureStorage>(MockBehavior.Strict);
+        mailServiceMock = new Mock<IMailService>(MockBehavior.Strict);
+        processor = new Processor(geodiensteApiMock.Object, azureStorageMock.Object, loggerMock.Object, mailServiceMock.Object);
     }
 
     [TestCleanup]
@@ -65,13 +70,19 @@ public async Task GetTopicsToUpdate()
                 },
             ]);
 
-        loggerMock.Setup(LogLevel.Information, $"Perimeter LN- und Sömmerungsflächen (SH): Thema wurde am {datestring_delta4:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet");
-        loggerMock.Setup(LogLevel.Information, $"Perimeter LN- und Sömmerungsflächen (ZG): Thema wurde am {datestring_delta23:yyyy-MM-dd HH:mm:ss} aktualisiert und wird verarbeitet");
-        loggerMock.Setup(LogLevel.Information, $"Rebbaukataster (SH): Thema wurde seit {datestring_delta30:yyyy-MM-dd HH:mm:ss} nicht aktualisiert");
+        loggerMock.Setup(LogLevel.Information, "Laden der Themen...");
+        loggerMock.Setup(LogLevel.Information, $"Perimeter LN- und Sömmerungsflächen (SH): Thema wurde am {datestring_delta4.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet");
+        loggerMock.Setup(LogLevel.Information, $"Perimeter LN- und Sömmerungsflächen (ZG): Thema wurde am {datestring_delta23.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} aktualisiert und wird verarbeitet");
+        loggerMock.Setup(LogLevel.Information, $"Rebbaukataster (SH): Thema wurde seit {datestring_delta30.ToString("G", CultureInfo.GetCultureInfo("de-CH"))} nicht aktualisiert");
         loggerMock.Setup(LogLevel.Information, "Rebbaukataster (ZG): Thema ist nicht verfügbar");
         loggerMock.Setup(LogLevel.Information, "2 Themen werden prozessiert");
 
-        var topicsToProcess = await new Processor(geodiensteApiMock.Object, azureStorageMock.Object, loggerMock.Object).GetTopicsToProcess();
+        azureStorageMock.SetupSequence(storage => storage.GetLastProcessed(It.IsAny<Topic>()))
+            .ReturnsAsync(datestring_delta23)
+            .ReturnsAsync((DateTime?)null)
+            .ReturnsAsync(datestring_delta23);
+
+        var topicsToProcess = await processor.GetTopicsToProcess();
         Assert.AreEqual(2, topicsToProcess.Count);
         Assert.AreEqual(BaseTopic.lwb_perimeter_ln_sf, topicsToProcess[0].BaseTopic);
         Assert.AreEqual(Canton.SH, topicsToProcess[0].Canton);

diff --git a/Geodatenbezug.Test/Processors/TopicProcessorTest.cs b/Geodatenbezug.Test/Processors/TopicProcessorTest.cs
@@ -49,6 +49,7 @@ public async Task ExportTopic()
             Code = HttpStatusCode.Processing,
             TopicTitle = topic.TopicTitle,
             Canton = topic.Canton,
+            UpdatedAt = topic.UpdatedAt.Value,
         };
         geodiensteApiMock
             .Setup(api => api.StartExportAsync(It.IsAny<Topic>()))
@@ -73,6 +74,7 @@ public async Task ExportTopicOnlyOneExportIn24h()
             Code = HttpStatusCode.Processing,
             TopicTitle = topic.TopicTitle,
             Canton = topic.Canton,
+            UpdatedAt = topic.UpdatedAt.Value,
         };
         geodiensteApiMock
             .Setup(api => api.StartExportAsync(It.IsAny<Topic>()))
@@ -99,6 +101,7 @@ public async Task ExportTopicStartExportFails()
             Info = "Data export information not found. Invalid token?",
             TopicTitle = topic.TopicTitle,
             Canton = topic.Canton,
+            UpdatedAt = topic.UpdatedAt.Value,
         };
         geodiensteApiMock
             .Setup(api => api.StartExportAsync(It.IsAny<Topic>()))
@@ -123,6 +126,7 @@ public async Task ExportTopicCheckExportStatusFailed()
             Info = "An unexpected error occurred. Please try again by starting a new data export.",
             TopicTitle = topic.TopicTitle,
             Canton = topic.Canton,
+            UpdatedAt = topic.UpdatedAt.Value,
         };
         geodiensteApiMock
             .Setup(api => api.StartExportAsync(It.IsAny<Topic>()))
@@ -151,6 +155,7 @@ public async Task ExportTopicCheckExportStatusError()
             Info = "Data export information not found. Invalid token?",
             TopicTitle = topic.TopicTitle,
             Canton = topic.Canton,
+            UpdatedAt = topic.UpdatedAt.Value,
         };
         geodiensteApiMock
             .Setup(api => api.StartExportAsync(It.IsAny<Topic>()))
@@ -179,6 +184,7 @@ public async Task PrepareDataFails()
             Info = "Inner exception details",
             TopicTitle = topic.TopicTitle,
             Canton = topic.Canton,
+            UpdatedAt = topic.UpdatedAt.Value,
         };
 
         processorMock.Setup(p => p.PrepareDataAsync())

diff --git a/Geodatenbezug.Test/test.runsettings b/Geodatenbezug.Test/test.runsettings
@@ -5,6 +5,10 @@
         <tokens_lwb_rebbaukataster>AG=token1;BE=token2;NW=token3</tokens_lwb_rebbaukataster>
         <AuthUser>user123</AuthUser>
         <AuthPw>password123</AuthPw>
+        <SmtpHost>mailhog.geow.cloud</SmtpHost>
+        <SmtpFrom>notifications@geowerkstatt.ch</SmtpFrom>
+        <SmtpTo>geodaten@blw.admin.ch</SmtpTo>
+        <SmtpCc>support@geowerkstatt.ch</SmtpCc>
     </EnvironmentVariables>
   </RunConfiguration>
 </RunSettings>
diff --git a/Geodatenbezug/AzureStorage.cs b/Geodatenbezug/AzureStorage.cs
@@ -1,6 +1,8 @@
 using Azure.Storage;
 using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
 using Azure.Storage.Sas;
+using Geodatenbezug.Models;
 using Microsoft.Extensions.Logging;
 
 namespace Geodatenbezug;
@@ -13,18 +15,60 @@ public class AzureStorage(ILogger<AzureStorage> logger) : IAzureStorage
 #pragma warning restore SA1009 // Closing parenthesis should be spaced correctly
 {
     private const string StorageContainerName = "processed-topics";
+    private string connectionString = string.Empty;
+
+    /// <summary>
+    /// Connection string to the Azure Storage.
+    /// </summary>
+    protected string ConnectionString
+    {
+        get
+        {
+            if (string.IsNullOrEmpty(connectionString))
+            {
+                var value = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
+                if (string.IsNullOrEmpty(value))
+                {
+                    throw new InvalidOperationException("AzureWebJobsStorage is not set");
+                }
+
+                connectionString = value;
+            }
+
+            return connectionString;
+        }
+    }
 
     /// <inheritdoc />
-    public async Task<string> UploadFileAsync(string storageFilePath, string localFilePath)
+    public async Task<DateTime?> GetLastProcessed(Topic topic)
     {
-        logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch");
-        var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
-        if (string.IsNullOrEmpty(connectionString))
+        logger.LogInformation($"{topic.TopicTitle} ({topic.Canton}): Frage letzte Prozessierung des Themas ab");
+        var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName);
+        var creationDates = new List<DateTime>();
+
+        await foreach (BlobItem blobItem in containerClient.GetBlobsAsync())
         {
-            throw new InvalidOperationException("AzureWebJobsStorage is not set");
+            if (blobItem.Name.Contains(topic.Canton.ToString(), StringComparison.OrdinalIgnoreCase)
+                && blobItem.Name.Contains(topic.BaseTopic.ToString(), StringComparison.OrdinalIgnoreCase))
+            {
+                creationDates.Add(blobItem.Properties.CreatedOn.GetValueOrDefault().DateTime);
+            }
         }
 
-        var containerClient = new BlobServiceClient(connectionString).GetBlobContainerClient(StorageContainerName);
+        if (creationDates.Count > 0)
+        {
+            return creationDates.Max();
+        }
+
+        return null;
+    }
+
+    /// <inheritdoc />
+    public async Task<string> UploadFileAsync(string storageFilePath, string localFilePath)
+    {
+        logger.LogInformation($"Lade Datei {localFilePath} in den Azure Storage hoch");
+
+        var containerClient = new BlobServiceClient(ConnectionString).GetBlobContainerClient(StorageContainerName);
         var blobClient = containerClient.GetBlobClient(storageFilePath);
 
         using var localFileStream = File.OpenRead(localFilePath);
@@ -42,8 +86,8 @@ public async Task<string> UploadFileAsync(string storageFilePath, string localFi
         };
         sasBuilder.SetPermissions(BlobSasPermissions.Read);
 
-        var accountName = connectionString.ExtractValueByKey("AccountName");
-        var accountKey = connectionString.ExtractValueByKey("AccountKey");
+        var accountName = ConnectionString.ExtractValueByKey("AccountName");
+        var accountKey = ConnectionString.ExtractValueByKey("AccountKey");
         string sasToken = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(accountName, accountKey)).ToString();
         return $"{blobClient.Uri}?{sasToken}";
     }