diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0ddecdb --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +################################################################################ +# This .gitignore file was automatically created by Microsoft(R) Visual Studio. +################################################################################ + +/CroppedImages diff --git a/SnowProfileScanner/Controllers/UploadController.cs b/SnowProfileScanner/Controllers/UploadController.cs index e775b05..4901931 100644 --- a/SnowProfileScanner/Controllers/UploadController.cs +++ b/SnowProfileScanner/Controllers/UploadController.cs @@ -9,12 +9,21 @@ using SnowProfileScanner.Services.Caaml; using System.Text; using System.Net; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.Processing; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Formats; +using System.IO; +using System.Runtime.InteropServices; +using SixLabors.ImageSharp.Advanced; +using Microsoft.AspNetCore.WebUtilities; public class UploadController : Controller { private readonly IConfiguration _configuration; private readonly SnowProfileService _snowProfileService; + private readonly SymbolRecognitionService _symbolRecognitionService; private readonly HttpClient _httpClient; private const string RECAPTCHA_URL = "https://www.google.com/recaptcha/api/siteverify"; private static readonly HashSet VALID_LWC = ValidLwc(); @@ -26,11 +35,13 @@ public class UploadController : Controller public UploadController( IConfiguration configuration, SnowProfileService snowProfileService, - HttpClient httpClient + HttpClient httpClient, + SymbolRecognitionService symbolService ) { _httpClient = httpClient; _configuration = configuration; _snowProfileService = snowProfileService; + _symbolRecognitionService = symbolService; } public IActionResult Index() @@ -67,10 +78,13 @@ public async Task Upload( AnalyzeResult result = operation.Value; var numberOfTables = result.Tables.Count(); + + Image image = Image.Load(memoryStream.ToArray()); + image.Mutate(x => x.AutoOrient()); var snowProfile = new SnowProfile { SnowTemp = numberOfTables > 1 ? DecodeSnowTemperature(result.Tables[1]) : new(), - Layers = numberOfTables > 0 ? DecodeSnowProfile(result.Tables[0]) : new(), + Layers = numberOfTables > 0 ? await DecodeSnowProfile(image, result.Tables[0]) : new(), AirTemp = numberOfTables > 1 ? DecodeAirTemperature(result.Tables[1]) : null, }; @@ -81,10 +95,29 @@ public async Task Upload( return View("Result", snowProfileEntity); } - - - private List DecodeSnowTemperature(DocumentTable tbl1) + + private Image CropImageToBoundingBox(Image image, IReadOnlyList boundingBox) + { + image.Mutate(x => x.AutoOrient()); + var rectangle = ConvertBoundingBoxToRectangle(boundingBox); + return image.Clone(x => x.Crop(rectangle)); + } + + + private Rectangle ConvertBoundingBoxToRectangle(IReadOnlyList boundingBox) + { + + var topLeft = new Point((int)boundingBox.Select(p => p.X).Min(), (int)boundingBox.Select(p => p.Y).Min()); + var bottomRight = new Point((int)boundingBox.Select(p => p.X).Max(), (int)boundingBox.Select(p => p.Y).Max()); + var width = bottomRight.X - topLeft.X; + var height = bottomRight.Y - topLeft.Y; + return new Rectangle(topLeft.X, topLeft.Y, width, height); + } + + + + private List DecodeSnowTemperature(DocumentTable tbl1) { var rows = tbl1.Cells .Where(cell => cell.RowIndex > 1 && !string.IsNullOrWhiteSpace(cell.Content)) @@ -117,7 +150,7 @@ public async Task Upload( return temps; } - private static List DecodeSnowProfile(DocumentTable tbl1) + private async Task> DecodeSnowProfile(Image image, DocumentTable tbl1) { var snowProfiles = new List(); var rowsCount = tbl1.Cells @@ -157,7 +190,8 @@ public async Task Upload( .OrderBy(vh => -vh.Length) .FirstOrDefault(); - var grainTypeText = ToString(row.SingleOrDefault(cell => cell.ColumnIndex == 3)) + var grainCell = row.SingleOrDefault(cell => cell.ColumnIndex == 3); + var grainTypeText = ToString(grainCell) .Replace("t", "f") .Replace("I", "l") .Replace("1", "l") @@ -172,6 +206,16 @@ public async Task Upload( .Where(vg => vg.Length <= grainTypeText.Length && vg.ToUpper() == grainTypeText[..vg.Length]) .OrderBy(vg => -vg.Length) .FirstOrDefault(); + if (grainTypeText == null || grainTypeText == string.Empty) + { + var polygon = grainCell.BoundingRegions[0].BoundingPolygon; + var croppedImage = CropImageToBoundingBox(image, polygon); + var stream = new MemoryStream(); + croppedImage.SaveAsPng(stream); // Save the image to the stream in PNG format + stream.Position = 0; + grainTypeText = await _symbolRecognitionService.ClassifyImage(stream); + } + var grainType = grainTypeText?.GetPrimaryGrainForm(); if (grainType?.Count() == 4) { @@ -182,7 +226,7 @@ public async Task Upload( { grainTypeSec = grainTypeSec.Substring(0, 2) + grainTypeSec.Substring(2).ToLower(); } - + var grainSizeText = row.SingleOrDefault(cell => cell.ColumnIndex == 4)?.Content ?? ""; var grainSizeSplit = grainSizeText.Split(new Char[] { '-', '–', '—', '_' }); @@ -192,6 +236,7 @@ public async Task Upload( { grainSizeMax = ToDouble(grainSizeSplit.Last()); } + var snowProfile = new SnowProfile.Layer { diff --git a/SnowProfileScanner/Models/ICustomVistionDecodeResult.cs b/SnowProfileScanner/Models/ICustomVistionDecodeResult.cs new file mode 100644 index 0000000..e5b589d --- /dev/null +++ b/SnowProfileScanner/Models/ICustomVistionDecodeResult.cs @@ -0,0 +1,22 @@ +namespace SnowProfileScanner.Models +{ + /** + * Resultat fra klassifisering av et symbol + */ + public interface ICustomVistionDecodeResult + { + string id { get; set; } + string project { get; set; } + string iteration { get; set; } + string created { get; set; } + IPrediction[] predictions { get; set; } + } + + public interface IPrediction + { + double Probability { get; set; } + string TagId { get; set; } + string TagName { get; set; } + } + +} diff --git a/SnowProfileScanner/Pages/Shared/Result.cshtml b/SnowProfileScanner/Pages/Shared/Result.cshtml index 60efb20..6dd4e5a 100644 --- a/SnowProfileScanner/Pages/Shared/Result.cshtml +++ b/SnowProfileScanner/Pages/Shared/Result.cshtml @@ -70,7 +70,7 @@ } else { -

Ingen snø temperaturer funnet.

+

Fant ingen temperaturprofil

} Bilde tatt av brukeren diff --git a/SnowProfileScanner/Program.cs b/SnowProfileScanner/Program.cs index cd081ec..5c4eca2 100644 --- a/SnowProfileScanner/Program.cs +++ b/SnowProfileScanner/Program.cs @@ -10,6 +10,7 @@ builder.Services.AddRazorPages(); builder.Services.AddControllersWithViews(); builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddHttpClient(); var app = builder.Build(); diff --git a/SnowProfileScanner/Services/SymbolRecognitionService.cs b/SnowProfileScanner/Services/SymbolRecognitionService.cs new file mode 100644 index 0000000..c870ada --- /dev/null +++ b/SnowProfileScanner/Services/SymbolRecognitionService.cs @@ -0,0 +1,47 @@ +using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction; +using Newtonsoft.Json; +using SnowProfileScanner.Models; +using System.Net.Http.Headers; + +namespace SnowProfileScanner.Services +{ + public class SymbolRecognitionService + { + private string url, predictionKey, projectId; + private ILogger logger; + private CustomVisionPredictionClient predictionClient; + + public SymbolRecognitionService(IConfiguration configuration, ILogger logger) + { + url = configuration["CustomVisionUrl"]; + predictionKey = configuration["CustomVisionPredictionKey"]; + projectId = configuration["CustomVisionProjectId"]; + + this.logger = logger; + predictionClient = AuthenticatePrediction(url, predictionKey); + } + + public async Task ClassifyImage(MemoryStream image) + { + var value = await predictionClient.ClassifyImageAsync(Guid.Parse(projectId), "Iteration5", image); + if (value.Predictions[0].Probability > 0.30) + { + return value.Predictions[0].TagName; + } + else + { + return null; + } + + } + private static CustomVisionPredictionClient AuthenticatePrediction(string endpoint, string predictionKey) + { + // Create a prediction endpoint, passing in the obtained prediction key + CustomVisionPredictionClient predictionApi = new CustomVisionPredictionClient(new Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction.ApiKeyServiceClientCredentials(predictionKey)) + { + Endpoint = endpoint + }; + return predictionApi; + } + } +} diff --git a/SnowProfileScanner/SnowProfileScanner.csproj b/SnowProfileScanner/SnowProfileScanner.csproj index 99ff26d..1dcf930 100644 --- a/SnowProfileScanner/SnowProfileScanner.csproj +++ b/SnowProfileScanner/SnowProfileScanner.csproj @@ -10,6 +10,8 @@ + +