Add detail string to download page

Author: Simyon264

ReplayBrowser/Controllers/DataController.cs

@@ -40,13 +40,16 @@ public DownloadProgress GetDownloadProgress()
         return new DownloadProgress()
         {
             Progress = ReplayParserService.DownloadProgress.ToDictionary(x => x.Key, x => x.Value),
-            Status = ReplayParserService.Status.ToFriendlyString()
+            Status = ReplayParserService.Status.ToFriendlyString(),
+            Details = ReplayParserService.Details
         };
     }
 }
 
 public class DownloadProgress
 {
-    public string Status { get; set; }
-    public Dictionary<string, double> Progress { get; set; }
+    public required string Status { get; set; }
+    public required Dictionary<string, double> Progress { get; set; }
+    
+    public required string Details { get; set; }
 }

ReplayBrowser/Pages/Downloads.razor

@@ -5,7 +5,7 @@
 
 <h3>Downloads</h3>
 <p>Here you can see the progress of the internal downloads of the website.<br/>The website looks for new replays every 10th minute. So 12:00, 12:10, 12:20, 12:30, etc.</p>
-<p>Status: <b id="status-text">Waiting...</b></p>
+<p>Status: <b id="status-text">Waiting...</b> <span id="details"></span></p>
 
 <table class="table table-striped">
     <thead>
@@ -20,20 +20,26 @@
 </table>
 
 <script>
-    var Downloads = [];
-    var Status = "";
+    let Downloads = [];
+    let Status = "";
+    let Details = "";
     
-    // Every second, fetch active downloads from the server using JQuery
     setInterval(() => {
+        download();
+    }, 500);
+    
+    function download() {
         $.get('/api/Data/download-progress', (data) => {
             Downloads = data.progress;
             Status = data.status;
+            Details = data.details;
             updateDownloads();
         });
-    }, 1000);
+    }
+    
 
     function updateDownloads() {
-        var downloads = document.getElementById('downloads');
+        let downloads = document.getElementById('downloads');
         downloads.innerHTML = '';
         if (Object.keys(Downloads).length === 0) {
             document.getElementById('download-text-no').style.display = 'block';
@@ -42,20 +48,21 @@
         }
 
         document.getElementById('status-text').innerText = Status;
+        document.getElementById('details').innerText = Details;
         
-        for (var key in Downloads) {
-            var row = document.createElement('tr');
-            var filename = document.createElement('td');
-            var progressCell = document.createElement('td');
+        for (let key in Downloads) {
+            let row = document.createElement('tr');
+            let filename = document.createElement('td');
+            let progressCell = document.createElement('td');
 
             filename.innerText = key;
 
-            var progressBar = document.createElement('progress');
+            let progressBar = document.createElement('progress');
             progressBar.value = Downloads[key];
             progressBar.max = 1;
             progressCell.appendChild(progressBar);
-            
-            var percentage = document.createElement('span');
+
+            let percentage = document.createElement('span');
             percentage.innerText = (Downloads[key] * 100).toFixed(2) + '%';
             progressCell.appendChild(percentage);
             
@@ -64,4 +71,6 @@
             downloads.appendChild(row);
         }
     }
+
+    download();
 </script>

ReplayBrowser/Services/ReplayParser/Providers/ReplayProvider.cs

@@ -7,7 +7,6 @@ public abstract class ReplayProvider
     public HttpClient GetHttpClient()
     {
         var httpClient = new HttpClient();
-        httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
         httpClient.DefaultRequestHeaders.Add("User-Agent", "ReplayBrowser");
         return httpClient;
     }

ReplayBrowser/Services/ReplayParser/ReplayParserService.cs

@@ -16,6 +16,7 @@ public class ReplayParserService : IHostedService, IDisposable
     public static List<string> Queue = new();
     public static ConcurrentDictionary<string, double> DownloadProgress = new();
     public static ParserStatus Status = ParserStatus.Off;
+    public static string Details = "";
 
     /// <summary>
     /// Since the Replay Meta file was added just yesterday, we want to cut off all replays that were uploaded before that.
@@ -68,6 +69,7 @@ private async Task FetchReplays(CancellationToken token, StorageUrl[] storageUrl
         while (!token.IsCancellationRequested)
         {
             Status = ParserStatus.Discovering;
+            Details = $"0/{storageUrls.Length}";
             foreach (var storageUrl in storageUrls)
             {
                 Log.Information("Fetching replays from " + storageUrl);
@@ -80,6 +82,8 @@ private async Task FetchReplays(CancellationToken token, StorageUrl[] storageUrl
                 {
                     Log.Error(e, "Error while fetching replays from " + storageUrl);
                 }
+                
+                Details = $"{Array.IndexOf(storageUrls, storageUrl) + 1}/{storageUrls.Length}";
             }
 
             var now = DateTime.Now;
@@ -88,20 +92,30 @@ private async Task FetchReplays(CancellationToken token, StorageUrl[] storageUrl
             await ConsumeQueue(token);
             Log.Information("Next run in " + delay.TotalMinutes + " minutes.");
             Status = ParserStatus.Idle;
+            Details = "";
             await Task.Delay(delay, token);
         }
     }
 
     private async Task ConsumeQueue(CancellationToken token)
     {
-        DownloadProgress.Clear();
+        if (Queue.Count > 0)
+        {
+            DownloadProgress.Clear();
+        }
+        
+        var total = Queue.Count;
+        var completed = 0;
+        
         // Consume the queue.
         while (Queue.Count > 0)
         {
             var timeoutToken = new CancellationTokenSource(10000);
             var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutToken.Token);
             var startTime = DateTime.Now;
             // Clear the download progress.
+            Details = $"{completed}/{total}";
+            
             DownloadProgress.Clear();
             Status = ParserStatus.Downloading;
             var tasks = new List<Task>();
@@ -131,6 +145,8 @@ private async Task ConsumeQueue(CancellationToken token)
                         client.DefaultRequestHeaders.Add("User-Agent", "ReplayBrowser");
                         Log.Information("Downloading " + replay);
                         var fileStream = await client.GetStreamAsync(replay, progress, token);
+                        completed++;
+                        Details = $"{completed}/{total}";
                         Replay? parsedReplay = null;
                         try
                         {