From f22fe8a0bde603a2ca7828bd03774f53fddcd458 Mon Sep 17 00:00:00 2001 From: Simon <63975668+Simyon264@users.noreply.github.com> Date: Fri, 8 Mar 2024 08:30:35 +0100 Subject: [PATCH 1/3] Remove loop Since the queue consume gets called after the queue gets filled, a loop checking the queue is not needed. --- Server/ReplayParser.cs | 142 ++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 74 deletions(-) diff --git a/Server/ReplayParser.cs b/Server/ReplayParser.cs index 06f5657..71ace56 100644 --- a/Server/ReplayParser.cs +++ b/Server/ReplayParser.cs @@ -51,92 +51,86 @@ public static Task AddParsedReplayToDb(string replay) public static async Task ConsumeQueue(CancellationToken token) { - while (!token.IsCancellationRequested) + // Consume the queue. + while (Queue.Count > 0) { - // Consume the queue. - while (Queue.Count > 0) - { - var timeoutToken = new CancellationTokenSource(10000); - var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutToken.Token); - var startTime = DateTime.Now; + var timeoutToken = new CancellationTokenSource(10000); + var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutToken.Token); + var startTime = DateTime.Now; - // Since replays are like 200mb long, we want to parrallelize this. - var tasks = new List(); - for (var i = 0; i < 10; i++) + // Since replays are like 200mb long, we want to parrallelize this. + var tasks = new List(); + for (var i = 0; i < 10; i++) + { + if (Queue.Count == 0) { - if (Queue.Count == 0) - { - break; - } - var replay = Queue[0]; - Queue.RemoveAt(0); - // If it's already in the database, skip it. - if (await IsReplayParsed(replay)) - { - continue; - } - - tasks.Add(Task.Run(async () => + break; + } + var replay = Queue[0]; + Queue.RemoveAt(0); + // If it's already in the database, skip it. + if (await IsReplayParsed(replay)) + { + continue; + } + tasks.Add(Task.Run(async () => + { + try { + var client = new HttpClient(); + Log.Information("Downloading " + replay); + var fileStream = await client.GetStreamAsync(replay, token); + Replay? parsedReplay = null; try { - var client = new HttpClient(); - Log.Information("Downloading " + replay); - var fileStream = await client.GetStreamAsync(replay, token); - Replay? parsedReplay = null; - try - { - parsedReplay = ParseReplay(fileStream); - } - catch (Exception e) - { - // Ignore - await AddParsedReplayToDb(replay); - return; - } - parsedReplay.Link = replay; - // See if the link matches the date regex, if it does set the date - var replayFileName = Path.GetFileName(replay); - var match = RegexList.ReplayRegex.Match(replayFileName); - if (match.Success) - { - var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy_MM_dd-HH_mm", CultureInfo.InvariantCulture); - // Need to mark it as UTC, since the server is in UTC. - parsedReplay.Date = date.ToUniversalTime(); - } - - // One more check to see if it's already in the database. - if (await IsReplayParsed(replay)) - { - return; - } - - await AddReplayToDb(parsedReplay); - await AddParsedReplayToDb(replay); - Log.Information("Parsed " + replay); + parsedReplay = ParseReplay(fileStream); } catch (Exception e) { - Log.Error(e, "Error while parsing " + replay); + // Ignore + await AddParsedReplayToDb(replay); + return; } - }, tokenSource.Token)); - } + parsedReplay.Link = replay; + // See if the link matches the date regex, if it does set the date + var replayFileName = Path.GetFileName(replay); + var match = RegexList.ReplayRegex.Match(replayFileName); + if (match.Success) + { + var date = DateTime.ParseExact(match.Groups[1].Value, "yyyy_MM_dd-HH_mm", CultureInfo.InvariantCulture); + // Need to mark it as UTC, since the server is in UTC. + parsedReplay.Date = date.ToUniversalTime(); + } + + // One more check to see if it's already in the database. + if (await IsReplayParsed(replay)) + { + return; + } + + await AddReplayToDb(parsedReplay); + await AddParsedReplayToDb(replay); + Log.Information("Parsed " + replay); + } + catch (Exception e) + { + Log.Error(e, "Error while parsing " + replay); + } + }, tokenSource.Token)); + } - // If the download takes too long, cancel it. - // 10 minutes should be enough - await Task.WhenAny(Task.WhenAll(tasks), Task.Delay(600000, token)); - await timeoutToken.CancelAsync(); - // Cancel the timeout token, so the background tasks cancel as well. + // If the download takes too long, cancel it. + // 10 minutes should be enough + await Task.WhenAny(Task.WhenAll(tasks), Task.Delay(600000, token)); + await timeoutToken.CancelAsync(); + // Cancel the timeout token, so the background tasks cancel as well. - // If we timed out, log a warning. - if (DateTime.Now - startTime > TimeSpan.FromMinutes(10)) - { - Log.Warning("Parsing took too long for " + string.Join(", ", tasks.Select(x => x.Id))); - } + // If we timed out, log a warning. + if (DateTime.Now - startTime > TimeSpan.FromMinutes(10)) + { + Log.Warning("Parsing took too long for " + string.Join(", ", tasks.Select(x => x.Id))); } - - await Task.Delay(5000, token); - } + } } /// @@ -328,4 +322,4 @@ public static List SearchReplays(SearchMode mode, string query, ReplayDb throw new NotImplementedException(); } } -} \ No newline at end of file +} From 2660889cfa3373369d88de5d050d679766f260dd Mon Sep 17 00:00:00 2001 From: Simon <63975668+Simyon264@users.noreply.github.com> Date: Fri, 8 Mar 2024 08:50:20 +0100 Subject: [PATCH 2/3] Add README.md --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..520cc1b --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +# Replay Browser + +This is the repository for a replay browser for Space Station 14. The replay browser downloads replays from the provided paths, parses them, and then finally inserts it into the provided DB. + +## Setup + +Setting up a dev env is simple. +These instructions assume that you have a postgres database set up. + +1. Clone the repository. +2. Set up the appsettings file. + Create a file named `appsettings.Secret.json` in the server project. + This is where you can put your connection string for the postgres DB. +3. Run both the server and client using `dotnet`. The server will now download a lot of replays. This will take some time and it will use about 50 Mbps. You can keep using your computer during this time. + +## Screenshots +
+ View + + ![image](https://github.com/Simyon264/ReplayBrowser/assets/63975668/f46c954f-cab1-4b95-be62-ee4d79329305) + +![image](https://github.com/Simyon264/ReplayBrowser/assets/63975668/c1e7b857-d643-4ca2-a69d-c62f3bbc383e) + +![image](https://github.com/Simyon264/ReplayBrowser/assets/63975668/3efa2506-cc35-44f3-91d5-d99cdcba7a66) + +![image](https://github.com/Simyon264/ReplayBrowser/assets/63975668/9bf1753d-ca22-466b-89ab-9f4aba186666) + +![image](https://github.com/Simyon264/ReplayBrowser/assets/63975668/c4b2212c-9644-448e-9458-551d6e5b6edc) +
From b8381e2960beba2fa2d586b5901eb89ac941fbd2 Mon Sep 17 00:00:00 2001 From: Simon <63975668+Simyon264@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:27:39 +0100 Subject: [PATCH 3/3] Add website link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 520cc1b..fdcb653 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Replay Browser This is the repository for a replay browser for Space Station 14. The replay browser downloads replays from the provided paths, parses them, and then finally inserts it into the provided DB. +You can view the deployed website [here](https://replay.unstablefoundation.de/) ## Setup