From b2c2a1c792d6c24a85ec77470c21e9c9a0725403 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 29 May 2024 01:10:06 -0400 Subject: [PATCH] only fetch CurseForge/Nexus exports when they change --- .../CurseForgeExportApiClient.cs | 23 +++++++++++++- .../ICurseForgeExportApiClient.cs | 5 ++- .../NexusExport/INexusExportApiClient.cs | 5 ++- .../NexusExport/NexusExportApiClient.cs | 31 +++++++++++++++++-- src/SMAPI.Web/BackgroundService.cs | 24 +++++++++----- .../CurseForgeExportCacheMemoryRepository.cs | 14 +++++++-- .../ICurseForgeExportCacheRepository.cs | 9 ++++-- .../INexusExportCacheRepository.cs | 9 ++++-- .../NexusExportCacheMemoryRepository.cs | 14 +++++++-- .../DisabledCurseForgeExportApiClient.cs | 7 +++++ .../Nexus/DisabledNexusExportApiClient.cs | 6 ++++ 11 files changed, 124 insertions(+), 23 deletions(-) diff --git a/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/CurseForgeExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/CurseForgeExportApiClient.cs index a564d9989..7cac5a6e8 100644 --- a/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/CurseForgeExportApiClient.cs +++ b/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/CurseForgeExportApiClient.cs @@ -1,4 +1,5 @@ using System; +using System.Net.Http; using System.Threading.Tasks; using Pathoschild.Http.Client; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels; @@ -26,13 +27,21 @@ public CurseForgeExportApiClient(string userAgent, string baseUrl) this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); } + /// + public async Task FetchLastModifiedDateAsync() + { + IResponse response = await this.Client.SendAsync(HttpMethod.Head, ""); + + return this.ReadLastModified(response); + } + /// public async Task FetchExportAsync() { IResponse response = await this.Client.GetAsync(""); CurseForgeFullExport export = await response.As(); - export.LastModified = response.Message.Content.Headers.LastModified ?? throw new InvalidOperationException("Can't fetch from CurseForge export API: expected Last-Modified header wasn't set."); + export.LastModified = this.ReadLastModified(response); return export; } @@ -42,5 +51,17 @@ public void Dispose() { this.Client.Dispose(); } + + + /********* + ** Private methods + *********/ + /// Read the Last-Modified header from an API response. + /// The response from the CurseForge API. + /// The response doesn't include the required Last-Modified header. + private DateTimeOffset ReadLastModified(IResponse response) + { + return response.Message.Content.Headers.LastModified ?? throw new InvalidOperationException("Can't fetch from CurseForge export API: expected Last-Modified header wasn't set."); + } } } diff --git a/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ICurseForgeExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ICurseForgeExportApiClient.cs index dd57a0a23..10aa083d7 100644 --- a/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ICurseForgeExportApiClient.cs +++ b/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ICurseForgeExportApiClient.cs @@ -7,7 +7,10 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport /// An HTTP client for fetching the mod export from the CurseForge export API. public interface ICurseForgeExportApiClient : IDisposable { + /// Fetch the date when the export on the server was last modified. + Task FetchLastModifiedDateAsync(); + /// Fetch the latest export file from the CurseForge export API. - public Task FetchExportAsync(); + Task FetchExportAsync(); } } diff --git a/src/SMAPI.Toolkit/Framework/Clients/NexusExport/INexusExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/NexusExport/INexusExportApiClient.cs index 0e1007b6f..875a59ff2 100644 --- a/src/SMAPI.Toolkit/Framework/Clients/NexusExport/INexusExportApiClient.cs +++ b/src/SMAPI.Toolkit/Framework/Clients/NexusExport/INexusExportApiClient.cs @@ -7,7 +7,10 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.NexusExport /// An HTTP client for fetching the mod export from the Nexus Mods export API. public interface INexusExportApiClient : IDisposable { + /// Fetch the date when the export on the server was last modified. + Task FetchLastModifiedDateAsync(); + /// Fetch the latest export file from the Nexus Mods export API. - public Task FetchExportAsync(); + Task FetchExportAsync(); } } diff --git a/src/SMAPI.Toolkit/Framework/Clients/NexusExport/NexusExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/NexusExport/NexusExportApiClient.cs index e3d235ac5..26f5aa113 100644 --- a/src/SMAPI.Toolkit/Framework/Clients/NexusExport/NexusExportApiClient.cs +++ b/src/SMAPI.Toolkit/Framework/Clients/NexusExport/NexusExportApiClient.cs @@ -1,3 +1,5 @@ +using System; +using System.Net.Http; using System.Threading.Tasks; using Pathoschild.Http.Client; using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels; @@ -25,12 +27,23 @@ public NexusExportApiClient(string userAgent, string baseUrl) this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); } + /// + public async Task FetchLastModifiedDateAsync() + { + IResponse response = await this.Client.SendAsync(HttpMethod.Head, ""); + + return this.ReadLastModified(response); + } + /// public async Task FetchExportAsync() { - return await this.Client - .GetAsync("") - .As(); + IResponse response = await this.Client.GetAsync(""); + + NexusFullExport export = await response.As(); + export.LastUpdated = this.ReadLastModified(response); + + return export; } /// @@ -38,5 +51,17 @@ public void Dispose() { this.Client.Dispose(); } + + + /********* + ** Private methods + *********/ + /// Read the Last-Modified header from an API response. + /// The response from the Nexus API. + /// The response doesn't include the required Last-Modified header. + private DateTimeOffset ReadLastModified(IResponse response) + { + return response.Message.Content.Headers.LastModified ?? throw new InvalidOperationException("Can't fetch from Nexus export API: expected Last-Modified header wasn't set."); + } } } diff --git a/src/SMAPI.Web/BackgroundService.cs b/src/SMAPI.Web/BackgroundService.cs index c3cb7160f..09854fde6 100644 --- a/src/SMAPI.Web/BackgroundService.cs +++ b/src/SMAPI.Web/BackgroundService.cs @@ -167,12 +167,17 @@ public static async Task UpdateCurseForgeExportAsync() if (!BackgroundService.IsStarted) throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks."); - CurseForgeFullExport data = await BackgroundService.CurseForgeExportApiClient.FetchExportAsync(); - var cache = BackgroundService.CurseForgeExportCache; - cache.SetData(data); + var client = BackgroundService.CurseForgeExportApiClient; + + if (await cache.CanRefreshFromAsync(client, BackgroundService.ExportStaleAge)) + { + CurseForgeFullExport data = await client.FetchExportAsync(); + cache.SetData(data); + } + if (cache.IsStale(BackgroundService.ExportStaleAge)) - cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead + cache.SetData(null); // if the export is too old, fetch fresh mod data from the API instead } /// Update the cached Nexus mod dump. @@ -182,10 +187,15 @@ public static async Task UpdateNexusExportAsync() if (!BackgroundService.IsStarted) throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks."); - NexusFullExport data = await BackgroundService.NexusExportApiClient.FetchExportAsync(); - var cache = BackgroundService.NexusExportCache; - cache.SetData(data); + var client = BackgroundService.NexusExportApiClient; + + if (await cache.CanRefreshFromAsync(client, BackgroundService.ExportStaleAge)) + { + NexusFullExport data = await client.FetchExportAsync(); + cache.SetData(data); + } + if (cache.IsStale(BackgroundService.ExportStaleAge)) cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead } diff --git a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs index ab9b3407e..6fe80425a 100644 --- a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs @@ -1,5 +1,7 @@ using System; using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.CurseForgeExport @@ -18,15 +20,23 @@ internal class CurseForgeExportCacheMemoryRepository : BaseCacheRepository, ICur ** Public methods *********/ /// + [MemberNotNullWhen(true, nameof(CurseForgeExportCacheMemoryRepository.Data))] public bool IsLoaded() { return this.Data?.Mods.Count > 0; } /// - public DateTimeOffset? GetLastRefreshed() + public async Task CanRefreshFromAsync(ICurseForgeExportApiClient client, int staleMinutes) { - return this.Data?.LastModified; + DateTimeOffset serverLastModified = await client.FetchLastModifiedDateAsync(); + + return + !this.IsStale(serverLastModified, staleMinutes) + && ( + !this.IsLoaded() + || this.Data.LastModified < serverLastModified + ); } /// diff --git a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs index f74bdee63..5027217ec 100644 --- a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs @@ -1,5 +1,6 @@ -using System; using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.CurseForgeExport @@ -13,8 +14,10 @@ internal interface ICurseForgeExportCacheRepository : ICacheRepository /// Get whether the export data is currently available. bool IsLoaded(); - /// Get when the export data was last fetched, or null if no data is currently available. - DateTimeOffset? GetLastRefreshed(); + /// Get whether newer non-stale data can be fetched from the server. + /// The CurseForge API client. + /// The age in minutes before data is considered stale. + Task CanRefreshFromAsync(ICurseForgeExportApiClient client, int staleMinutes); /// Get the cached data for a mod, if it exists in the export. /// The CurseForge mod ID. diff --git a/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs index 2c813f465..5b12dad66 100644 --- a/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs @@ -1,5 +1,6 @@ -using System; using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport; using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.NexusExport @@ -13,8 +14,10 @@ internal interface INexusExportCacheRepository : ICacheRepository /// Get whether the export data is currently available. bool IsLoaded(); - /// Get when the export data was last fetched, or null if no data is currently available. - DateTimeOffset? GetLastRefreshed(); + /// Get whether newer non-stale data can be fetched from the server. + /// The Nexus API client. + /// The age in minutes before data is considered stale. + Task CanRefreshFromAsync(INexusExportApiClient client, int staleMinutes); /// Get the cached data for a mod, if it exists in the export. /// The Nexus mod ID. diff --git a/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs b/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs index 52f64725a..d80f48204 100644 --- a/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs @@ -1,5 +1,7 @@ using System; using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport; using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels; namespace StardewModdingAPI.Web.Framework.Caching.NexusExport @@ -18,15 +20,23 @@ internal class NexusExportCacheMemoryRepository : BaseCacheRepository, INexusExp ** Public methods *********/ /// + [MemberNotNullWhen(true, nameof(NexusExportCacheMemoryRepository.Data))] public bool IsLoaded() { return this.Data?.Data.Count > 0; } /// - public DateTimeOffset? GetLastRefreshed() + public async Task CanRefreshFromAsync(INexusExportApiClient client, int staleMinutes) { - return this.Data?.LastUpdated; + DateTimeOffset serverLastModified = await client.FetchLastModifiedDateAsync(); + + return + !this.IsStale(serverLastModified, staleMinutes) + && ( + !this.IsLoaded() + || this.Data.LastUpdated < serverLastModified + ); } /// diff --git a/src/SMAPI.Web/Framework/Clients/CurseForge/DisabledCurseForgeExportApiClient.cs b/src/SMAPI.Web/Framework/Clients/CurseForge/DisabledCurseForgeExportApiClient.cs index 80cab5808..8d9e92f00 100644 --- a/src/SMAPI.Web/Framework/Clients/CurseForge/DisabledCurseForgeExportApiClient.cs +++ b/src/SMAPI.Web/Framework/Clients/CurseForge/DisabledCurseForgeExportApiClient.cs @@ -1,3 +1,4 @@ +using System; using System.Threading.Tasks; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport; using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels; @@ -10,6 +11,12 @@ internal class DisabledCurseForgeExportApiClient : ICurseForgeExportApiClient /********* ** Public methods *********/ + /// + public Task FetchLastModifiedDateAsync() + { + return Task.FromResult(DateTimeOffset.MinValue); + } + /// public Task FetchExportAsync() { diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/DisabledNexusExportApiClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/DisabledNexusExportApiClient.cs index 71f12c0cb..60fa4dd54 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/DisabledNexusExportApiClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/DisabledNexusExportApiClient.cs @@ -11,6 +11,12 @@ internal class DisabledNexusExportApiClient : INexusExportApiClient /********* ** Public methods *********/ + /// + public Task FetchLastModifiedDateAsync() + { + return Task.FromResult(DateTimeOffset.MinValue); + } + /// public Task FetchExportAsync() {