only fetch CurseForge/Nexus exports when they change

Pathoschild · May 29, 2024 · b2c2a1c · b2c2a1c
1 parent 73f2c07
commit b2c2a1c
Show file tree

Hide file tree

Showing 11 changed files with 124 additions and 23 deletions.
diff --git a/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/CurseForgeExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/CurseForgeExportApiClient.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Pathoschild.Http.Client;
 using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels;
@@ -26,13 +27,21 @@ public CurseForgeExportApiClient(string userAgent, string baseUrl)
             this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent);
         }
 
+        /// <inheritdoc />
+        public async Task<DateTimeOffset> FetchLastModifiedDateAsync()
+        {
+            IResponse response = await this.Client.SendAsync(HttpMethod.Head, "");
+
+            return this.ReadLastModified(response);
+        }
+
         /// <inheritdoc />
         public async Task<CurseForgeFullExport> FetchExportAsync()
         {
             IResponse response = await this.Client.GetAsync("");
 
             CurseForgeFullExport export = await response.As<CurseForgeFullExport>();
-            export.LastModified = response.Message.Content.Headers.LastModified ?? throw new InvalidOperationException("Can't fetch from CurseForge export API: expected Last-Modified header wasn't set.");
+            export.LastModified = this.ReadLastModified(response);
 
             return export;
         }
@@ -42,5 +51,17 @@ public void Dispose()
         {
             this.Client.Dispose();
         }
+
+
+        /*********
+        ** Private methods
+        *********/
+        /// <summary>Read the <c>Last-Modified</c> header from an API response.</summary>
+        /// <param name="response">The response from the CurseForge API.</param>
+        /// <exception cref="InvalidOperationException">The response doesn't include the required <c>Last-Modified</c> header.</exception>
+        private DateTimeOffset ReadLastModified(IResponse response)
+        {
+            return response.Message.Content.Headers.LastModified ?? throw new InvalidOperationException("Can't fetch from CurseForge export API: expected Last-Modified header wasn't set.");
+        }
     }
 }
diff --git a/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ICurseForgeExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ICurseForgeExportApiClient.cs
@@ -7,7 +7,10 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport
     /// <summary>An HTTP client for fetching the mod export from the CurseForge export API.</summary>
     public interface ICurseForgeExportApiClient : IDisposable
     {
+        /// <summary>Fetch the date when the export on the server was last modified.</summary>
+        Task<DateTimeOffset> FetchLastModifiedDateAsync();
+
         /// <summary>Fetch the latest export file from the CurseForge export API.</summary>
-        public Task<CurseForgeFullExport> FetchExportAsync();
+        Task<CurseForgeFullExport> FetchExportAsync();
     }
 }
diff --git a/src/SMAPI.Toolkit/Framework/Clients/NexusExport/INexusExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/NexusExport/INexusExportApiClient.cs
@@ -7,7 +7,10 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.NexusExport
     /// <summary>An HTTP client for fetching the mod export from the Nexus Mods export API.</summary>
     public interface INexusExportApiClient : IDisposable
     {
+        /// <summary>Fetch the date when the export on the server was last modified.</summary>
+        Task<DateTimeOffset> FetchLastModifiedDateAsync();
+
         /// <summary>Fetch the latest export file from the Nexus Mods export API.</summary>
-        public Task<NexusFullExport> FetchExportAsync();
+        Task<NexusFullExport> FetchExportAsync();
     }
 }
diff --git a/src/SMAPI.Toolkit/Framework/Clients/NexusExport/NexusExportApiClient.cs b/src/SMAPI.Toolkit/Framework/Clients/NexusExport/NexusExportApiClient.cs
@@ -1,3 +1,5 @@
+using System;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Pathoschild.Http.Client;
 using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
@@ -25,18 +27,41 @@ public NexusExportApiClient(string userAgent, string baseUrl)
             this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent);
         }
 
+        /// <inheritdoc />
+        public async Task<DateTimeOffset> FetchLastModifiedDateAsync()
+        {
+            IResponse response = await this.Client.SendAsync(HttpMethod.Head, "");
+
+            return this.ReadLastModified(response);
+        }
+
         /// <inheritdoc />
         public async Task<NexusFullExport> FetchExportAsync()
         {
-            return await this.Client
-                .GetAsync("")
-                .As<NexusFullExport>();
+            IResponse response = await this.Client.GetAsync("");
+
+            NexusFullExport export = await response.As<NexusFullExport>();
+            export.LastUpdated = this.ReadLastModified(response);
+
+            return export;
         }
 
         /// <inheritdoc />
         public void Dispose()
         {
             this.Client.Dispose();
         }
+
+
+        /*********
+        ** Private methods
+        *********/
+        /// <summary>Read the <c>Last-Modified</c> header from an API response.</summary>
+        /// <param name="response">The response from the Nexus API.</param>
+        /// <exception cref="InvalidOperationException">The response doesn't include the required <c>Last-Modified</c> header.</exception>
+        private DateTimeOffset ReadLastModified(IResponse response)
+        {
+            return response.Message.Content.Headers.LastModified ?? throw new InvalidOperationException("Can't fetch from Nexus export API: expected Last-Modified header wasn't set.");
+        }
     }
 }
diff --git a/src/SMAPI.Web/BackgroundService.cs b/src/SMAPI.Web/BackgroundService.cs
@@ -167,12 +167,17 @@ public static async Task UpdateCurseForgeExportAsync()
             if (!BackgroundService.IsStarted)
                 throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
 
-            CurseForgeFullExport data = await BackgroundService.CurseForgeExportApiClient.FetchExportAsync();
-
             var cache = BackgroundService.CurseForgeExportCache;
-            cache.SetData(data);
+            var client = BackgroundService.CurseForgeExportApiClient;
+
+            if (await cache.CanRefreshFromAsync(client, BackgroundService.ExportStaleAge))
+            {
+                CurseForgeFullExport data = await client.FetchExportAsync();
+                cache.SetData(data);
+            }
+
             if (cache.IsStale(BackgroundService.ExportStaleAge))
-                cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead
+                cache.SetData(null); // if the export is too old, fetch fresh mod data from the API instead
         }
 
         /// <summary>Update the cached Nexus mod dump.</summary>
@@ -182,10 +187,15 @@ public static async Task UpdateNexusExportAsync()
             if (!BackgroundService.IsStarted)
                 throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
 
-            NexusFullExport data = await BackgroundService.NexusExportApiClient.FetchExportAsync();
-
             var cache = BackgroundService.NexusExportCache;
-            cache.SetData(data);
+            var client = BackgroundService.NexusExportApiClient;
+
+            if (await cache.CanRefreshFromAsync(client, BackgroundService.ExportStaleAge))
+            {
+                NexusFullExport data = await client.FetchExportAsync();
+                cache.SetData(data);
+            }
+
             if (cache.IsStale(BackgroundService.ExportStaleAge))
                 cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead
         }

diff --git a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/CurseForgeExportCacheMemoryRepository.cs
@@ -1,5 +1,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
+using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport;
 using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels;
 
 namespace StardewModdingAPI.Web.Framework.Caching.CurseForgeExport
@@ -18,15 +20,23 @@ internal class CurseForgeExportCacheMemoryRepository : BaseCacheRepository, ICur
         ** Public methods
         *********/
         /// <inheritdoc />
+        [MemberNotNullWhen(true, nameof(CurseForgeExportCacheMemoryRepository.Data))]
         public bool IsLoaded()
         {
             return this.Data?.Mods.Count > 0;
         }
 
         /// <inheritdoc />
-        public DateTimeOffset? GetLastRefreshed()
+        public async Task<bool> CanRefreshFromAsync(ICurseForgeExportApiClient client, int staleMinutes)
         {
-            return this.Data?.LastModified;
+            DateTimeOffset serverLastModified = await client.FetchLastModifiedDateAsync();
+
+            return
+                !this.IsStale(serverLastModified, staleMinutes)
+                && (
+                    !this.IsLoaded()
+                    || this.Data.LastModified < serverLastModified
+                );
         }
 
         /// <inheritdoc />

diff --git a/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/CurseForgeExport/ICurseForgeExportCacheRepository.cs
@@ -1,5 +1,6 @@
-using System;
 using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
+using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport;
 using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels;
 
 namespace StardewModdingAPI.Web.Framework.Caching.CurseForgeExport
@@ -13,8 +14,10 @@ internal interface ICurseForgeExportCacheRepository : ICacheRepository
         /// <summary>Get whether the export data is currently available.</summary>
         bool IsLoaded();
 
-        /// <summary>Get when the export data was last fetched, or <c>null</c> if no data is currently available.</summary>
-        DateTimeOffset? GetLastRefreshed();
+        /// <summary>Get whether newer non-stale data can be fetched from the server.</summary>
+        /// <param name="client">The CurseForge API client.</param>
+        /// <param name="staleMinutes">The age in minutes before data is considered stale.</param>
+        Task<bool> CanRefreshFromAsync(ICurseForgeExportApiClient client, int staleMinutes);
 
         /// <summary>Get the cached data for a mod, if it exists in the export.</summary>
         /// <param name="id">The CurseForge mod ID.</param>

diff --git a/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs
@@ -1,5 +1,6 @@
-using System;
 using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
+using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport;
 using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
 
 namespace StardewModdingAPI.Web.Framework.Caching.NexusExport
@@ -13,8 +14,10 @@ internal interface INexusExportCacheRepository : ICacheRepository
         /// <summary>Get whether the export data is currently available.</summary>
         bool IsLoaded();
 
-        /// <summary>Get when the export data was last fetched, or <c>null</c> if no data is currently available.</summary>
-        DateTimeOffset? GetLastRefreshed();
+        /// <summary>Get whether newer non-stale data can be fetched from the server.</summary>
+        /// <param name="client">The Nexus API client.</param>
+        /// <param name="staleMinutes">The age in minutes before data is considered stale.</param>
+        Task<bool> CanRefreshFromAsync(INexusExportApiClient client, int staleMinutes);
 
         /// <summary>Get the cached data for a mod, if it exists in the export.</summary>
         /// <param name="id">The Nexus mod ID.</param>

diff --git a/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs b/src/SMAPI.Web/Framework/Caching/NexusExport/NexusExportCacheMemoryRepository.cs
@@ -1,5 +1,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
+using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport;
 using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
 
 namespace StardewModdingAPI.Web.Framework.Caching.NexusExport
@@ -18,15 +20,23 @@ internal class NexusExportCacheMemoryRepository : BaseCacheRepository, INexusExp
         ** Public methods
         *********/
         /// <inheritdoc />
+        [MemberNotNullWhen(true, nameof(NexusExportCacheMemoryRepository.Data))]
         public bool IsLoaded()
         {
             return this.Data?.Data.Count > 0;
         }
 
         /// <inheritdoc />
-        public DateTimeOffset? GetLastRefreshed()
+        public async Task<bool> CanRefreshFromAsync(INexusExportApiClient client, int staleMinutes)
         {
-            return this.Data?.LastUpdated;
+            DateTimeOffset serverLastModified = await client.FetchLastModifiedDateAsync();
+
+            return
+                !this.IsStale(serverLastModified, staleMinutes)
+                && (
+                    !this.IsLoaded()
+                    || this.Data.LastUpdated < serverLastModified
+                );
         }
 
         /// <inheritdoc />

diff --git a/src/SMAPI.Web/Framework/Clients/CurseForge/DisabledCurseForgeExportApiClient.cs b/src/SMAPI.Web/Framework/Clients/CurseForge/DisabledCurseForgeExportApiClient.cs
@@ -1,3 +1,4 @@
+using System;
 using System.Threading.Tasks;
 using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport;
 using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels;
@@ -10,6 +11,12 @@ internal class DisabledCurseForgeExportApiClient : ICurseForgeExportApiClient
         /*********
         ** Public methods
         *********/
+        /// <inheritdoc />
+        public Task<DateTimeOffset> FetchLastModifiedDateAsync()
+        {
+            return Task.FromResult(DateTimeOffset.MinValue);
+        }
+
         /// <inheritdoc />
         public Task<CurseForgeFullExport> FetchExportAsync()
         {

diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/DisabledNexusExportApiClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/DisabledNexusExportApiClient.cs
@@ -11,6 +11,12 @@ internal class DisabledNexusExportApiClient : INexusExportApiClient
         /*********
         ** Public methods
         *********/
+        /// <inheritdoc />
+        public Task<DateTimeOffset> FetchLastModifiedDateAsync()
+        {
+            return Task.FromResult(DateTimeOffset.MinValue);
+        }
+
         /// <inheritdoc />
         public Task<NexusFullExport> FetchExportAsync()
         {