Merge pull request #1823 from tgstation/1822-Cron [APIDeploy][NugetDe…

…ploy]

Cron Scheduled Auto-Updates

build/TestCommon.props

@@ -8,19 +8,19 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <!-- Usage: Logging specific for GitHub actions -->
-    <PackageReference Include="GitHubActionsTestLogger" Version="2.3.3">
+    <PackageReference Include="GitHubActionsTestLogger" Version="2.4.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <!-- Usage: Hard to say what exactly this is for, but not including it removes the test icon and breaks vstest.console.exe for some reason -->
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
     <!-- Usage: Dependency mocking for tests -->
     <!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
     <PackageReference Include="Moq" Version="4.20.70" />
     <!-- Usage: MSTest execution -->
-    <PackageReference Include="MSTest.TestAdapter" Version="3.3.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.4.3" />
     <!-- Usage: MSTest asserts etc... -->
-    <PackageReference Include="MSTest.TestFramework" Version="3.3.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.4.3" />
   </ItemGroup>
 
 </Project>

build/Version.props

@@ -3,12 +3,12 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.5.0</TgsCoreVersion>
+    <TgsCoreVersion>6.6.0</TgsCoreVersion>
     <TgsConfigVersion>5.1.0</TgsConfigVersion>
-    <TgsApiVersion>10.3.0</TgsApiVersion>
+    <TgsApiVersion>10.4.0</TgsApiVersion>
     <TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
-    <TgsApiLibraryVersion>13.3.0</TgsApiLibraryVersion>
-    <TgsClientVersion>15.3.0</TgsClientVersion>
+    <TgsApiLibraryVersion>13.4.0</TgsApiLibraryVersion>
+    <TgsClientVersion>15.4.0</TgsClientVersion>
     <TgsDmapiVersion>7.1.2</TgsDmapiVersion>
     <TgsInteropVersion>5.9.0</TgsInteropVersion>
     <TgsHostWatchdogVersion>1.4.1</TgsHostWatchdogVersion>
@@ -17,8 +17,10 @@
     <TgsNugetNetFramework>netstandard2.0</TgsNugetNetFramework>
     <TgsNetMajorVersion>8</TgsNetMajorVersion>
     <!-- Update this frequently with dotnet runtime patches. MAJOR MUST MATCH ABOVE! -->
-    <TgsDotnetRedistUrl>https://download.visualstudio.microsoft.com/download/pr/00397fee-1bd9-44ef-899b-4504b26e6e96/ab9c73409659f3238d33faee304a8b7c/dotnet-hosting-8.0.4-win.exe</TgsDotnetRedistUrl>
-    <TgsMariaDBRedistVersion>10.11.6</TgsMariaDBRedistVersion>
+    <TgsDotnetRedistUrl>https://download.visualstudio.microsoft.com/download/pr/751d3fcd-72db-4da2-b8d0-709c19442225/33cc492bde704bfd6d70a2b9109005a0/dotnet-hosting-8.0.6-win.exe</TgsDotnetRedistUrl>
+    <TgsMariaDBRedistVersion>10.11.8</TgsMariaDBRedistVersion>
+    <!-- Only have this uncommented if the mariadb servers are shitting the bed, update if the version updates -->
+    <TgsMariaDBFallbackRedist>https://mirror.its.dal.ca/mariadb//mariadb-10.11.8/winx64-packages/mariadb-10.11.8-winx64.msi</TgsMariaDBFallbackRedist>
     <TgsYarnVersion>1.22.21</TgsYarnVersion>
   </PropertyGroup>
 </Project>

build/package/winget/prepare_installer_input_artifacts.ps1

@@ -35,7 +35,15 @@ try
     try
     {
         Invoke-WebRequest -Uri $redistUrl -OutFile artifacts/hosting-bundle.exe
-        Invoke-WebRequest -Uri $dbRedistUrl -OutFile artifacts/mariadb.msi
+        try
+        {
+            Invoke-WebRequest -Uri $dbRedistUrl -OutFile artifacts/mariadb.msi
+        }
+        catch
+        {
+            $dbRedistUrl = $versionXML.Project.PropertyGroup.TgsMariaDBFallbackRedist
+            Invoke-WebRequest -Uri $dbRedistUrl -OutFile artifacts/mariadb.msi
+        }
     } finally {
         $ProgressPreference = $previousProgressPreference
     }

src/Tgstation.Server.Api/Models/Instance.cs

@@ -31,9 +31,18 @@ public abstract class Instance : NamedEntity
 		/// <summary>
 		/// The time interval in minutes the repository is automatically pulled and compiles. 0 disables.
 		/// </summary>
+		/// <remarks>Auto-updates intervals start counting when set, TGS is started, or from the completion of the previous update. Incompatible with <see cref="AutoUpdateCron"/>.</remarks>
 		[Required]
 		public uint? AutoUpdateInterval { get; set; }
 
+		/// <summary>
+		/// A cron expression indicating when auto-updates should trigger. Must be a valid 6 part cron schedule (SECONDS MINUTES HOURS DAY/MONTH MONTH DAY/WEEK). Empty <see cref="string"/> disables.
+		/// </summary>
+		/// <remarks>Updates will not be triggered if the previous update is still running. Incompatible with <see cref="AutoUpdateInterval"/>.</remarks>
+		[Required]
+		[StringLength(Limits.MaximumStringLength)]
+		public string? AutoUpdateCron { get; set; }
+
 		/// <summary>
 		/// The maximum number of chat bots the <see cref="Instance"/> may contain.
 		/// </summary>

src/Tgstation.Server.Api/Tgstation.Server.Api.csproj

@@ -27,7 +27,7 @@
     <!-- Usage: HTTP constants reference -->
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.2.0" />
     <!-- Usage: Decoding the 'nbf' property of JWTs -->
-    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.5.1" />
+    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.6.2" />
     <!-- Usage: Primary JSON library -->
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <!-- Usage: Data model annotating -->

src/Tgstation.Server.Client/Tgstation.Server.Client.csproj

@@ -11,9 +11,9 @@
 
   <ItemGroup>
     <!-- Usage: Connecting to SignalR hubs in API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.6" />
     <!-- Usage: Using target JSON serializer for API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="8.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="8.0.6" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "dotnet-ef": {
-      "version": "8.0.4",
+      "version": "8.0.6",
       "commands": [
         "dotnet-ef"
       ]

src/Tgstation.Server.Host/Components/IInstanceCore.cs

@@ -45,10 +45,11 @@ public interface IInstanceCore : ILatestCompileJobProvider, IRenameNotifyee
 		IConfiguration Configuration { get; }
 
 		/// <summary>
-		/// Change the <see cref="Api.Models.Instance.AutoUpdateInterval"/> for the <see cref="IInstanceCore"/>.
+		/// Change the auto-update timing for the <see cref="IInstanceCore"/>.
 		/// </summary>
-		/// <param name="newInterval">The new auto update inteval.</param>
+		/// <param name="newInterval">The new auto-update inteval.</param>
+		/// <param name="newCron">The new auto-update cron schedule.</param>
 		/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
-		ValueTask SetAutoUpdateInterval(uint newInterval);
+		ValueTask ScheduleAutoUpdate(uint newInterval, string? newCron);
 	}
 }

src/Tgstation.Server.Host/Components/Instance.cs

@@ -1,11 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+
+using NCrontab;
+
 using Serilog.Context;
 
 using Tgstation.Server.Api.Rights;
@@ -183,7 +187,7 @@ public async Task StartAsync(CancellationToken cancellationToken)
 			using (LogContext.PushProperty(SerilogContextHelper.InstanceIdContextProperty, metadata.Id))
 			{
 				await Task.WhenAll(
-					SetAutoUpdateInterval(metadata.Require(x => x.AutoUpdateInterval)).AsTask(),
+					ScheduleAutoUpdate(metadata.Require(x => x.AutoUpdateInterval), metadata.AutoUpdateCron).AsTask(),
 					Configuration.StartAsync(cancellationToken),
 					EngineManager.StartAsync(cancellationToken),
 					Chat.StartAsync(cancellationToken),
@@ -202,7 +206,7 @@ public async Task StopAsync(CancellationToken cancellationToken)
 			using (LogContext.PushProperty(SerilogContextHelper.InstanceIdContextProperty, metadata.Id))
 			{
 				logger.LogDebug("Stopping instance...");
-				await SetAutoUpdateInterval(0);
+				await ScheduleAutoUpdate(0, null);
 				await Watchdog.StopAsync(cancellationToken);
 				await Task.WhenAll(
 					Configuration.StopAsync(cancellationToken),
@@ -213,11 +217,13 @@ await Task.WhenAll(
 		}
 
 		/// <inheritdoc />
-		public async ValueTask SetAutoUpdateInterval(uint newInterval)
+		public async ValueTask ScheduleAutoUpdate(uint newInterval, string? newCron)
 		{
+			if (newInterval > 0 && !String.IsNullOrWhiteSpace(newCron))
+				throw new ArgumentException("Only one of newInterval and newCron may be set!");
+
 			Task toWait;
 			lock (timerLock)
-			{
 				if (timerTask != null)
 				{
 					logger.LogTrace("Cancelling auto-update task");
@@ -229,12 +235,11 @@ public async ValueTask SetAutoUpdateInterval(uint newInterval)
 				}
 				else
 					toWait = Task.CompletedTask;
-			}
 
 			await toWait;
-			if (newInterval == 0)
+			if (newInterval == 0 && String.IsNullOrWhiteSpace(newCron))
 			{
-				logger.LogTrace("New auto-update interval is 0. Not starting task.");
+				logger.LogTrace("Auto-update disabled 0. Not starting task.");
 				return;
 			}
 
@@ -243,12 +248,12 @@ public async ValueTask SetAutoUpdateInterval(uint newInterval)
 				// race condition, just quit
 				if (timerTask != null)
 				{
-					logger.LogWarning("Aborting auto update interval change due to race condition!");
+					logger.LogWarning("Aborting auto-update scheduling change due to race condition!");
 					return;
 				}
 
 				timerCts = new CancellationTokenSource();
-				timerTask = TimerLoop(newInterval, timerCts.Token);
+				timerTask = TimerLoop(newInterval, newCron, timerCts.Token);
 			}
 		}
 
@@ -484,82 +489,128 @@ await repo.ResetToOrigin(
 		/// Pull the repository and compile for every set of given <paramref name="minutes"/>.
 		/// </summary>
 		/// <param name="minutes">How many minutes the operation should repeat. Does not include running time.</param>
+		/// <param name="cron">Alternative cron schedule.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
 #pragma warning disable CA1502 // TODO: Decomplexify
-		async Task TimerLoop(uint minutes, CancellationToken cancellationToken)
+		async Task TimerLoop(uint minutes, string? cron, CancellationToken cancellationToken)
 		{
 			logger.LogDebug("Entering auto-update loop");
 			while (true)
 				try
 				{
-					await asyncDelayer.Delay(TimeSpan.FromMinutes(minutes > Int32.MaxValue ? Int32.MaxValue : minutes), cancellationToken);
-					logger.LogInformation("Beginning auto update...");
-					await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, Enumerable.Empty<string>(), true, cancellationToken);
-					try
+					TimeSpan delay;
+					if (!String.IsNullOrWhiteSpace(cron))
 					{
-						var repositoryUpdateJob = Job.Create(Api.Models.JobCode.RepositoryAutoUpdate, null, metadata, RepositoryRights.CancelPendingChanges);
-						await jobManager.RegisterOperation(
-							repositoryUpdateJob,
-							RepositoryAutoUpdateJob,
-							cancellationToken);
+						logger.LogTrace("Using cron schedule: {cron}", cron);
+						var schedule = CrontabSchedule.Parse(
+							cron,
+							new CrontabSchedule.ParseOptions
+							{
+								IncludingSeconds = true,
+							});
+						var now = DateTime.UtcNow;
+						var nextOccurrence = schedule.GetNextOccurrence(now);
+						delay = nextOccurrence - now;
+					}
+					else
+					{
+						logger.LogTrace("Using interval: {interval}m", minutes);
 
-						var repoUpdateJobResult = await jobManager.WaitForJobCompletion(repositoryUpdateJob, null, cancellationToken, cancellationToken);
-						if (repoUpdateJobResult == false)
-						{
-							logger.LogWarning("Aborting auto-update due to repository update error!");
-							continue;
-						}
+						delay = TimeSpan.FromMinutes(minutes);
+					}
 
-						Job compileProcessJob;
-						using (var repo = await RepositoryManager.LoadRepository(cancellationToken))
-						{
-							if (repo == null)
-								throw new JobException(Api.Models.ErrorCode.RepoMissing);
+					logger.LogInformation("Next auto-update will occur at {time}", DateTimeOffset.UtcNow + delay);
 
-							var deploySha = repo.Head;
-							if (deploySha == null)
-							{
-								logger.LogTrace("Aborting auto update, repository error!");
-								continue;
-							}
+					// https://learn.microsoft.com/en-us/dotnet/api/system.threading.tasks.task.delay?view=net-8.0#system-threading-tasks-task-delay(system-timespan)
+					const uint DelayMinutesLimit = UInt32.MaxValue - 1;
+					Debug.Assert(DelayMinutesLimit == 4294967294, "Delay limit assertion failure!");
 
-							if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
-							{
-								logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
-								continue;
-							}
+					var maxDelayIterations = 0UL;
+					if (delay.TotalMilliseconds >= UInt32.MaxValue)
+					{
+						maxDelayIterations = (ulong)Math.Floor(delay.TotalMilliseconds / DelayMinutesLimit);
+						logger.LogDebug("Breaking interval into {iterationCount} iterations", maxDelayIterations + 1);
+						delay = TimeSpan.FromMilliseconds(delay.TotalMilliseconds - (maxDelayIterations * DelayMinutesLimit));
+					}
 
-							// finally set up the job
-							compileProcessJob = Job.Create(Api.Models.JobCode.AutomaticDeployment, null, metadata, DreamMakerRights.CancelCompile);
-							await jobManager.RegisterOperation(
-								compileProcessJob,
-								(core, databaseContextFactory, job, progressReporter, jobCancellationToken) =>
-								{
-									if (core != this)
-										throw new InvalidOperationException(DifferentCoreExceptionMessage);
-									return DreamMaker.DeploymentProcess(
-										job,
-										databaseContextFactory,
-										progressReporter,
-										jobCancellationToken);
-								},
-								cancellationToken);
+					if (maxDelayIterations > 0)
+					{
+						var longDelayTimeSpan = TimeSpan.FromMilliseconds(DelayMinutesLimit);
+						for (var i = 0UL; i < maxDelayIterations; ++i)
+						{
+							logger.LogTrace("Long delay #{iteration}...", i + 1);
+							await asyncDelayer.Delay(longDelayTimeSpan, cancellationToken);
 						}
 
-						await jobManager.WaitForJobCompletion(compileProcessJob, null, default, cancellationToken);
+						logger.LogTrace("Final delay iteration #{iteration}...", maxDelayIterations + 1);
 					}
-					catch (Exception e) when (e is not OperationCanceledException)
+
+					await asyncDelayer.Delay(delay, cancellationToken);
+					logger.LogInformation("Beginning auto update...");
+					await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, Enumerable.Empty<string>(), true, cancellationToken);
+
+					var repositoryUpdateJob = Job.Create(Api.Models.JobCode.RepositoryAutoUpdate, null, metadata, RepositoryRights.CancelPendingChanges);
+					await jobManager.RegisterOperation(
+						repositoryUpdateJob,
+						RepositoryAutoUpdateJob,
+						cancellationToken);
+
+					var repoUpdateJobResult = await jobManager.WaitForJobCompletion(repositoryUpdateJob, null, cancellationToken, cancellationToken);
+					if (repoUpdateJobResult == false)
 					{
-						logger.LogWarning(e, "Error in auto update loop!");
+						logger.LogWarning("Aborting auto-update due to repository update error!");
 						continue;
 					}
+
+					Job compileProcessJob;
+					using (var repo = await RepositoryManager.LoadRepository(cancellationToken))
+					{
+						if (repo == null)
+							throw new JobException(Api.Models.ErrorCode.RepoMissing);
+
+						var deploySha = repo.Head;
+						if (deploySha == null)
+						{
+							logger.LogTrace("Aborting auto update, repository error!");
+							continue;
+						}
+
+						if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
+						{
+							logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
+							continue;
+						}
+
+						// finally set up the job
+						compileProcessJob = Job.Create(Api.Models.JobCode.AutomaticDeployment, null, metadata, DreamMakerRights.CancelCompile);
+						await jobManager.RegisterOperation(
+							compileProcessJob,
+							(core, databaseContextFactory, job, progressReporter, jobCancellationToken) =>
+							{
+								if (core != this)
+									throw new InvalidOperationException(DifferentCoreExceptionMessage);
+								return DreamMaker.DeploymentProcess(
+									job,
+									databaseContextFactory,
+									progressReporter,
+									jobCancellationToken);
+							},
+							cancellationToken);
+					}
+
+					await jobManager.WaitForJobCompletion(compileProcessJob, null, default, cancellationToken);
 				}
 				catch (OperationCanceledException)
 				{
 					logger.LogDebug("Cancelled auto update loop!");
 					break;
 				}
+				catch (Exception e)
+				{
+					logger.LogError(e, "Error in auto update loop!");
+					continue;
+				}
 
 			logger.LogTrace("Leaving auto update loop...");
 		}