From f5213c83a92b40fed0c5c50c989ddc759dbe4652 Mon Sep 17 00:00:00 2001 From: Markus Keil Date: Wed, 17 Apr 2024 14:44:11 +0200 Subject: [PATCH] seperate create and delete queues --- RunnerTask.cs => CreateRunnerTask.cs | 10 ++++-- PoolManager.cs | 50 ++++++++++++++++------------ Program.cs | 8 ++--- RunnerAction.cs | 7 ---- 4 files changed, 39 insertions(+), 36 deletions(-) rename RunnerTask.cs => CreateRunnerTask.cs (65%) delete mode 100644 RunnerAction.cs diff --git a/RunnerTask.cs b/CreateRunnerTask.cs similarity index 65% rename from RunnerTask.cs rename to CreateRunnerTask.cs index e76ff39..f221051 100644 --- a/RunnerTask.cs +++ b/CreateRunnerTask.cs @@ -1,12 +1,16 @@ namespace GithubActionsOrchestrator; -public record RunnerTask +public record CreateRunnerTask { public string Arch { get; set; } public string Size { get; set; } public string RunnerToken { get; set; } public string OrgName { get; set; } public int RetryCount { get; set; } - public RunnerAction Action { get; set; } public long ServerId { get; set; } -} \ No newline at end of file +} +public record DeleteRunnerTask +{ + public int RetryCount { get; set; } + public long ServerId { get; set; } +} diff --git a/PoolManager.cs b/PoolManager.cs index ab348f1..a618ebb 100644 --- a/PoolManager.cs +++ b/PoolManager.cs @@ -14,13 +14,15 @@ public class PoolManager : BackgroundService private static readonly Gauge QueueSize = Metrics .CreateGauge("github_queue", "Number of queued runner tasks"); - public ConcurrentQueue Tasks { get; } + public ConcurrentQueue CreateTasks { get; } + public ConcurrentQueue DeleteTasks { get; } public PoolManager(CloudController cc, ILogger logger) { _cc = cc; _logger = logger; - Tasks = new ConcurrentQueue(); + CreateTasks = new ConcurrentQueue(); + DeleteTasks = new ConcurrentQueue(); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -108,9 +110,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) for (int i = 0; i < missingCt; i++) { // Queue VM creation - Tasks.Enqueue(new RunnerTask + CreateTasks.Enqueue(new CreateRunnerTask { - Action = RunnerAction.Create, Arch = arch, Size = pool.Size, RunnerToken = runnerToken, @@ -127,26 +128,33 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) while (!stoppingToken.IsCancellationRequested) { - QueueSize.Set(Tasks.Count); - if (Tasks.TryDequeue(out RunnerTask? task)) + QueueSize.Set(CreateTasks.Count + DeleteTasks.Count); + + if (DeleteTasks.TryDequeue(out DeleteRunnerTask? dtask)) { - _logger.LogInformation($"Current Queue length: {Tasks.Count}"); - if (task != null) + _logger.LogInformation($"Current Queue length: C:{CreateTasks.Count} D:{DeleteTasks.Count}"); + if (dtask != null) { - bool success = false; - switch (task.Action) + bool success = await DeleteRunner(dtask); + if (!success) { - case RunnerAction.Create: - success = await CreateRunner(task); - break; - case RunnerAction.Delete: - success = await DeleteRunner(task); - break; + // Deletion didn't succeed. Let's hold processing runners for a minute + _logger.LogWarning("Encountered a problem deleting runners. Will hold queue processing for 1 minute."); + await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); } + } + } + + if (CreateTasks.TryDequeue(out CreateRunnerTask? task)) + { + _logger.LogInformation($"Current Queue length: C:{CreateTasks.Count} D:{DeleteTasks.Count}"); + if (task != null) + { + bool success = await CreateRunner(task); if (!success) { // Creation didn't succeed. Let's hold of creating new runners for a minute - _logger.LogWarning("Encountered a problem creating runners. Will hold creation for 1 minute."); + _logger.LogWarning("Encountered a problem creating runners. Will hold queue processing for 1 minute."); await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); } } @@ -156,7 +164,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - private async Task DeleteRunner(RunnerTask rt) + private async Task DeleteRunner(DeleteRunnerTask rt) { try { @@ -168,12 +176,12 @@ private async Task DeleteRunner(RunnerTask rt) _logger.LogError( $"Unable to delete runner [{rt.ServerId} | Retry: {rt.RetryCount}]: {ex.Message}"); rt.RetryCount += 1; - Tasks.Enqueue(rt); + DeleteTasks.Enqueue(rt); return false; } } - private async Task CreateRunner(RunnerTask rt) + private async Task CreateRunner(CreateRunnerTask rt) { try { @@ -186,7 +194,7 @@ private async Task CreateRunner(RunnerTask rt) { _logger.LogError($"Unable to create runner [{rt.Size} on {rt.Arch} | Retry: {rt.RetryCount}]: {ex.Message}"); rt.RetryCount += 1; - Tasks.Enqueue(rt); + CreateTasks.Enqueue(rt); return false; } } diff --git a/Program.cs b/Program.cs index 03a4963..89eb54b 100644 --- a/Program.cs +++ b/Program.cs @@ -196,10 +196,9 @@ private static void JobCompleted(ILogger logger, long jobId, CloudContr } else { - poolMgr.Tasks.Enqueue(new RunnerTask + poolMgr.DeleteTasks.Enqueue(new DeleteRunnerTask { - Action = RunnerAction.Delete, - ServerId = vm.Id + ServerId = vm.Id, }); ProcessedJobCount.Labels(vm.OrgName, vm.Size).Inc(); @@ -265,9 +264,8 @@ private static async Task JobQueued(ILogger logger, string? repoName, L return; } - poolMgr.Tasks.Enqueue(new RunnerTask + poolMgr.CreateTasks.Enqueue(new CreateRunnerTask { - Action = RunnerAction.Create, Arch = arch, Size = size, RunnerToken = runnerToken, diff --git a/RunnerAction.cs b/RunnerAction.cs deleted file mode 100644 index abdc2bf..0000000 --- a/RunnerAction.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace GithubActionsOrchestrator; - -public enum RunnerAction -{ - Create, - Delete -} \ No newline at end of file