From d2bd6b1420804ecd6f505bf19bec290c76edac3e Mon Sep 17 00:00:00 2001
From: Peter Hedenskog <peter@soulgalore.com>
Date: Wed, 26 Jun 2024 10:44:46 +0200
Subject: [PATCH] Make removeOnComplete and removeOnFail configurable (#35)

---
 testrunner/config/default.yaml                  | 7 +++++++
 testrunner/src/testrunners/docker-testrunner.js | 8 ++++++--
 testrunner/src/testrunners/testrunner.js        | 9 +++++++--
 testrunner/src/validateconfig.js                | 8 +++++++-
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/testrunner/config/default.yaml b/testrunner/config/default.yaml
index 5015224..db9502d 100644
--- a/testrunner/config/default.yaml
+++ b/testrunner/config/default.yaml
@@ -68,3 +68,10 @@ executable: "sitespeed.io"
 docker:
   container: "sitespeedio/sitespeed.io:latest"
   extraparameters: "--cap-add=NET_ADMIN"
+
+# The number of jobs to keep in the queue
+# Depending on how many jobs you run and how much memory 
+# you have on your Redis/KeyDb instance you can tune this
+queue:
+  removeOnComplete: 200
+  removeOnFail: 200
\ No newline at end of file
diff --git a/testrunner/src/testrunners/docker-testrunner.js b/testrunner/src/testrunners/docker-testrunner.js
index e35b0a4..b2b6011 100644
--- a/testrunner/src/testrunners/docker-testrunner.js
+++ b/testrunner/src/testrunners/docker-testrunner.js
@@ -22,6 +22,10 @@ export default async function runJob(job) {
   const dockerLogger = log.getLogger(
     `sitespeedio.dockertestrunner.process.${job.id}`
   );
+  // The number of objects to keep in the queue before removal
+  const removeOnComplete = nconf.get('queue:removeOnComplete') || 200;
+  const removeOnFail = nconf.get('queue:removeOnFail') || 200;
+
   let workingDirectory;
   try {
     logger.info(`Start with job ${job.id}`);
@@ -103,8 +107,8 @@ export default async function runJob(job) {
         runTime
       },
       {
-        removeOnComplete: 200,
-        removeOnFail: 200
+        removeOnComplete,
+        removeOnFail
       }
     );
 
diff --git a/testrunner/src/testrunners/testrunner.js b/testrunner/src/testrunners/testrunner.js
index ef84836..3b44020 100644
--- a/testrunner/src/testrunners/testrunner.js
+++ b/testrunner/src/testrunners/testrunner.js
@@ -20,6 +20,11 @@ const { join } = path;
 export default async function runJob(job) {
   const logger = log.getLogger(`sitespeedio.testrunner.${job.id}`);
   let workingDirectory;
+
+  // The number of objects to keep in the queue before removal
+  const removeOnComplete = nconf.get('queue:removeOnComplete') || 200;
+  const removeOnFail = nconf.get('queue:removeOnFail') || 200;
+
   // Make sure we vatch everything that can go wrong
   try {
     logger.info('Start job');
@@ -50,8 +55,8 @@ export default async function runJob(job) {
         runTime
       },
       {
-        removeOnComplete: 200,
-        removeOnFail: 200
+        removeOnComplete,
+        removeOnFail
       }
     );
     if (testResult.exitCode > 0) {
diff --git a/testrunner/src/validateconfig.js b/testrunner/src/validateconfig.js
index b224c3c..a787787 100644
--- a/testrunner/src/validateconfig.js
+++ b/testrunner/src/validateconfig.js
@@ -33,6 +33,11 @@ const locationSchema = Joi.object({
   setup: setupSchema.required()
 });
 
+const queueSchema = Joi.object({
+  removeOnComplete: Joi.number().allow(null).optional(),
+  removeOnFail: Joi.number().allow(null).optional()
+});
+
 // Redis schema
 const redisSchema = Joi.object({
   port: Joi.number().allow(null),
@@ -60,7 +65,8 @@ const configSchema = Joi.object({
   sitespeedioConfigFile: Joi.string().optional(),
   workingDirectory: Joi.string().optional(),
   executable: Joi.string().required(),
-  docker: dockerSchema.required()
+  docker: dockerSchema.required(),
+  queue: queueSchema.optional()
 });
 
 export function validate(config) {