Make removeOnComplete and removeOnFail configurable (#35)

sitespeedio · Jun 26, 2024 · d2bd6b1 · d2bd6b1
1 parent 5d19120
commit d2bd6b1
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 5 deletions.
diff --git a/testrunner/config/default.yaml b/testrunner/config/default.yaml
@@ -68,3 +68,10 @@ executable: "sitespeed.io"
 docker:
   container: "sitespeedio/sitespeed.io:latest"
   extraparameters: "--cap-add=NET_ADMIN"
+
+# The number of jobs to keep in the queue
+# Depending on how many jobs you run and how much memory 
+# you have on your Redis/KeyDb instance you can tune this
+queue:
+  removeOnComplete: 200
+  removeOnFail: 200
diff --git a/testrunner/src/testrunners/docker-testrunner.js b/testrunner/src/testrunners/docker-testrunner.js
@@ -22,6 +22,10 @@ export default async function runJob(job) {
   const dockerLogger = log.getLogger(
     `sitespeedio.dockertestrunner.process.${job.id}`
   );
+  // The number of objects to keep in the queue before removal
+  const removeOnComplete = nconf.get('queue:removeOnComplete') || 200;
+  const removeOnFail = nconf.get('queue:removeOnFail') || 200;
+
   let workingDirectory;
   try {
     logger.info(`Start with job ${job.id}`);
@@ -103,8 +107,8 @@ export default async function runJob(job) {
         runTime
       },
       {
-        removeOnComplete: 200,
-        removeOnFail: 200
+        removeOnComplete,
+        removeOnFail
       }
     );
 

diff --git a/testrunner/src/testrunners/testrunner.js b/testrunner/src/testrunners/testrunner.js
@@ -20,6 +20,11 @@ const { join } = path;
 export default async function runJob(job) {
   const logger = log.getLogger(`sitespeedio.testrunner.${job.id}`);
   let workingDirectory;
+
+  // The number of objects to keep in the queue before removal
+  const removeOnComplete = nconf.get('queue:removeOnComplete') || 200;
+  const removeOnFail = nconf.get('queue:removeOnFail') || 200;
+
   // Make sure we vatch everything that can go wrong
   try {
     logger.info('Start job');
@@ -50,8 +55,8 @@ export default async function runJob(job) {
         runTime
       },
       {
-        removeOnComplete: 200,
-        removeOnFail: 200
+        removeOnComplete,
+        removeOnFail
       }
     );
     if (testResult.exitCode > 0) {

diff --git a/testrunner/src/validateconfig.js b/testrunner/src/validateconfig.js
@@ -33,6 +33,11 @@ const locationSchema = Joi.object({
   setup: setupSchema.required()
 });
 
+const queueSchema = Joi.object({
+  removeOnComplete: Joi.number().allow(null).optional(),
+  removeOnFail: Joi.number().allow(null).optional()
+});
+
 // Redis schema
 const redisSchema = Joi.object({
   port: Joi.number().allow(null),
@@ -60,7 +65,8 @@ const configSchema = Joi.object({
   sitespeedioConfigFile: Joi.string().optional(),
   workingDirectory: Joi.string().optional(),
   executable: Joi.string().required(),
-  docker: dockerSchema.required()
+  docker: dockerSchema.required(),
+  queue: queueSchema.optional()
 });
 
 export function validate(config) {