diff --git a/testrunner/src/testrunners/docker-testrunner.js b/testrunner/src/testrunners/docker-testrunner.js
index bfc963e..a2d77df 100644
--- a/testrunner/src/testrunners/docker-testrunner.js
+++ b/testrunner/src/testrunners/docker-testrunner.js
@@ -47,6 +47,15 @@ export default async function runJob(job) {
       job.data.config.extends = nconf.get('sitespeedioConfigFile');
     }
 
+    // If we use baseliing setup the directory by default
+    if (
+      (job.data.extras && job.data.extras.includes('--compare.')) ||
+      job.data.config.compare
+    ) {
+      // This is inside the container and we always use /baseline
+      job.data.config.compare.baselinePath = '/baseline';
+    }
+
     await writeFile(
       join(workingDirectory, configFileName),
       JSON.stringify(job.data.config)