Add "retry" command to spackbot.

Utilize the "retry" API endpoint for pipelines to re-run the jobs that
had failed during the most recent failed run of a pipeline.

It does require a new comment to listen to and a call to acquire the ID
of the most recently failed pipeline.

spackbot/handlers/pipelines.py

@@ -16,7 +16,7 @@
 GITLAB_TOKEN = os.environ.get("GITLAB_TOKEN")
 
 
-async def run_pipeline(event, gh):
+async def run_pipeline(event, gh, retry_pipeline=False):
     """
     Make a request to re-run a pipeline.
     """
@@ -52,20 +52,42 @@ async def run_pipeline(event, gh):
     branch = pr["head"]["ref"]
     branch = f"github/pr{number}_{branch}"
     branch = urllib.parse.quote_plus(branch)
-
-    url = f"{helpers.gitlab_spack_project_url}/pipeline?ref={branch}"
-    headers = {"PRIVATE-TOKEN": GITLAB_TOKEN}
-
-    logger.info(f"{sender} triggering pipeline, url = {url}")
-
-    # Don't provide GitHub credentials to GitLab!
-    async with aiohttp.ClientSession() as session:
-        async with session.post(url, headers=headers) as response:
-            result = await response.json()
-
+    if not retry_pipeline:
+        trigger_type = "started"
+        url = f"{helpers.gitlab_spack_project_url}/pipeline?ref={branch}"
+        headers = {"PRIVATE-TOKEN": GITLAB_TOKEN}
+
+        # Don't provide GitHub credentials to GitLab!
+        async with aiohttp.ClientSession() as session:
+            async with session.post(url, headers=headers) as response:
+                result = await response.json()
+    else:
+        # Get a list of failed pipelines
+        trigger_type = "retried"
+        url = f"{helpers.gitlab_spack_project_url}/pipelines?status=failed&order_by=updated_at"
+        headers = {"PRIVATE-TOKEN": GITLAB_TOKEN}
+
+        # Don't provide GitHub credentials to GitLab!
+        async with aiohttp.ClientSession() as session:
+            async with session.post(url, headers=headers) as response:
+                pipelines = await response.json()
+        logger.info(f"{sender} triggering pipeline, url = {url}")
+
+        if pipelines and "error" not in pipelines:
+            # Index 0 has the latest pipeline which has been marked as failed
+            # Uses the retry API: https://docs.gitlab.com/ee/api/pipelines.html#retry-jobs-in-a-pipeline
+            url = f"{helpers.gitlab_spack_project_url}/pipelines?{pipelines[0]['id']}/retry"
+            headers = {"PRIVATE-TOKEN": GITLAB_TOKEN}
+
+            # Don't provide GitHub credentials to GitLab!
+            async with aiohttp.ClientSession() as session:
+                async with session.post(url, headers=headers) as response:
+                    result = await response.json()
+        else:
+            result = []
     if "detailed_status" in result and "details_path" in result["detailed_status"]:
         url = f"{helpers.spack_gitlab_url}/{result['detailed_status']['details_path']}"
-        return f"I've started that [pipeline]({url}) for you!"
+        return f"I've {trigger_type} that [pipeline]({url}) for you!"
 
     logger.info(f"Problem triggering pipeline on {branch}")
     logger.info(result)

spackbot/routes.py

@@ -112,6 +112,10 @@ async def add_comments(event, gh, *args, session, **kwargs):
     elif re.search(f"{helpers.botname} (re-?)?run pipeline", comment, re.IGNORECASE):
         logger.info("Responding to request to re-run pipeline...")
         message = await handlers.run_pipeline(event, gh)
+    # @spackbot retry pipeline
+    elif re.search(f"{helpers.botname} retry pipeline", comment, re.IGNORECASE):
+        logger.info("Responding to request to retry failed jobs in pipeline...")
+        message = await handlers.run_pipeline(event, gh, retry_pipeline=True)
 
     if message:
         await gh.post(event.data["issue"]["comments_url"], {}, data={"body": message})