feat: Create version 1 🎉 (#2)

* init * fix * try node script * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * test * fixes * remove job * fix * fix * fix * fix * fix * debug * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * python * use python * add notify cache * again * logging * logging * change * save_cache * lint * test * test * test * logging * logging * add job * test job * test job
Lumen5 · Jul 14, 2023 · a982ef3 · a982ef3
1 parent 30076b1
commit a982ef3
Show file tree

Hide file tree

Showing 9 changed files with 282 additions and 50 deletions.
diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml
@@ -23,19 +23,28 @@ jobs:
   # Create jobs to test the commands of your orbs.
   # You may want to add additional validation steps to ensure the commands are working as expected.
   command-test:
-    docker:
-      - image: cimg/base:current
+    executor: flaky-tests-notify-slack/default
     steps:
-      - checkout
-      # Run your orb's commands to validate them.
-      - flaky-tests-notify-slack/greet
+      - flaky-tests-notify-slack/notify:
+          channel: "test-notify"
+          circle_token: CIRCLE_TOKEN_FOR_FLAKY_NOTIFY
+          notify_threshold: 1
+          project_slug: "gh/Lumen5/luminary"
+          cache_break: v2
 workflows:
   test-deploy:
     jobs:
       # Make sure to include "filters: *filters" in every test job you want to run as part of your deployment.
       # Test your orb's commands in a custom job and test your orb's jobs directly as a part of this workflow.
       - command-test:
           filters: *filters
+      - flaky-tests-notify-slack/notify:
+          filters: *filters
+          channel: "test-notify"
+          circle_token: CIRCLE_TOKEN_FOR_FLAKY_NOTIFY
+          notify_threshold: 1
+          project_slug: "gh/Lumen5/luminary"
+          cache_break: v3
       # The orb must be re-packed for publishing, and saved to the workspace.
       - orb-tools/pack:
           filters: *release-filters
@@ -47,5 +56,6 @@ workflows:
           requires:
             - orb-tools/pack
             - command-test
+            - flaky-tests-notify-slack/notify
           context: orb-publishing
           filters: *release-filters
diff --git a/src/@orb.yml b/src/@orb.yml
@@ -9,5 +9,5 @@ display:
   source_url: "https://github.com/Lumen5/"
 
 # If your orb requires other orbs, you can import them like this. Otherwise remove the "orbs" stanza.
-# orbs:
-#  hello: circleci/hello-build@0.0.5
+orbs:
+  slack: circleci/slack@4.12.5
diff --git a/src/commands/greet.yml b/src/commands/greet.yml
diff --git a/src/commands/notify.yml b/src/commands/notify.yml
@@ -0,0 +1,67 @@
+description: >
+  This command checks for flaky tests in the project and notifies a slack channel if
+  any are found and have flaked more than the desired threshold. This command uses the circleci cache
+  to prevent notifying slack more than once for the same flaky test.
+
+  This command uses the following environment variables:
+  - CIRCLE_TOKEN: CircleCI token for fetching flaky tests via API. (can also specify this via a parameter)
+  - SLACK_ACCESS_TOKEN: Slack token for posting messages to Slack. (see https://circleci.com/developer/orbs/orb/circleci/slack)
+  - SLACK_DEFAULT_CHANNEL: Default channel to post to. (can also specify this via a parameter)
+
+parameters:
+  cache_break:
+    type: string
+    default: v0
+    description: >
+      A manual way to break the cache so notifications are set again. This could be helpful if you want to send
+      flake notifications, for example, once per day by dynamically setting this to the current date e.g. "2020-01-01".
+  channel:
+    type: string
+    default: $SLACK_DEFAULT_CHANNEL
+    description: >
+      Select which channel in which to post to.
+      Channel name or ID will work.
+      You may include a comma separated list of channels if you wish to post to multiple channels at once.
+      You may also set the "SLACK_DEFAULT_CHANNEL" environment variable.
+      Refer to https://circleci.com/developer/orbs/orb/circleci/slack for more information.
+  circle_token:
+    type: env_var_name
+    default: CIRCLE_TOKEN
+    description: >
+      The name of the env variable containing the CircleCI token for fetching flaky tests via API.
+      If not specified, CIRCLE_TOKEN environment variable will be used.
+      Please refer to this document how to generate it.
+      https://circleci.com/docs/managing-api-tokens (requires a v2 token)
+  project_slug:
+    type: string
+    default: ''
+    description: >
+      Example: gh/CircleCI-Public/api-preview-docs.
+      Project slug in the form vcs-slug/org-name/repo-name.
+      If not specified, current project's slug will be used.
+  notify_threshold:
+    type: integer
+    default: 5
+    description: >
+      Threshold for notifying slack channel.
+      If a test has flaked at least this many times, the channel will be notified.
+
+steps:
+  - restore_cache:
+      keys:
+        - notify-test-cache-<< parameters.cache_break >>
+  - run:
+      name: Collecting Flaky Tests
+      environment:
+        PARAM_CIRCLE_TOKEN: <<parameters.circle_token>>
+        PROJECT_SLUG: <<parameters.project_slug>>
+        NOTIFY_THRESHOLD: <<parameters.notify_threshold>>
+      shell: '/usr/bin/env python'
+      command: <<include(scripts/notify.py)>>
+  - slack/notify:
+      template: FLAKY_TESTS_SLACK_TEMPLATE
+      channel: <<parameters.channel>>
+  - save_cache:
+      key: notify-test-cache-<< parameters.cache_break >>-{{ epoch }}
+      paths:
+        - /tmp/notify_record.json
diff --git a/src/executors/default.yml b/src/executors/default.yml
@@ -1,12 +1,9 @@
 description: >
-  This is a sample executor using Docker and Node. If you want to provide a custom environment in your orb, insert your image here.
-  If you do not require an executor, you can simply delete this directory.
-docker:
-  - image: 'cimg/node:<<parameters.tag>>'
+  This orb requires python3 to be available in the environment.
 parameters:
   tag:
-    default: lts
-    description: >
-      Pick a specific cimg/node image variant:
-      https://hub.docker.com/r/cimg/node/tags
+    description: "The `cimg/python` Docker image version tag."
     type: string
+    default: "3.8"
+docker:
+  - image: cimg/python:<< parameters.tag >>
diff --git a/src/jobs/hello.yml b/src/jobs/hello.yml
diff --git a/src/jobs/notify.yml b/src/jobs/notify.yml
@@ -0,0 +1,57 @@
+description: >
+  This job checks for flaky tests in the project and notifies a slack channel if
+  any are found and have flaked more than the desired threshold. This command uses the circleci cache
+  to prevent notifying slack more than once for the same flaky test.
+
+  This job uses the following environment variables:
+  - CIRCLE_TOKEN: CircleCI token for fetching flaky tests via API. (can also specify this via a parameter)
+  - SLACK_ACCESS_TOKEN: Slack token for posting messages to Slack. (see https://circleci.com/developer/orbs/orb/circleci/slack)
+  - SLACK_DEFAULT_CHANNEL: Default channel to post to. (can also specify this via a parameter)
+
+executor: default
+
+parameters:
+  cache_break:
+    type: string
+    default: v0
+    description: >
+      A manual way to break the cache so notifications are set again. This could be helpful if you want to send
+      flake notifications, for example, once per day by dynamically setting this to the current date e.g. "2020-01-01".
+  channel:
+    type: string
+    default: $SLACK_DEFAULT_CHANNEL
+    description: >
+      Select which channel in which to post to.
+      Channel name or ID will work.
+      You may include a comma separated list of channels if you wish to post to multiple channels at once.
+      You may also set the "SLACK_DEFAULT_CHANNEL" environment variable.
+      Refer to https://circleci.com/developer/orbs/orb/circleci/slack for more information.
+  circle_token:
+    type: env_var_name
+    default: CIRCLE_TOKEN
+    description: >
+      The name of the env variable containing the CircleCI token for fetching flaky tests via API.
+      If not specified, CIRCLE_TOKEN environment variable will be used.
+      Please refer to this document how to generate it.
+      https://circleci.com/docs/managing-api-tokens (requires a v2 token)
+  project_slug:
+    type: string
+    default: ''
+    description: >
+      Example: gh/CircleCI-Public/api-preview-docs.
+      Project slug in the form vcs-slug/org-name/repo-name.
+      If not specified, current project's slug will be used.
+  notify_threshold:
+    type: integer
+    default: 5
+    description: >
+      Threshold for notifying slack channel.
+      If a test has flaked at least this many times, the channel will be notified.
+
+steps:
+  - notify:
+      cache_break: <<parameters.cache_break>>
+      channel: <<parameters.channel>>
+      circle_token: <<parameters.circle_token>>
+      notify_threshold: <<parameters.notify_threshold>>
+      project_slug: <<parameters.project_slug>>
diff --git a/src/scripts/greet.sh b/src/scripts/greet.sh
diff --git a/src/scripts/notify.py b/src/scripts/notify.py
@@ -0,0 +1,136 @@
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.request
+
+param_circle_token = os.environ.get('PARAM_CIRCLE_TOKEN', '')
+token = os.environ.get(param_circle_token, '')
+project_slug = os.environ.get('PROJECT_SLUG', '')
+notify_threshold = int(os.environ.get('NOTIFY_THRESHOLD', 1))
+
+# extract the project slug from the CIRCLE_BUILD_URL if not provided
+if not project_slug:
+    circle_build_url = os.environ.get('CIRCLE_BUILD_URL', '')
+    pattern = r"https://circleci.com/|/[0-9]*$"
+    project_slug = re.sub(pattern, "", circle_build_url)
+
+print(f"Using project slug: {project_slug}")
+
+pattern = r"/([^/]+)$"
+match = re.search(pattern, project_slug)
+if match:
+    repo_name = match.group(1)
+else:
+    repo_name = os.environ.get('CIRCLE_PROJECT_REPONAME', 'unknown')
+
+# Create a GET request to https://circleci.com/docs/api/v2/index.html#operation/getFlakyTests
+url = "https://circleci.com/api/v2/insights/{}/flaky-tests".format(project_slug)
+req = urllib.request.Request(url, headers={"circle-token": token})
+
+try:
+    with urllib.request.urlopen(req) as response:
+        res = response.read().decode('utf-8')
+except Exception as e:
+    print("Request failed with status code:", e.code)
+    print("Response:", e.read().decode('utf-8'))
+    sys.exit(1)
+
+print("Request successful")
+data = json.loads(res)
+
+tests_above_threshold = [test for test in data["flaky_tests"] if test["times_flaked"] >= notify_threshold]
+
+# filter out any tests that we've already notified about
+notify_record_path = "/tmp/notify_record.json"
+notified_tests = {}
+if os.path.exists(notify_record_path):
+    print("Found existing notify record")
+    with open(notify_record_path, "r") as f:
+        notified_tests = json.load(f)
+
+filtered_tests = [test for test in tests_above_threshold if test["test_name"] not in notified_tests]
+
+if len(filtered_tests) == 0:
+    print(f"No flaky tests to notify about.")
+    num_below_threshold = len(data["flaky_tests"]) - len(tests_above_threshold)
+    if num_below_threshold > 0:
+        print(f"However, there are {num_below_threshold} flaky tests that are below the threshold ({notify_threshold}).")
+    subprocess.run(["circleci-agent", "step", "halt"])
+    sys.exit(0)
+
+# record the tests above threshold, so we don't notify about them again
+for test in tests_above_threshold:
+    notified_tests[test["test_name"]] = True
+with open(notify_record_path, "w") as f:
+    json.dump(notified_tests, f, ensure_ascii=False, indent=4)
+
+print(f"Found {len(tests_above_threshold)} flaky tests at or above the threshold ({notify_threshold}).")
+
+blocks = [
+    {
+        "type": "header",
+        "text": {
+            "type": "plain_text",
+            "text": f":warning: Flaky tests detected in the {repo_name} repo",
+            "emoji": True
+        }
+    },
+]
+
+for test in filtered_tests:
+    blocks.append(
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": f"*<https://app.circleci.com/pipelines/{project_slug}/{test['pipeline_number']}/workflows/{test['workflow_id']}/jobs/{test['job_number']}/tests|{test['test_name']}>*"
+            },
+        },
+    )
+    blocks.append(
+        {
+            "type": "context",
+            "elements": [
+                {
+                    "type": "mrkdwn",
+                    "text": f"*Job:* {test['job_name']}"
+                },
+                {
+                    "type": "mrkdwn",
+                    "text": f"*File:* {test['file']}"
+                },
+                {
+                    "type": "mrkdwn",
+                    "text": f"*Times flaked:* {test['times_flaked']}"
+                }
+            ],
+        }
+    )
+
+blocks.append(
+    {
+        "type": "actions",
+        "elements": [
+            {
+                "type": "button",
+                "text": {
+                    "type": "plain_text",
+                    "text": "Open insights dashboard",
+                },
+                "url": f"https://app.circleci.com/insights/{project_slug}"
+            }
+        ]
+    }
+)
+
+template_path = '/tmp/flaky_tests_slack_template.json'
+with open(template_path, "w") as outfile:
+    json.dump({"blocks": blocks}, outfile, ensure_ascii=False, indent=4)
+
+# Export the template as an environment variable so the Slack orb can use it
+bash_env_file = os.environ.get('BASH_ENV')
+if bash_env_file:
+    with open(bash_env_file, 'a') as env_file:
+        env_file.write(f'export FLAKY_TESTS_SLACK_TEMPLATE=$(cat {template_path})\n')