From a982ef3aefc827be1bebc6ccde5e444f1c5d94b7 Mon Sep 17 00:00:00 2001 From: Michael Herrmann Date: Fri, 14 Jul 2023 14:07:31 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20Create=20version=201=20=F0=9F=8E=89=20?= =?UTF-8?q?=20(#2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * init * fix * try node script * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * test * fixes * remove job * fix * fix * fix * fix * fix * debug * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * python * use python * add notify cache * again * logging * logging * change * save_cache * lint * test * test * test * logging * logging * add job * test job * test job --- .circleci/test-deploy.yml | 20 ++++-- src/@orb.yml | 4 +- src/commands/greet.yml | 15 ----- src/commands/notify.yml | 67 +++++++++++++++++++ src/executors/default.yml | 13 ++-- src/jobs/hello.yml | 14 ---- src/jobs/notify.yml | 57 ++++++++++++++++ src/scripts/greet.sh | 6 -- src/scripts/notify.py | 136 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 282 insertions(+), 50 deletions(-) delete mode 100755 src/commands/greet.yml create mode 100755 src/commands/notify.yml delete mode 100755 src/jobs/hello.yml create mode 100644 src/jobs/notify.yml delete mode 100644 src/scripts/greet.sh create mode 100644 src/scripts/notify.py diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml index 1ebfda2..5628e7c 100644 --- a/.circleci/test-deploy.yml +++ b/.circleci/test-deploy.yml @@ -23,12 +23,14 @@ jobs: # Create jobs to test the commands of your orbs. # You may want to add additional validation steps to ensure the commands are working as expected. command-test: - docker: - - image: cimg/base:current + executor: flaky-tests-notify-slack/default steps: - - checkout - # Run your orb's commands to validate them. - - flaky-tests-notify-slack/greet + - flaky-tests-notify-slack/notify: + channel: "test-notify" + circle_token: CIRCLE_TOKEN_FOR_FLAKY_NOTIFY + notify_threshold: 1 + project_slug: "gh/Lumen5/luminary" + cache_break: v2 workflows: test-deploy: jobs: @@ -36,6 +38,13 @@ workflows: # Test your orb's commands in a custom job and test your orb's jobs directly as a part of this workflow. - command-test: filters: *filters + - flaky-tests-notify-slack/notify: + filters: *filters + channel: "test-notify" + circle_token: CIRCLE_TOKEN_FOR_FLAKY_NOTIFY + notify_threshold: 1 + project_slug: "gh/Lumen5/luminary" + cache_break: v3 # The orb must be re-packed for publishing, and saved to the workspace. - orb-tools/pack: filters: *release-filters @@ -47,5 +56,6 @@ workflows: requires: - orb-tools/pack - command-test + - flaky-tests-notify-slack/notify context: orb-publishing filters: *release-filters diff --git a/src/@orb.yml b/src/@orb.yml index 5629442..dbff047 100755 --- a/src/@orb.yml +++ b/src/@orb.yml @@ -9,5 +9,5 @@ display: source_url: "https://github.com/Lumen5/" # If your orb requires other orbs, you can import them like this. Otherwise remove the "orbs" stanza. -# orbs: -# hello: circleci/hello-build@0.0.5 +orbs: + slack: circleci/slack@4.12.5 diff --git a/src/commands/greet.yml b/src/commands/greet.yml deleted file mode 100755 index f4e2205..0000000 --- a/src/commands/greet.yml +++ /dev/null @@ -1,15 +0,0 @@ -description: > - This command echos "Hello World" using file inclusion. -# What will this command do? -# Descriptions should be short, simple, and clear. -parameters: - to: - type: string - default: "World" - description: "Hello to whom?" -steps: - - run: - environment: - PARAM_TO: <> - name: Hello Greeting - command: <> diff --git a/src/commands/notify.yml b/src/commands/notify.yml new file mode 100755 index 0000000..8f4db8f --- /dev/null +++ b/src/commands/notify.yml @@ -0,0 +1,67 @@ +description: > + This command checks for flaky tests in the project and notifies a slack channel if + any are found and have flaked more than the desired threshold. This command uses the circleci cache + to prevent notifying slack more than once for the same flaky test. + + This command uses the following environment variables: + - CIRCLE_TOKEN: CircleCI token for fetching flaky tests via API. (can also specify this via a parameter) + - SLACK_ACCESS_TOKEN: Slack token for posting messages to Slack. (see https://circleci.com/developer/orbs/orb/circleci/slack) + - SLACK_DEFAULT_CHANNEL: Default channel to post to. (can also specify this via a parameter) + +parameters: + cache_break: + type: string + default: v0 + description: > + A manual way to break the cache so notifications are set again. This could be helpful if you want to send + flake notifications, for example, once per day by dynamically setting this to the current date e.g. "2020-01-01". + channel: + type: string + default: $SLACK_DEFAULT_CHANNEL + description: > + Select which channel in which to post to. + Channel name or ID will work. + You may include a comma separated list of channels if you wish to post to multiple channels at once. + You may also set the "SLACK_DEFAULT_CHANNEL" environment variable. + Refer to https://circleci.com/developer/orbs/orb/circleci/slack for more information. + circle_token: + type: env_var_name + default: CIRCLE_TOKEN + description: > + The name of the env variable containing the CircleCI token for fetching flaky tests via API. + If not specified, CIRCLE_TOKEN environment variable will be used. + Please refer to this document how to generate it. + https://circleci.com/docs/managing-api-tokens (requires a v2 token) + project_slug: + type: string + default: '' + description: > + Example: gh/CircleCI-Public/api-preview-docs. + Project slug in the form vcs-slug/org-name/repo-name. + If not specified, current project's slug will be used. + notify_threshold: + type: integer + default: 5 + description: > + Threshold for notifying slack channel. + If a test has flaked at least this many times, the channel will be notified. + +steps: + - restore_cache: + keys: + - notify-test-cache-<< parameters.cache_break >> + - run: + name: Collecting Flaky Tests + environment: + PARAM_CIRCLE_TOKEN: <> + PROJECT_SLUG: <> + NOTIFY_THRESHOLD: <> + shell: '/usr/bin/env python' + command: <> + - slack/notify: + template: FLAKY_TESTS_SLACK_TEMPLATE + channel: <> + - save_cache: + key: notify-test-cache-<< parameters.cache_break >>-{{ epoch }} + paths: + - /tmp/notify_record.json diff --git a/src/executors/default.yml b/src/executors/default.yml index ca3f5dc..aa52c6a 100755 --- a/src/executors/default.yml +++ b/src/executors/default.yml @@ -1,12 +1,9 @@ description: > - This is a sample executor using Docker and Node. If you want to provide a custom environment in your orb, insert your image here. - If you do not require an executor, you can simply delete this directory. -docker: - - image: 'cimg/node:<>' + This orb requires python3 to be available in the environment. parameters: tag: - default: lts - description: > - Pick a specific cimg/node image variant: - https://hub.docker.com/r/cimg/node/tags + description: "The `cimg/python` Docker image version tag." type: string + default: "3.8" +docker: + - image: cimg/python:<< parameters.tag >> diff --git a/src/jobs/hello.yml b/src/jobs/hello.yml deleted file mode 100755 index 8f1df04..0000000 --- a/src/jobs/hello.yml +++ /dev/null @@ -1,14 +0,0 @@ -description: > - Sample description -# What will this job do? - -executor: default - -parameters: - to: - type: string - default: "World" - description: "Hello to whom?" -steps: - - greet: - to: << parameters.to >> diff --git a/src/jobs/notify.yml b/src/jobs/notify.yml new file mode 100644 index 0000000..461aab5 --- /dev/null +++ b/src/jobs/notify.yml @@ -0,0 +1,57 @@ +description: > + This job checks for flaky tests in the project and notifies a slack channel if + any are found and have flaked more than the desired threshold. This command uses the circleci cache + to prevent notifying slack more than once for the same flaky test. + + This job uses the following environment variables: + - CIRCLE_TOKEN: CircleCI token for fetching flaky tests via API. (can also specify this via a parameter) + - SLACK_ACCESS_TOKEN: Slack token for posting messages to Slack. (see https://circleci.com/developer/orbs/orb/circleci/slack) + - SLACK_DEFAULT_CHANNEL: Default channel to post to. (can also specify this via a parameter) + +executor: default + +parameters: + cache_break: + type: string + default: v0 + description: > + A manual way to break the cache so notifications are set again. This could be helpful if you want to send + flake notifications, for example, once per day by dynamically setting this to the current date e.g. "2020-01-01". + channel: + type: string + default: $SLACK_DEFAULT_CHANNEL + description: > + Select which channel in which to post to. + Channel name or ID will work. + You may include a comma separated list of channels if you wish to post to multiple channels at once. + You may also set the "SLACK_DEFAULT_CHANNEL" environment variable. + Refer to https://circleci.com/developer/orbs/orb/circleci/slack for more information. + circle_token: + type: env_var_name + default: CIRCLE_TOKEN + description: > + The name of the env variable containing the CircleCI token for fetching flaky tests via API. + If not specified, CIRCLE_TOKEN environment variable will be used. + Please refer to this document how to generate it. + https://circleci.com/docs/managing-api-tokens (requires a v2 token) + project_slug: + type: string + default: '' + description: > + Example: gh/CircleCI-Public/api-preview-docs. + Project slug in the form vcs-slug/org-name/repo-name. + If not specified, current project's slug will be used. + notify_threshold: + type: integer + default: 5 + description: > + Threshold for notifying slack channel. + If a test has flaked at least this many times, the channel will be notified. + +steps: + - notify: + cache_break: <> + channel: <> + circle_token: <> + notify_threshold: <> + project_slug: <> diff --git a/src/scripts/greet.sh b/src/scripts/greet.sh deleted file mode 100644 index 7d511d5..0000000 --- a/src/scripts/greet.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# This example uses envsubst to support variable substitution in the string parameter type. -# https://circleci.com/docs/orbs-best-practices/#accepting-parameters-as-strings-or-environment-variables -TO=$(circleci env subst "${PARAM_TO}") -# If for any reason the TO variable is not set, default to "World" -echo "Hello ${TO:-World}!" diff --git a/src/scripts/notify.py b/src/scripts/notify.py new file mode 100644 index 0000000..301d15d --- /dev/null +++ b/src/scripts/notify.py @@ -0,0 +1,136 @@ +import json +import os +import re +import subprocess +import sys +import urllib.request + +param_circle_token = os.environ.get('PARAM_CIRCLE_TOKEN', '') +token = os.environ.get(param_circle_token, '') +project_slug = os.environ.get('PROJECT_SLUG', '') +notify_threshold = int(os.environ.get('NOTIFY_THRESHOLD', 1)) + +# extract the project slug from the CIRCLE_BUILD_URL if not provided +if not project_slug: + circle_build_url = os.environ.get('CIRCLE_BUILD_URL', '') + pattern = r"https://circleci.com/|/[0-9]*$" + project_slug = re.sub(pattern, "", circle_build_url) + +print(f"Using project slug: {project_slug}") + +pattern = r"/([^/]+)$" +match = re.search(pattern, project_slug) +if match: + repo_name = match.group(1) +else: + repo_name = os.environ.get('CIRCLE_PROJECT_REPONAME', 'unknown') + +# Create a GET request to https://circleci.com/docs/api/v2/index.html#operation/getFlakyTests +url = "https://circleci.com/api/v2/insights/{}/flaky-tests".format(project_slug) +req = urllib.request.Request(url, headers={"circle-token": token}) + +try: + with urllib.request.urlopen(req) as response: + res = response.read().decode('utf-8') +except Exception as e: + print("Request failed with status code:", e.code) + print("Response:", e.read().decode('utf-8')) + sys.exit(1) + +print("Request successful") +data = json.loads(res) + +tests_above_threshold = [test for test in data["flaky_tests"] if test["times_flaked"] >= notify_threshold] + +# filter out any tests that we've already notified about +notify_record_path = "/tmp/notify_record.json" +notified_tests = {} +if os.path.exists(notify_record_path): + print("Found existing notify record") + with open(notify_record_path, "r") as f: + notified_tests = json.load(f) + +filtered_tests = [test for test in tests_above_threshold if test["test_name"] not in notified_tests] + +if len(filtered_tests) == 0: + print(f"No flaky tests to notify about.") + num_below_threshold = len(data["flaky_tests"]) - len(tests_above_threshold) + if num_below_threshold > 0: + print(f"However, there are {num_below_threshold} flaky tests that are below the threshold ({notify_threshold}).") + subprocess.run(["circleci-agent", "step", "halt"]) + sys.exit(0) + +# record the tests above threshold, so we don't notify about them again +for test in tests_above_threshold: + notified_tests[test["test_name"]] = True +with open(notify_record_path, "w") as f: + json.dump(notified_tests, f, ensure_ascii=False, indent=4) + +print(f"Found {len(tests_above_threshold)} flaky tests at or above the threshold ({notify_threshold}).") + +blocks = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": f":warning: Flaky tests detected in the {repo_name} repo", + "emoji": True + } + }, +] + +for test in filtered_tests: + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"**" + }, + }, + ) + blocks.append( + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": f"*Job:* {test['job_name']}" + }, + { + "type": "mrkdwn", + "text": f"*File:* {test['file']}" + }, + { + "type": "mrkdwn", + "text": f"*Times flaked:* {test['times_flaked']}" + } + ], + } + ) + +blocks.append( + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": "Open insights dashboard", + }, + "url": f"https://app.circleci.com/insights/{project_slug}" + } + ] + } +) + +template_path = '/tmp/flaky_tests_slack_template.json' +with open(template_path, "w") as outfile: + json.dump({"blocks": blocks}, outfile, ensure_ascii=False, indent=4) + +# Export the template as an environment variable so the Slack orb can use it +bash_env_file = os.environ.get('BASH_ENV') +if bash_env_file: + with open(bash_env_file, 'a') as env_file: + env_file.write(f'export FLAKY_TESTS_SLACK_TEMPLATE=$(cat {template_path})\n')