From b31ca84bfaa7eb610582b8a26f803d5b29e86d0c Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sun, 25 Feb 2024 12:49:24 -0500 Subject: [PATCH 01/19] Add click-spinner --- circfirm/cli/__init__.py | 12 +++++++++++- requirements.txt | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/circfirm/cli/__init__.py b/circfirm/cli/__init__.py index 05cac9c..40440b4 100644 --- a/circfirm/cli/__init__.py +++ b/circfirm/cli/__init__.py @@ -16,6 +16,7 @@ from typing import Any, Callable, Dict, Iterable, Optional import click +import click_spinner import circfirm import circfirm.backend @@ -34,13 +35,22 @@ def announce_and_await( func: Callable, args: Iterable = (), kwargs: Optional[Dict[str, Any]] = None, + *, + use_spinner: bool = True, ) -> Any: """Announce an action to be performed, do it, then announce its completion.""" if kwargs is None: kwargs = {} fmt_msg = f"{msg}..." + spinner = click_spinner.spinner() click.echo(fmt_msg, nl=False) - resp = func(*args, **kwargs) + if use_spinner: + spinner.start() + try: + resp = func(*args, **kwargs) + finally: + if use_spinner: + spinner.stop() click.echo(" done") return resp diff --git a/requirements.txt b/requirements.txt index 5b004c1..e34c218 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,11 @@ # # SPDX-License-Identifier: MIT +boto3~=1.34 click~=8.0 +click-spinner~=0.1 packaging~=23.2 psutil~=5.9 pyyaml~=6.0 requests~=2.31 +boto3-stubs[essential]~=1.34 From eabccd41cf4b32144e4da655ebde22b0681f0c59 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sun, 25 Feb 2024 21:22:48 -0500 Subject: [PATCH 02/19] Add failed text printing to announcing function --- circfirm/cli/__init__.py | 17 ++++++++++------- circfirm/cli/cache.py | 1 - 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/circfirm/cli/__init__.py b/circfirm/cli/__init__.py index 40440b4..ef5c92e 100644 --- a/circfirm/cli/__init__.py +++ b/circfirm/cli/__init__.py @@ -47,12 +47,16 @@ def announce_and_await( if use_spinner: spinner.start() try: - resp = func(*args, **kwargs) - finally: - if use_spinner: - spinner.stop() - click.echo(" done") - return resp + try: + resp = func(*args, **kwargs) + finally: + if use_spinner: + spinner.stop() + click.echo(" done") + return resp + except BaseException as err: # pragma: no cover + click.echo(" failed") + raise err def load_subcmd_folder(path: str, super_import_name: str) -> None: @@ -120,7 +124,6 @@ def install(version: str, language: str, board: Optional[str]) -> None: args=(board, version, language), ) except ConnectionError as err: - click.echo(" failed") # Mark as failed click.echo(f"Error: {err.args[0]}") sys.exit(4) else: diff --git a/circfirm/cli/cache.py b/circfirm/cli/cache.py index e79e30c..e865a31 100644 --- a/circfirm/cli/cache.py +++ b/circfirm/cli/cache.py @@ -94,5 +94,4 @@ def cache_save(board: str, version: str, language: str) -> None: args=(board, version, language), ) except ConnectionError as err: - click.echo(" failed") # Mark as failed raise click.exceptions.ClickException(err.args[0]) From c66f48750a24c09808457b69ebac9ad0b8924d70 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sun, 25 Feb 2024 21:27:48 -0500 Subject: [PATCH 03/19] Fix typo in error message --- circfirm/backend.py | 2 +- tests/cli/test_cli_cache.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/circfirm/backend.py b/circfirm/backend.py index 0af2453..fb949e1 100644 --- a/circfirm/backend.py +++ b/circfirm/backend.py @@ -102,7 +102,7 @@ def download_uf2(board: str, version: str, language: str = "en_US") -> None: if response.status_code != SUCCESS: if not list(uf2_file.parent.glob("*")): uf2_file.parent.rmdir() - raise ConnectionError(f"Could not download spectified UF2 file:\n{url}") + raise ConnectionError(f"Could not download the specified UF2 file:\n{url}") with open(uf2_file, mode="wb") as uf2file: uf2file.write(response.content) diff --git a/tests/cli/test_cli_cache.py b/tests/cli/test_cli_cache.py index 0d79484..0e030ee 100644 --- a/tests/cli/test_cli_cache.py +++ b/tests/cli/test_cli_cache.py @@ -84,7 +84,7 @@ def test_cache_save() -> None: assert result.exit_code == 1 assert result.output == ( "Caching firmware version 7.3.0 for feather_m4_express... failed\n" - "Error: Could not download spectified UF2 file:\n" + "Error: Could not download the specified UF2 file:\n" "https://downloads.circuitpython.org/bin/feather_m4_express/" "nolanguage/" "adafruit-circuitpython-feather_m4_express-nolanguage-7.3.0.uf2\n" From 0b685101e000c45d7ff2004c53c4fa618823c5e7 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sun, 25 Feb 2024 23:44:31 -0500 Subject: [PATCH 04/19] Enforce testing of failed printing --- circfirm/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circfirm/cli/__init__.py b/circfirm/cli/__init__.py index ef5c92e..79bd2e8 100644 --- a/circfirm/cli/__init__.py +++ b/circfirm/cli/__init__.py @@ -54,7 +54,7 @@ def announce_and_await( spinner.stop() click.echo(" done") return resp - except BaseException as err: # pragma: no cover + except BaseException as err: click.echo(" failed") raise err From f01467e414d2feec9a7d3c68cab0f3864d6e96ff Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 10:39:26 -0500 Subject: [PATCH 05/19] Fix issue with accidentally matching versions For Issue #50 --- circfirm/cli/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circfirm/cli/cache.py b/circfirm/cli/cache.py index e865a31..7c6d6b6 100644 --- a/circfirm/cli/cache.py +++ b/circfirm/cli/cache.py @@ -43,7 +43,7 @@ def clear( glob_pattern = "*-*" if board is None else f"*-{board}" language_pattern = "-*" if language is None else f"-{language}" glob_pattern += language_pattern - version_pattern = "-*" if version is None else f"-{version}*" + version_pattern = "-*" if version is None else f"-{version}.uf2" glob_pattern += version_pattern matching_files = pathlib.Path(circfirm.UF2_ARCHIVE).rglob(glob_pattern) for matching_file in matching_files: From 893d56db5dbda17d7a5dc2a67f471814801cc1ab Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 10:52:50 -0500 Subject: [PATCH 06/19] Improve typehinting for announce function --- circfirm/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circfirm/cli/__init__.py b/circfirm/cli/__init__.py index 79bd2e8..e006971 100644 --- a/circfirm/cli/__init__.py +++ b/circfirm/cli/__init__.py @@ -32,12 +32,12 @@ def cli() -> None: def announce_and_await( msg: str, - func: Callable, + func: Callable[..., _T], args: Iterable = (), kwargs: Optional[Dict[str, Any]] = None, *, use_spinner: bool = True, -) -> Any: +) -> _T: """Announce an action to be performed, do it, then announce its completion.""" if kwargs is None: kwargs = {} From c40c7a4c3c71dcf9605ea736235ec1affc46f940 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 14:18:28 -0500 Subject: [PATCH 07/19] Add circfirm cache command --- circfirm/__init__.py | 21 ++++- circfirm/cli/__init__.py | 7 ++ circfirm/cli/config.py | 105 +++++++++++++++++++++++ circfirm/startup.py | 27 ++++-- circfirm/templates/settings.yaml | 5 ++ circfirm/templates/settings.yaml.license | 3 + 6 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 circfirm/cli/config.py create mode 100644 circfirm/templates/settings.yaml create mode 100644 circfirm/templates/settings.yaml.license diff --git a/circfirm/__init__.py b/circfirm/__init__.py index 11a5775..301289b 100644 --- a/circfirm/__init__.py +++ b/circfirm/__init__.py @@ -7,14 +7,27 @@ Author(s): Alec Delaney """ -from circfirm.startup import setup_app_dir, setup_file, setup_folder +import os + +from circfirm.startup import ( + specify_app_dir, + specify_file, + specify_folder, + specify_template, +) # Folders -APP_DIR = setup_app_dir("circfirm") -UF2_ARCHIVE = setup_folder(APP_DIR, "archive") +APP_DIR = specify_app_dir("circfirm") +UF2_ARCHIVE = specify_folder(APP_DIR, "archive") # Files -SETTINGS_FILE = setup_file(APP_DIR, "settings.yaml") +_SETTINGS_FILE_SRC = os.path.abspath( + os.path.join(__file__, "..", "templates", "settings.yaml") +) +SETTINGS_FILE = specify_template( + _SETTINGS_FILE_SRC, os.path.join(APP_DIR, "settings.yaml") +) +UF2_BOARD_LIST = specify_file(APP_DIR, "boards.txt") UF2INFO_FILE = "info_uf2.txt" BOOTOUT_FILE = "boot_out.txt" diff --git a/circfirm/cli/__init__.py b/circfirm/cli/__init__.py index e006971..af43e1f 100644 --- a/circfirm/cli/__init__.py +++ b/circfirm/cli/__init__.py @@ -17,6 +17,7 @@ import click import click_spinner +import yaml import circfirm import circfirm.backend @@ -59,6 +60,12 @@ def announce_and_await( raise err +def get_settings() -> Dict[str, Any]: + """Get the contents of the settings file.""" + with open(circfirm.SETTINGS_FILE, encoding="utf-8") as yamlfile: + return yaml.safe_load(yamlfile) + + def load_subcmd_folder(path: str, super_import_name: str) -> None: """Load subcommands dynamically from a folder of modules and packages.""" subcmd_names = [ diff --git a/circfirm/cli/config.py b/circfirm/cli/config.py new file mode 100644 index 0000000..a50f468 --- /dev/null +++ b/circfirm/cli/config.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: 2024 Alec Delaney, for Adafruit Industries +# Based on code from circlink (Authored by Alec Delaney, licensed under MIT license) +# +# SPDX-License-Identifier: MIT + +"""CLI functionality for the config subcommand. + +Author(s): Alec Delaney +""" + +import json +import os + +import click +import yaml + +import circfirm +import circfirm.cli +import circfirm.startup + + +@click.group() +def cli(): + """View and update the configuration settings for the circfirm CLI.""" + + +@cli.command(name="view") +@click.argument("setting", default="all") +def config_view(setting: str) -> None: + """View a config setting.""" + # Get the settings, show all settings if no specific on is specified + settings = circfirm.cli.get_settings() + if setting == "all": + click.echo(json.dumps(settings, indent=4)) + return + + # Get the specified settings + config_args = setting.split(".") + try: + for extra_arg in config_args[:-1]: + settings = settings[extra_arg] + value = settings[config_args[-1]] + except KeyError: + raise click.ClickException(f"Setting {setting} does not exist") + + # Show the specified setting + click.echo(json.dumps(value, indent=4)) + + +@cli.command(name="edit") +@click.argument("setting") +@click.argument("value") +def config_edit( + setting: str, + value: str, +) -> None: + """Update a config setting.""" + # Get the settings, use another reference to parse + orig_settings = circfirm.cli.get_settings() + target_setting = orig_settings + config_args = setting.split(".") + + # Handle bool conversions + if value.lower() == "true": + value = True + elif value.lower() == "false": + value = False + + # Attempt to parse for the specified config setting and set it + try: + for extra_arg in config_args[:-1]: + target_setting = target_setting[extra_arg] + prev_value = target_setting[config_args[-1]] + prev_value_type = type(prev_value) + if prev_value_type == dict: + raise ValueError + if prev_value_type == bool and value not in (True, False): + raise TypeError + target_setting[config_args[-1]] = prev_value_type(value) + except KeyError: + raise click.ClickException(f"Setting {setting} does not exist") + except TypeError: + raise click.ClickException( + f"Cannot use that value for this setting, must be of type {prev_value_type}" + ) + except ValueError: + raise click.ClickException( + "Cannot change this setting, please change the sub-settings within it" + ) + + # Write the settings back to the file + with open(circfirm.SETTINGS_FILE, mode="w", encoding="utf-8") as yamlfile: + yaml.safe_dump(orig_settings, yamlfile) + + +@cli.command(name="reset") +def config_reset() -> None: + """Reset the configuration file with the provided template.""" + os.remove(circfirm.SETTINGS_FILE) + src, dest = [ + (src, dest) + for src, dest in circfirm.startup.TEMPLATE_LIST + if dest == circfirm.SETTINGS_FILE + ][0] + circfirm.startup.ensure_template(src, dest) diff --git a/circfirm/startup.py b/circfirm/startup.py index 5bd2a8f..04b45e2 100644 --- a/circfirm/startup.py +++ b/circfirm/startup.py @@ -9,34 +9,43 @@ import os import pathlib +import shutil +from typing import List, Tuple import click -FOLDER_LIST = [] -FILE_LIST = [] +FOLDER_LIST: List[str] = [] +FILE_LIST: List[str] = [] +TEMPLATE_LIST: List[Tuple[str, str]] = [] -def setup_app_dir(app_name: str) -> str: +def specify_app_dir(app_name: str) -> str: """Set up the application directory.""" app_path = click.get_app_dir(app_name) FOLDER_LIST.append(app_path) return app_path -def setup_folder(*path_parts: str) -> str: +def specify_folder(*path_parts: str) -> str: """Add a folder to the global record list.""" folder_path = os.path.join(*path_parts) FOLDER_LIST.append(folder_path) return folder_path -def setup_file(*path_parts: str) -> str: +def specify_file(*path_parts: str) -> str: """Add a file to the global record list.""" file_path = os.path.join(*path_parts) FILE_LIST.append(file_path) return file_path +def specify_template(src_path: str, dest_path: str) -> str: + """Add a template to the global record list.""" + TEMPLATE_LIST.append((src_path, dest_path)) + return dest_path + + def ensure_dir(dir_path: str, /) -> None: """Ensure the directory exists, create if needed.""" if not os.path.exists(dir_path): @@ -49,9 +58,17 @@ def _ensure_file(file_path: str, /) -> None: pathlib.Path(file_path).touch(exist_ok=True) +def ensure_template(src_path: str, dest_path: str, /) -> None: + """Ensure the template exists, copy it if needed.""" + if not os.path.exists(dest_path): + shutil.copyfile(src_path, dest_path) + + def ensure_app_setup() -> None: """Ensure the entire application folder is set up.""" for folder in FOLDER_LIST: ensure_dir(folder) for file in FILE_LIST: _ensure_file(file) + for template in TEMPLATE_LIST: + ensure_template(*template) diff --git a/circfirm/templates/settings.yaml b/circfirm/templates/settings.yaml new file mode 100644 index 0000000..1a90cdf --- /dev/null +++ b/circfirm/templates/settings.yaml @@ -0,0 +1,5 @@ +token: + github: "" +output: + supporting: + silence: false diff --git a/circfirm/templates/settings.yaml.license b/circfirm/templates/settings.yaml.license new file mode 100644 index 0000000..0f4980d --- /dev/null +++ b/circfirm/templates/settings.yaml.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2024 Alec Delaney, for Adafruit Industries + +SPDX-License-Identifier: MIT From c83a3dc6ff470ed4faf71efc0bb4e8bde202bdbd Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 14:18:47 -0500 Subject: [PATCH 08/19] Add missing TypeVar --- circfirm/cli/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circfirm/cli/__init__.py b/circfirm/cli/__init__.py index af43e1f..d88850f 100644 --- a/circfirm/cli/__init__.py +++ b/circfirm/cli/__init__.py @@ -13,7 +13,7 @@ import shutil import sys import time -from typing import Any, Callable, Dict, Iterable, Optional +from typing import Any, Callable, Dict, Iterable, Optional, TypeVar import click import click_spinner @@ -23,6 +23,8 @@ import circfirm.backend import circfirm.startup +_T = TypeVar("_T") + @click.group() @click.version_option(package_name="circfirm") From 6a3fdb147a194a71cbabbbafc094a4706a818744 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 14:20:28 -0500 Subject: [PATCH 09/19] Allow cache list freeform text output to be silenced --- circfirm/cli/__init__.py | 8 ++++++++ circfirm/cli/cache.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/circfirm/cli/__init__.py b/circfirm/cli/__init__.py index d88850f..f6bf30e 100644 --- a/circfirm/cli/__init__.py +++ b/circfirm/cli/__init__.py @@ -33,6 +33,14 @@ def cli() -> None: circfirm.startup.ensure_app_setup() +def maybe_support(msg: str) -> None: + """Output supporting text based on the configurable settings.""" + settings = get_settings() + do_output: bool = not settings["output"]["supporting"]["silence"] + if do_output: + click.echo(msg) + + def announce_and_await( msg: str, func: Callable[..., _T], diff --git a/circfirm/cli/cache.py b/circfirm/cli/cache.py index 7c6d6b6..d675bb0 100644 --- a/circfirm/cli/cache.py +++ b/circfirm/cli/cache.py @@ -64,11 +64,11 @@ def cache_list(board: Optional[str]) -> None: board_list = os.listdir(circfirm.UF2_ARCHIVE) if not board_list: - click.echo("Versions have not been cached yet for any boards.") + circfirm.cli.maybe_support("Versions have not been cached yet for any boards.") sys.exit(0) if board is not None and board not in board_list: - click.echo(f"No versions for board '{board}' are not cached.") + circfirm.cli.maybe_support(f"No versions for board '{board}' are not cached.") sys.exit(0) specified_board = board if board is not None else None From 5fe4f827db77d95b9ead598a48e842ba355f12b5 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 14:21:16 -0500 Subject: [PATCH 10/19] Add circfirm query command --- circfirm/backend.py | 119 ++++++++++++++++++++++++++++++++++++++++-- circfirm/cli/query.py | 94 +++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 circfirm/cli/query.py diff --git a/circfirm/backend.py b/circfirm/backend.py index fb949e1..22208a0 100644 --- a/circfirm/backend.py +++ b/circfirm/backend.py @@ -7,15 +7,20 @@ Author(s): Alec Delaney """ +import datetime import enum import os import pathlib import re -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple, TypedDict +import boto3 +import botocore +import botocore.client import packaging.version import psutil import requests +from mypy_boto3_s3 import S3ServiceResource import circfirm import circfirm.startup @@ -48,15 +53,57 @@ class Language(enum.Enum): _ALL_LANGAGES = [language.value for language in Language] _ALL_LANGUAGES_REGEX = "|".join(_ALL_LANGAGES) -FIRMWARE_REGEX = "-".join( +_VALID_VERSIONS_CAPTURE = r"(\d+\.\d+\.\d+(?:-(?:\balpha\b|\bbeta\b)\.\d+)*)" +FIRMWARE_REGEX_PATTERN = "-".join( [ - r"adafruit-circuitpython-(.*)", - f"({_ALL_LANGUAGES_REGEX})", - r"(\d+\.\d+\.\d+(?:-(?:\balpha\b|\bbeta\b)\.\d+)*)\.uf2", + r"adafruit-circuitpython", + r"[board]", + r"[language]", + r"[version]\.uf2", ] ) +FIRMWARE_REGEX = ( + FIRMWARE_REGEX_PATTERN.replace(r"[board]", r"(.*)") + .replace(r"[language]", f"({_ALL_LANGUAGES_REGEX})") + .replace(r"[version]", _VALID_VERSIONS_CAPTURE) +) + BOARD_ID_REGEX = r"Board ID:\s*(.*)" +S3_CONFIG = botocore.client.Config(signature_version=botocore.UNSIGNED) +S3_RESOURCE: S3ServiceResource = boto3.resource("s3", config=S3_CONFIG) +BUCKET_NAME = "adafruit-circuit-python" +BUCKET = S3_RESOURCE.Bucket(BUCKET_NAME) + +BOARDS_REGEX = r"ports/.+/boards/([^/]+)" +BOARDS_REGEX_PATTERN2 = r"bin/([board_pattern])/en_US/.*\.uf2" + +_BASE_REQUESTS_HEADERS = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +} + + +class RateLimit(TypedDict): + """Format of a rate limit dictionary.""" + + limit: int + remaining: int + reset: int + used: int + resource: str + + +class GitTreeItem(TypedDict): + """Format of a git tree item dictionary.""" + + path: str + mode: str + type: str + size: int + sha: str + url: str + def _find_device(filename: str) -> Optional[str]: """Find a specific connected device.""" @@ -170,3 +217,65 @@ def get_sorted_boards(board: Optional[str]) -> Dict[str, Dict[str, Set[str]]]: sorted_versions[sorted_version] = versions[sorted_version] boards[board_folder] = sorted_versions return boards + + +def get_rate_limit() -> Tuple[int, int, datetime.datetime]: + """Get the rate limit for the GitHub REST endpoint.""" + response = requests.get( + url="https://api.github.com/rate_limit", + headers=_BASE_REQUESTS_HEADERS, + ) + limit_info: RateLimit = response.json()["rate"] + available: int = limit_info["remaining"] + total: int = limit_info["limit"] + reset_time = datetime.datetime.fromtimestamp(limit_info["reset"]) + return available, total, reset_time + + +def get_board_list(token: str) -> List[str]: + """Get a list of CircuitPython boards.""" + boards = set() + headers = _BASE_REQUESTS_HEADERS.copy() + if token: + headers["Authorization"] = f"Bearer {token}" + response = requests.get( + url="https://api.github.com/repos/adafruit/circuitpython/git/trees/main", + params={ + "recursive": True, + }, + headers=headers, + ) + try: + tree_items: List[GitTreeItem] = response.json()["tree"] + except KeyError as err: + raise ValueError("Could not parse JSON response, check token") from err + for tree_item in tree_items: + if tree_item["type"] != "tree": + continue + result = re.match(BOARDS_REGEX, tree_item["path"]) + if result: + boards.add(result[1]) + return sorted(boards) + + +def get_board_versions( + board: str, language: str = "en_US", *, regex: Optional[str] = None +) -> List[str]: + """Get a list of CircuitPython versions for a given board.""" + prefix = f"bin/{board}/{language}" + firmware_regex = FIRMWARE_REGEX_PATTERN.replace(r"[board]", board).replace( + r"[language]", language + ) + version_regex = f"({regex})" if regex else _VALID_VERSIONS_CAPTURE + firmware_regex = firmware_regex.replace(r"[version]", version_regex) + s3_objects = BUCKET.objects.filter(Prefix=prefix) + versions = set() + for s3_object in s3_objects: + result = re.match(f"{prefix}/{firmware_regex}", s3_object.key) + if result: + try: + _ = packaging.version.Version(result[1]) + versions.add(result[1]) + except packaging.version.InvalidVersion: + pass + return sorted(versions, key=packaging.version.Version, reverse=True) diff --git a/circfirm/cli/query.py b/circfirm/cli/query.py new file mode 100644 index 0000000..aa744dd --- /dev/null +++ b/circfirm/cli/query.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: 2024 Alec Delaney, for Adafruit Industries +# +# SPDX-License-Identifier: MIT + +"""CLI functionality for the query subcommand. + +Author(s): Alec Delaney +""" + +import re + +import click +import packaging.version +import requests + +import circfirm +import circfirm.backend +import circfirm.cli + + +@click.group() +def cli(): + """Query things like latest versions and board lists.""" + + +@cli.command(name="boards") +@click.option( + "-r", "--regex", default=".*", help="Regex pattern to use for board names" +) +def query_boards(regex: str) -> None: + """Query the local CircuitPython board list.""" + settings = circfirm.cli.get_settings() + gh_token = settings["token"]["github"] + do_output = not settings["output"]["supporting"]["silence"] + circfirm.cli.maybe_support( + "Boards list will now be synchronized with the git repository." + ) + if not gh_token: + circfirm.cli.maybe_support( + "Please note that this operation can only be performed 60 times per hour due to GitHub rate limiting." + ) + try: + if do_output: + boards = circfirm.cli.announce_and_await( + "Fetching boards list", + circfirm.backend.get_board_list, + args=(gh_token,), + ) + else: + boards = circfirm.backend.get_board_list(gh_token) + except ValueError as err: + raise click.ClickException(err.args[0]) + except requests.ConnectionError as err: + raise click.ClickException( + "Issue with requesting information from git repository, check network connection" + ) + for board in boards: + board_name = board.strip() + result = re.match(regex, board_name) + if result: + click.echo(board_name) + + +@cli.command(name="versions") +@click.argument("board") +@click.option("-l", "--language", default="en_US", help="CircuitPython language/locale") +@click.option("-r", "--regex", default=".*", help="Regex pattern to use for versions") +def query_versions(board: str, language: str, regex: str) -> None: + """Query the CircuitPython versions available for a board.""" + versions = circfirm.backend.get_board_versions(board, language, regex=regex) + for version in reversed(versions): + click.echo(version) + + +@cli.command(name="latest") +@click.argument("board", default="raspberry_pi_pico") +@click.option("-l", "--language", default="en_US", help="CircuitPython language/locale") +@click.option( + "-p", + "--pre-release", + is_flag=True, + default=False, + help="Consider pre-release versions", +) +def query_latest(board: str, language: str, pre_release: bool) -> None: + """Query the latest CircuitPython versions available.""" + versions = circfirm.backend.get_board_versions(board, language) + if not pre_release: + versions = [ + version + for version in versions + if not packaging.version.Version(version).is_prerelease + ] + click.echo(versions[0]) From 38ec0d1c3be01d8e0160eeda97a401eeff178489 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 14:21:43 -0500 Subject: [PATCH 11/19] Update testing process to git clone a circuitpython repo --- .gitignore | 1 + Makefile | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index fe06630..afc3cef 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,4 @@ testmount/ # Test related tests/backup/ +tests/sandbox/ diff --git a/Makefile b/Makefile index baec6ce..c9a6c58 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: MIT +include .env +export .PHONY: lint lint: @@ -35,6 +37,7 @@ else @echo "Current OS not supported" @exit 1 endif + -@git clone https://github.com/adafruit/circuitpython tests/sandbox/circuitpython --depth 1 .PHONY: test test: @@ -53,11 +56,14 @@ test-clean: ifeq "$(OS)" "Windows_NT" -@subst T: /d -@python scripts/rmdir.py testmount + -@python scripts/rmdir.py tests/sandbox/circuitpython else ifeq "$(shell uname -s)" "Linux" -@sudo umount testmount -@sudo rm -rf testmount -@rm testfs -f + -@rm -rf tests/sandbox/circuitpython else -@hdiutil detach /Volumes/TESTMOUNT -@rm testfs.dmg -f + -@rm -rf tests/sandbox/circuitpython endif From 60f4f371e06df448f8a17a618f67cbf206b6ca05 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 14:22:48 -0500 Subject: [PATCH 12/19] Add tests for backend --- pyproject.toml | 6 ++++++ tests/helpers.py | 8 ++++++++ tests/test_backend.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a626ec4..d9130f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,3 +78,9 @@ source = [ [tool.ruff.lint] select = ["D", "PL", "UP", "I"] ignore = ["D213", "D203"] + +[tool.pytest.ini_options] +filterwarnings = ["ignore:datetime.datetime.utcfromtimestamp():DeprecationWarning"] +addopts = [ + "--ignore=tests/sandbox/circuitpython" +] diff --git a/tests/helpers.py b/tests/helpers.py index cc0f92d..c7f336f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -11,6 +11,7 @@ import pathlib import platform import shutil +from typing import List import circfirm import circfirm.backend @@ -79,3 +80,10 @@ def copy_firmwares() -> None: shutil.copytree( board_folder, os.path.join(circfirm.UF2_ARCHIVE, board_folder.name) ) + + +def get_boards_from_git() -> List[str]: + """Get a list of board names from the sandbox git repository.""" + ports_path = pathlib.Path("tests/sandbox/circuitpython") + board_paths = ports_path.glob("ports/*/boards/*") + return sorted([board.name for board in board_paths if board.is_dir()]) diff --git a/tests/test_backend.py b/tests/test_backend.py index 6195605..c1d0968 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -152,3 +152,41 @@ def test_get_firmware_info() -> None: # Test failed parsing with pytest.raises(ValueError): circfirm.backend.get_firmware_info("cannotparse") + + +def test_get_board_list() -> None: + """Tests the ability of the backend to get the board list.""" + # Test successful retrieval + token = os.environ["GH_TOKEN"] + board_list = circfirm.backend.get_board_list(token) + expected_board_list = tests.helpers.get_boards_from_git() + assert board_list == expected_board_list + + # Test unsuccessful retrieval + with pytest.raises(ValueError): + circfirm.backend.get_board_list("badtoken") + + +def test_get_rate_limit() -> None: + """Tests getting the rate limit for an authenticated GitHub request.""" + available, total, reset_time = circfirm.backend.get_rate_limit() + total_rate_limit = 60 + assert available <= total + assert total == total_rate_limit + assert reset_time + + +def test_get_board_versions() -> None: + """Tests getting firmware versions for a given board.""" + board = "adafruit_feather_rp2040" + language = "cs" + expected_versions = [ + "6.2.0-beta.2", + "6.2.0-beta.1", + "6.2.0-beta.0", + ] + versions = circfirm.backend.get_board_versions(board, language) + assert versions == expected_versions + + # Chck that invalid versions are skipped for code coverage + _ = circfirm.backend.get_board_versions(board, regex=r".*") From 1275a88a3f913166023942909e40ef593291f3a2 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 14:54:32 -0500 Subject: [PATCH 13/19] Add tests for circfirm config command --- tests/test_cli_config.py | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tests/test_cli_config.py diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py new file mode 100644 index 0000000..c62e662 --- /dev/null +++ b/tests/test_cli_config.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2024 Alec Delaney, for Adafruit Industries +# +# SPDX-License-Identifier: MIT + +"""Tests the CLI's config command functionality. + +Author(s): Alec Delaney +""" + +import json + +import yaml +from click.testing import CliRunner + +from circfirm.cli import cli + + +def get_printed_default_settings() -> None: + """Get the default (template) settings as printed.""" + with open("circfirm/templates/settings.yaml", encoding="utf-8") as yamlfile: + settings = yaml.safe_load(yamlfile) + return f"{json.dumps(settings, indent=4)}\n" + + +def test_config() -> None: + """Tests the config view command.""" + runner = CliRunner() + expected_output = get_printed_default_settings() + + # Test viewing all settings + result = runner.invoke(cli, ["config", "view"]) + assert result.exit_code == 0 + assert result.output == expected_output + + # Test viewing specific setting + result = runner.invoke(cli, ["config", "view", "output.supporting.silence"]) + assert result.exit_code == 0 + assert result.output == "false\n" + + # Test viewing non-existent setting + result = runner.invoke(cli, ["config", "view", "doesnotexist"]) + assert result.exit_code != 0 + + # Test writing a setting + runner.invoke(cli, ["config", "edit", "output.supporting.silence", "true"]) + result = runner.invoke(cli, ["config", "view", "output.supporting.silence"]) + assert result.exit_code == 0 + assert result.output == "true\n" + runner.invoke(cli, ["config", "edit", "output.supporting.silence", "false"]) + result = runner.invoke(cli, ["config", "view", "output.supporting.silence"]) + assert result.exit_code == 0 + assert result.output == "false\n" + + # Test writing a non-existent setting + result = runner.invoke(cli, ["config", "edit", "doesnotexist", "123"]) + assert result.exit_code != 0 + + # Test writing to setting tree + result = runner.invoke(cli, ["config", "edit", "output", "123"]) + assert result.exit_code != 0 + + # Test writing bad type + runner.invoke(cli, ["config", "edit", "output.supporting.silence", "123"]) + assert result.exit_code != 0 + + +def test_config_reset() -> None: + """Tests the reseting of the config settings file.""" + runner = CliRunner() + + runner.invoke(cli, ["config", "edit", "output.supporting.silence", "true"]) + result = runner.invoke(cli, ["config", "view", "output.supporting.silence"]) + assert result.exit_code == 0 + assert result.output == "true\n" + + expected_settings = get_printed_default_settings() + + result = runner.invoke(cli, ["config", "reset"]) + result = runner.invoke(cli, ["config", "view"]) + assert result.exit_code == 0 + assert result.output == expected_settings From 42ba4fbf360e4e97328f099f857ca1cdf034059b Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 16:34:58 -0500 Subject: [PATCH 14/19] Using Python to check and load environment variables --- Makefile | 3 --- tests/conftest.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c9a6c58..bb1008a 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,6 @@ # # SPDX-License-Identifier: MIT -include .env -export - .PHONY: lint lint: @pre-commit run ruff --all-files diff --git a/tests/conftest.py b/tests/conftest.py index 275edf4..4fd5d4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ Author(s): Alec Delaney """ +import os import pathlib import shutil from typing import Union @@ -22,7 +23,20 @@ def pytest_sessionstart(session: pytest.Session) -> None: """Save the current cron table before testing.""" + # Load environment variables if not in GitHub Actions + if "GH_TOKEN" not in os.environ: + with open(".env", mode="r", encoding="utf-8") as envfile: + env_contents = envfile.read() + for envline in env_contents.split("\n"): + if not envline: + continue + name, value = envline.split("=") + os.environ[name] = value + + # Create the backup directory BACKUP_FOLDER.mkdir(exist_ok=True) + + # Save existing settings, if they exist if CONFIG_EXISTS: # pragma: no cover shutil.move(APP_DIR, "tests/backup") From 29c3b478b556cc31dc89c4321f7384cb490e9b3f Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 16:35:28 -0500 Subject: [PATCH 15/19] Only print versions if any matching ones were found --- circfirm/cli/query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circfirm/cli/query.py b/circfirm/cli/query.py index aa744dd..b8b5098 100644 --- a/circfirm/cli/query.py +++ b/circfirm/cli/query.py @@ -91,4 +91,5 @@ def query_latest(board: str, language: str, pre_release: bool) -> None: for version in versions if not packaging.version.Version(version).is_prerelease ] - click.echo(versions[0]) + if versions: + click.echo(versions[0]) From 9f5380ab1e538adf32cd2ba56df161b9ab7c851d Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 16:36:21 -0500 Subject: [PATCH 16/19] Update formatting for conftest.py --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4fd5d4f..df1a0fa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: """Save the current cron table before testing.""" # Load environment variables if not in GitHub Actions if "GH_TOKEN" not in os.environ: - with open(".env", mode="r", encoding="utf-8") as envfile: + with open(".env", encoding="utf-8") as envfile: env_contents = envfile.read() for envline in env_contents.split("\n"): if not envline: @@ -35,7 +35,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: # Create the backup directory BACKUP_FOLDER.mkdir(exist_ok=True) - + # Save existing settings, if they exist if CONFIG_EXISTS: # pragma: no cover shutil.move(APP_DIR, "tests/backup") From 367d3a5fc8dde00f320ed760c2e199fe11482f1d Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 16:36:33 -0500 Subject: [PATCH 17/19] Add tests for circfirm query command --- tests/test_cli_query.py | 130 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 tests/test_cli_query.py diff --git a/tests/test_cli_query.py b/tests/test_cli_query.py new file mode 100644 index 0000000..2fdf78e --- /dev/null +++ b/tests/test_cli_query.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: 2024 Alec Delaney, for Adafruit Industries +# +# SPDX-License-Identifier: MIT + +"""Tests the CLI's query command functionality. + +Author(s): Alec Delaney +""" + +import os +from typing import NoReturn + +import pytest +import requests +from click.testing import CliRunner + +import circfirm.backend +import tests.helpers +from circfirm.cli import cli + + +def simulate_no_connection(arg: str) -> NoReturn: + """Simulate a network error by raising requests.ConnectionError.""" + raise requests.ConnectionError + + +def test_query_boards(monkeypatch: pytest.MonkeyPatch) -> None: + """Tests the ability to query the boards using the CLI.""" + runner = CliRunner() + + # Test an unauthenticated request with supporting text + boards = tests.helpers.get_boards_from_git() + pre_expected_output = "".join([f"{board}\n" for board in boards]) + expected_output = "\n".join( + [ + "Boards list will now be synchronized with the git repository.", + "Please note that this operation can only be performed 60 times per hour due to GitHub rate limiting.", + "Fetching boards list... done", + pre_expected_output, + ] + ) + + result = runner.invoke(cli, ["query", "boards"]) + assert result.exit_code == 0 + assert result.output == expected_output + + # Test an authenticated request without supporting text + result = runner.invoke( + cli, ["config", "edit", "token.github", os.environ["GH_TOKEN"]] + ) + assert result.exit_code == 0 + result = runner.invoke(cli, ["config", "edit", "output.supporting.silence", "true"]) + assert result.exit_code == 0 + result = runner.invoke(cli, ["query", "boards"]) + assert result.exit_code == 0 + assert result.output == pre_expected_output + + # Test a request with a faulty token + result = runner.invoke(cli, ["config", "edit", "token.github", "badtoken"]) + assert result.exit_code == 0 + result = runner.invoke(cli, ["query", "boards"]) + assert result.exit_code != 0 + + result = runner.invoke(cli, ["config", "reset"]) + assert result.exit_code == 0 + + # Tests failure when cannot fetch results due to no network connection + monkeypatch.setattr(circfirm.backend, "get_board_list", simulate_no_connection) + result = runner.invoke(cli, ["query", "boards"]) + assert result.exit_code != 0 + assert ( + result.output.split("\n")[-2] + == "Error: Issue with requesting information from git repository, check network connection" + ) + + +def test_query_versions() -> None: + """Tests the ability to query firmware versions using the CLI.""" + board = "adafruit_feather_rp2040" + language = "cs" + expected_versions = [ + "6.2.0-beta.0", + "6.2.0-beta.1", + "6.2.0-beta.2", + ] + expected_output = "".join([f"{version}\n" for version in expected_versions]) + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "query", + "versions", + board, + "--language", + language, + ], + ) + assert result.exit_code == 0 + assert result.output == expected_output + + +def test_query_latest() -> None: + """Tests the ability to query the latest version of the firmware using the CLI.""" + board = "adafruit_feather_rp2040" + language = "cs" + expected_output = "6.2.0-beta.2\n" + + # Test without pre-releases included + runner = CliRunner() + result = runner.invoke( + cli, + [ + "query", + "latest", + board, + "--language", + language, + ], + ) + assert result.exit_code == 0 + assert result.output == "" + + # Test with pre-releases included + result = runner.invoke( + cli, + ["query", "latest", board, "--language", language, "--pre-release"], + ) + assert result.exit_code == 0 + assert result.output == expected_output From b9b3ce56ea180043aa64d32ead23211d9a04a3fb Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 16:40:50 -0500 Subject: [PATCH 18/19] Update workflow to use GitHub token in environment --- .github/workflows/push.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 76a05df..71c42f1 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -6,6 +6,8 @@ name: Build CI on: ["push", "pull_request"] +permissions: read-all + jobs: build: name: Run build CI @@ -45,6 +47,8 @@ jobs: - name: Run tests run: | make test-run + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Clean up after tests run: | make test-clean From 9e1de06609d4232f9825ad846ae5c2e607c4ad61 Mon Sep 17 00:00:00 2001 From: Alec Delaney Date: Sat, 2 Mar 2024 16:51:22 -0500 Subject: [PATCH 19/19] Don't test checking and loading environment variables if needed --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index df1a0fa..5796503 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: """Save the current cron table before testing.""" # Load environment variables if not in GitHub Actions - if "GH_TOKEN" not in os.environ: + if "GH_TOKEN" not in os.environ: # pragma: no cover with open(".env", encoding="utf-8") as envfile: env_contents = envfile.read() for envline in env_contents.split("\n"):