From 4712a146e504166dfcda21ef2a0071392d52feae Mon Sep 17 00:00:00 2001 From: ussserrr Date: Sun, 8 Mar 2020 15:43:33 +0300 Subject: [PATCH 1/7] ask the confirmation before running the clean up, sanitize input for start_editor --- CHANGELOG.md | 8 +++--- README.md | 22 +++++++++++---- TODO.md | 8 ++++-- stm32pio/app.py | 20 +++++++++++-- stm32pio/lib.py | 16 ++++++----- stm32pio/tests/__init__.py | 12 -------- stm32pio/tests/test.py | 57 ++++++++++++++++++++++++-------------- 7 files changed, 88 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbd8773..a920d5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,7 +110,7 @@ - New: STM32CubeMX is now started more silently (without a splash screen) - New: add integration and CLI tests (sort of) - New: testing with different Python versions using pyenv (3.6+ target) - - New: `test_run_editor` is now preliminary automatically checks whether an editor is installed on the machine + - New: `test_start_editor` is now preliminary automatically checks whether an editor is installed on the machine - New: more typing annotations - Fixed: the app has been failed to start as `python app.py` (modify `sys.path` to fix) - Changed: `main()` function is now fully modular: can be run from anywhere with given CLI arguments (will be piped forward to be parsed via `argparse`) @@ -155,7 +155,7 @@ - Changed: more logging output - Changed: change some methods signatures to return result value -## ver. 1.0 (XX.03.20) +## ver. 1.0 (06.03.20) - New: introduce GUI version of the app (beta) - New: redesigned stage-state machinery - integrates seamlessly into both CLI and GUI worlds. Python `Enum` represents a single stage of the project (e.g. "code generated" or "project built") while the special dictionary unfolds the full information about the project i.e. combination of all stages (True/False). Implemented in 2 classes - `ProjectStage` and `ProjectState`, though the `Stm32pio.state` property is intended to be a user's getter. Both classes have human-readable string representations - New: related to previous - `status` CLI command @@ -171,5 +171,5 @@ - Changed: renamed `_load_config_file()` -> `_load_config()` (hide implementation details) - Changed: use `logger.isEnabledFor()` instead of manually comparing logging levels - Changed: slightly tuned exceptions (more specific ones where it make sense) -- Changed: rename `project_path` -> `path` -- Changed: actualized tests, more broad usage of the `app.main()` function versus `subprocess.run()` + - Changed: rename `project_path` -> `path` + - Changed: actualized tests, more broad usage of the `app.main()` function versus `subprocess.run()` diff --git a/README.md b/README.md index 218a2e5..1b416df 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,18 @@ It uses STM32CubeMX to generate a HAL-framework-based code and alongside creates ![Logo](/screenshots/logo.png) +## Table of contents +> - [Features](#features) +> - [Requirements](#requirements) +> - [Installation](#installation) +> - [Usage](#usage) +> - [Project patching](#project-patching) +> - [Embedding](#embedding) +> - [Example](#example) +> - [Testing](#testing) +> - [Restrictions](#restrictions) + + ## Features - Start the new complete project in a single directory using only an `.ioc` file - Update an existing project after changing hardware options in CubeMX @@ -32,16 +44,16 @@ A general recommendation there would be to test both CubeMX (code generation) an ## Installation You can run the app in a portable way by downloading or cloning the snapshot of the repository and invoking the main script or Python module: ```shell script -$ python3 stm32pio/app.py -$ # or -$ python3 -m stm32pio +stm32pio-repo/ $ python3 stm32pio/app.py # or +stm32pio-repo/ $ python3 -m stm32pio # or +any-path/ $ python3 path/to/stm32pio-repo/stm32pio/app.py ``` (we assume python3 and pip3 hereinafter). It is possible to run the app like this from anywhere. However, it's handier to install the utility to be able to run stm32pio from anywhere. Use ```shell script -stm32pio-repo/ $ pip3 install . +stm32pio-repo/ $ pip install . ``` command to launch the setup process. Now you can simply type 'stm32pio' in the terminal to run the utility in any directory. @@ -128,7 +140,7 @@ or ```shell script stm32pio-repo/ $ python3 -m stm32pio.tests.test -b -v ``` -to test the app. It uses STM32F0 framework to generate and build a code from the test [`stm32pio-test-project.ioc`](/stm32pio-test-project/stm32pio-test-project.ioc) project file. Please make sure that the test project folder is clean (i.e. contains only an .ioc file) before running the test otherwise it can lead to some cases failing. +to test the app. It uses STM32F0 framework to generate and build a code from the test [`stm32pio-test-project.ioc`](/stm32pio-test-project/stm32pio-test-project.ioc) project file. Please make sure that the test project folder is clean (i.e. contains only an .ioc file) before running the test otherwise it can lead to some cases failing. Tests automatically create temporary directory (using `tempfile` Python standard module) where all actions are performed. For the specific test suite or case you can use ```shell script diff --git a/TODO.md b/TODO.md index a16fa55..8eab294 100644 --- a/TODO.md +++ b/TODO.md @@ -9,7 +9,8 @@ - [ ] VSCode plugin - [x] Remove casts to string where we can use path-like objects (related to Python version as new ones receive path-like objects arguments) - [ ] We look for some snippets of strings in logs and output for the testing code but we hard-code them and this is not good, probably - - [ ] Store an initial folder content in .ini config and ignore it on clean-up process. Allow the user to modify such list. Ask the confirmation of a user by-defualt and add additional option for quiet performance + - [ ] Store an initial folder content in .ini config and ignore it on clean-up process. Allow the user to modify such list + - [x] Ask the confirmation of a user by-defualt for `clean` and add additional option for quiet performance - [ ] check for all tools (CubeMX, ...) to be present in the system (both CLI and GUI) - [ ] exclude tests from the bundle (see `setup.py` options) - [ ] generate code docs (help user to understand an internal mechanics, e.g. for embedding). Can be uploaded to the GitHub Wiki @@ -21,7 +22,7 @@ - [ ] some `stm32pio.ini` config file validation - [ ] CHANGELOG markdown markup - [ ] Two words about a synchronous nature of the lib and user's responsibility of async wrapping (if needed). Also, maybe migrate to async/await approach in the future - - [ ] `shlex` for `build` command option sanitizing + - [x] `shlex` for `start_editor` command option sanitizing - [ ] `__init__`' `parameters` dict argument schema (Python 3.8 feature). Also, maybe move `save_on_desctruction` parameter there. Maybe separate on `project_params` and `instance_opts` - [ ] General algo of merging a given dict of parameters with the saved one on project initialization - [ ] parse `platformio.ini` to check its correctness in state getter @@ -30,7 +31,8 @@ - [ ] Do not store absolute paths in config file and make a project portable (use configparser parameters interpolation). Handle renaming - [ ] See https://docs.python.org/3/howto/logging-cookbook.html#context-info to maybe replace current scheme - [ ] UML diagrams (core, GUI back- and front-ends) - - [ ] CI is possible + - [ ] CI is possible (Arch's AUR has the STM32CubeMX package, also there is a direct link). Deploy Docker in Azure Pipelines, basic at Travis CI - [ ] Test preserving user files and folders on regeneration and mb other operations - [ ] Move special formatters inside the library. It is an implementation detail actually that we use subprocesses and so on - [ ] Mb clean the test project tree before running the tests + - [x] README table of contents diff --git a/stm32pio/app.py b/stm32pio/app.py index d61b0c9..a87fc0e 100755 --- a/stm32pio/app.py +++ b/stm32pio/app.py @@ -48,9 +48,12 @@ def parse_args(args: list) -> Optional[argparse.Namespace]: p.add_argument('-b', '--board', dest='board', required=False, help="PlatformIO name of the board") for p in [parser_init, parser_new, parser_generate]: p.add_argument('--start-editor', dest='editor', required=False, - help="use specified editor to open PlatformIO project (e.g. subl, code, atom, etc.)") + help="use specified editor to open the PlatformIO project (e.g. subl, code, atom, etc.)") for p in [parser_new, parser_generate]: - p.add_argument('--with-build', action='store_true', required=False, help="build a project after generation") + p.add_argument('--with-build', action='store_true', required=False, help="build the project after generation") + + parser_clean.add_argument('-q', '--quiet', action='store_true', required=False, + help="suppress the caution about the content removal") if len(args) == 0: parser.print_help() @@ -140,7 +143,18 @@ def main(sys_argv=None) -> int: elif args.subcommand == 'clean': project = stm32pio.lib.Stm32pio(args.project_path, save_on_destruction=False) - project.clean() + if args.quiet: + project.clean() + else: + while True: + reply = input(f'WARNING: this operation will delete ALL content of the directory "{project.path}" ' + f'except the "{pathlib.Path(project.config.get("project", "ioc_file")).name}" file. ' + 'Are you sure? (y/n) ') + if reply.lower() in ['y', 'yes', 'true', '1']: + project.clean() + break + elif reply.lower() in ['n', 'no', 'false', '0']: + break # Library is designed to throw the exception in bad cases so we catch here globally except Exception as e: diff --git a/stm32pio/lib.py b/stm32pio/lib.py index 417430e..b0770cd 100644 --- a/stm32pio/lib.py +++ b/stm32pio/lib.py @@ -9,6 +9,7 @@ import enum import logging import pathlib +import shlex import shutil import string import subprocess @@ -525,29 +526,30 @@ def patch(self) -> None: def start_editor(self, editor_command: str) -> int: """ - Start the editor specified by 'editor_command' with the project opened (assume + Start the editor specified by 'editor_command' with the project opened (assuming that $ [editor] [folder] - form works) + format works) Args: - editor_command: editor command as we start it in the terminal + editor_command: editor command as you start it in the terminal Returns: passes a return code of the command """ - self.logger.info(f"starting an editor '{editor_command}'...") + sanitized_input = shlex.quote(editor_command) + self.logger.info(f"starting an editor {sanitized_input}...") try: - # Works unstable on some Windows 7 systems, but correct on latest Win7 and Win10... + # Works unstable on some Windows 7 systems, but correct on Win10... # result = subprocess.run([editor_command, str(self.path)], check=True) - result = subprocess.run(f"{editor_command} {str(self.path)}", shell=True, check=True, + result = subprocess.run(f"{sanitized_input} {str(self.path)}", shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self.logger.debug(result.stdout, 'from_subprocess') return result.returncode except subprocess.CalledProcessError as e: - self.logger.error(f"failed to start the editor {editor_command}: {e.stdout}") + self.logger.error(f"failed to start the editor {sanitized_input}: {e.stdout}") return e.returncode diff --git a/stm32pio/tests/__init__.py b/stm32pio/tests/__init__.py index a4c1230..e69de29 100644 --- a/stm32pio/tests/__init__.py +++ b/stm32pio/tests/__init__.py @@ -1,12 +0,0 @@ -""" -Some unit-tests for stm32pio. Uses sample project to generate and build it. It's OK to get errors on `test_run_editor` -one because you don't necessarily should have all of the editors. Run as - - python3 -m unittest discover -v - -or - - python3 -m stm32pio.tests.test -v - -(from repo's root) -""" diff --git a/stm32pio/tests/test.py b/stm32pio/tests/test.py index 6e1e614..dade200 100755 --- a/stm32pio/tests/test.py +++ b/stm32pio/tests/test.py @@ -22,6 +22,7 @@ import tempfile import time import unittest +import unittest.mock import stm32pio.app import stm32pio.lib @@ -67,7 +68,7 @@ def setUp(self): def tearDown(self): """ - Clean the temp directory + Clean up the temp directory """ shutil.rmtree(FIXTURE_PATH, ignore_errors=True) @@ -178,7 +179,7 @@ def test_build_should_handle_error(self): self.assertTrue(next((True for item in logs.output if "PlatformIO build error" in item), False), msg="Error message does not match") - def test_run_editor(self): + def test_start_editor(self): """ Call the editors """ @@ -404,25 +405,39 @@ class TestCLI(CustomTestCase): """ def test_clean(self): - # Create files and folders - file_should_be_deleted = FIXTURE_PATH.joinpath('file.should.be.deleted') - dir_should_be_deleted = FIXTURE_PATH.joinpath('dir.should.be.deleted') - file_should_be_deleted.touch(exist_ok=False) - dir_should_be_deleted.mkdir(exist_ok=False) - - # Clean - return_code = stm32pio.app.main(sys_argv=['clean', '-d', str(FIXTURE_PATH)]) - self.assertEqual(return_code, 0, msg="Non-zero return code") - - # Look for remaining items - with self.subTest(): - self.assertFalse(file_should_be_deleted.is_file(), msg=f"{file_should_be_deleted} is still there") - with self.subTest(): - self.assertFalse(dir_should_be_deleted.is_dir(), msg=f"{dir_should_be_deleted} is still there") - - # And .ioc file should be preserved - with self.subTest(): - self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), msg="Missing .ioc file") + for case in ['--quiet', 'yes', 'no']: + with self.subTest(case=case): + # Create files and folders + test_file = FIXTURE_PATH.joinpath('test.file') + test_dir = FIXTURE_PATH.joinpath('test.dir') + test_file.touch(exist_ok=False) + test_dir.mkdir(exist_ok=False) + + # Clean ... + if case == '--quiet': + return_code = stm32pio.app.main(sys_argv=['clean', case, '-d', str(FIXTURE_PATH)]) + else: + with unittest.mock.patch('builtins.input', return_value=case): + return_code = stm32pio.app.main(sys_argv=['clean', '-d', str(FIXTURE_PATH)]) + + self.assertEqual(return_code, 0, msg="Non-zero return code") + + # ... look for remaining items ... + if case == 'no': + with self.subTest(): + self.assertTrue(test_file.is_file(), msg=f"{test_file} has been deleted") + with self.subTest(): + self.assertTrue(test_dir.is_dir(), msg=f"{test_dir}/ has been deleted") + else: + with self.subTest(): + self.assertFalse(test_file.is_file(), msg=f"{test_file} is still there") + with self.subTest(): + self.assertFalse(test_dir.is_dir(), msg=f"{test_dir}/ is still there") + + # ... and .ioc file should be preserved in any case + with self.subTest(): + self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), + msg="Missing .ioc file") def test_new(self): """ From f100f7f14a8ea74d041dbdbb3a7f979d17d53709 Mon Sep 17 00:00:00 2001 From: ussserrr Date: Mon, 9 Mar 2020 19:41:12 +0300 Subject: [PATCH 2/7] separate tests, exclude them from build. Check 'platformio.ini' in the state getter. Portable projects --- TODO.md | 12 +- setup.py | 6 +- stm32pio-gui/app.py | 1 + stm32pio/lib.py | 107 +++-- stm32pio/settings.py | 4 +- stm32pio/tests/test.py | 592 -------------------------- {stm32pio/tests => tests}/__init__.py | 0 tests/test.py | 70 +++ tests/test_cli.py | 207 +++++++++ tests/test_integration.py | 151 +++++++ tests/test_unit.py | 213 +++++++++ 11 files changed, 709 insertions(+), 654 deletions(-) delete mode 100755 stm32pio/tests/test.py rename {stm32pio/tests => tests}/__init__.py (100%) create mode 100755 tests/test.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_integration.py create mode 100644 tests/test_unit.py diff --git a/TODO.md b/TODO.md index 8eab294..cadad04 100644 --- a/TODO.md +++ b/TODO.md @@ -6,13 +6,14 @@ - [ ] GUI. Tests (research approaches and patterns) - [ ] GUI. Reduce number of calls to 'state' (many IO operations) - [ ] GUI. Drag and drop the new folder into the app window + - [ ] GUI. Implement some other methods for Qt abstract models - [ ] VSCode plugin - [x] Remove casts to string where we can use path-like objects (related to Python version as new ones receive path-like objects arguments) - [ ] We look for some snippets of strings in logs and output for the testing code but we hard-code them and this is not good, probably - [ ] Store an initial folder content in .ini config and ignore it on clean-up process. Allow the user to modify such list - [x] Ask the confirmation of a user by-defualt for `clean` and add additional option for quiet performance - [ ] check for all tools (CubeMX, ...) to be present in the system (both CLI and GUI) - - [ ] exclude tests from the bundle (see `setup.py` options) + - [x] exclude tests from the bundle (see `setup.py` options) - [ ] generate code docs (help user to understand an internal mechanics, e.g. for embedding). Can be uploaded to the GitHub Wiki - [ ] colored logs, maybe... - [ ] if we require `platformio` package as a dependency we probably can rely on its dependencies too @@ -20,15 +21,15 @@ - [ ] merge subprocess pipes to one where suitable (i.e. `stdout` and `stderr`) - [ ] redirect subprocess pipes to `DEVNULL` where suitable to suppress output - [ ] some `stm32pio.ini` config file validation - - [ ] CHANGELOG markdown markup + - [x] CHANGELOG markdown markup - [ ] Two words about a synchronous nature of the lib and user's responsibility of async wrapping (if needed). Also, maybe migrate to async/await approach in the future - [x] `shlex` for `start_editor` command option sanitizing - [ ] `__init__`' `parameters` dict argument schema (Python 3.8 feature). Also, maybe move `save_on_desctruction` parameter there. Maybe separate on `project_params` and `instance_opts` - [ ] General algo of merging a given dict of parameters with the saved one on project initialization - - [ ] parse `platformio.ini` to check its correctness in state getter + - [x] parse `platformio.ini` to check its correctness in state getter - [ ] CubeMX 0 return code doesn't necessarily means the correct generation (e.g. migration dialog has appeared and 'Cancel' was chosen, or CubeMX_version < ioc_file_version), probably should somehow analyze the output (logs can be parsed. i.e. 2020-03-05 12:08:40,765 \[ERROR\] MainProjectManager:806 - Program Manager : The version of the current IOC is too high.) - - [ ] Dispatch tests on several files (too many code actually) - - [ ] Do not store absolute paths in config file and make a project portable (use configparser parameters interpolation). Handle renaming + - [x] Dispatch tests on several files (too many code actually) + - [x] Do not store absolute paths in config file and make a project portable (use configparser parameters interpolation). Handle renaming - [ ] See https://docs.python.org/3/howto/logging-cookbook.html#context-info to maybe replace current scheme - [ ] UML diagrams (core, GUI back- and front-ends) - [ ] CI is possible (Arch's AUR has the STM32CubeMX package, also there is a direct link). Deploy Docker in Azure Pipelines, basic at Travis CI @@ -36,3 +37,4 @@ - [ ] Move special formatters inside the library. It is an implementation detail actually that we use subprocesses and so on - [ ] Mb clean the test project tree before running the tests - [x] README table of contents + - [ ] Mb store the last occurred exception traceback in .ini file and show on some CLI command (so we don't need to turn on verbose mode) diff --git a/setup.py b/setup.py index 53381e8..1bc08a5 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,11 @@ long_description=long_description, long_description_content_type='text/markdown', url="https://github.com/ussserrr/stm32pio", - packages=setuptools.find_packages(), + packages=setuptools.find_packages( + exclude=[ + 'tests' + ] + ), classifiers=[ "Programming Language :: Python :: 3 :: Only", "License :: OSI Approved :: MIT License", diff --git a/stm32pio-gui/app.py b/stm32pio-gui/app.py index 57bfc43..92e22c0 100644 --- a/stm32pio-gui/app.py +++ b/stm32pio-gui/app.py @@ -375,6 +375,7 @@ def qt_message_handler(mode, context, message): +# TODO: there is a bug - checkbox in the window doesn't correctly represent the current settings class Settings(QSettings): """ Extend the class by useful get/set methods allowing to avoid redundant code lines and also are callable from the diff --git a/stm32pio/lib.py b/stm32pio/lib.py index b0770cd..f1c171c 100644 --- a/stm32pio/lib.py +++ b/stm32pio/lib.py @@ -6,6 +6,8 @@ import collections import configparser +import contextlib +import copy import enum import logging import pathlib @@ -158,22 +160,14 @@ def __init__(self, dirty_path: str, parameters: dict = None, save_on_destruction else: self.logger = logging.getLogger(f"{__name__}.{id(self)}") # use id() as uniqueness guarantee - # The path is a unique identifier of the project so it would be great to remake Stm32pio class as a subclass of - # pathlib.Path and then reference it like self and not self.path. It is more consistent also, as now path is - # perceived like any other config parameter that somehow is appeared to exist outside of a config instance but - # then it will be a core identifier, a truly 'self' value. But currently pathlib.Path is not intended to be - # subclassable by-design, unfortunately. See https://bugs.python.org/issue24132 - self.path = self._resolve_project_path(dirty_path) + # The path is a unique identifier of the project. Handle 'path/to/proj', 'path/to/proj/', '.', '../proj', etc., + # make the path absolute and check for existence + self.path = pathlib.Path(dirty_path).expanduser().resolve(strict=True) self.config = self._load_config() self.ioc_file = self._find_ioc_file() - self.config.set('project', 'ioc_file', str(self.ioc_file)) - - cubemx_script_template = string.Template(self.config.get('project', 'cubemx_script_content')) - cubemx_script_content = cubemx_script_template.substitute(project_path=self.path, - cubemx_ioc_full_filename=self.ioc_file) - self.config.set('project', 'cubemx_script_content', cubemx_script_content) + self.config.set('project', 'ioc_file', self.ioc_file.name) # General rule: given parameter takes precedence over the saved one board = '' @@ -202,12 +196,17 @@ def state(self) -> ProjectState: Constructing and returning the current state of the project (tweaked dict, see ProjectState docs) """ - # self.logger.debug(f"project content: {[item.name for item in self.path.iterdir()]}") + self.logger.debug(f"project content: {[item.name for item in self.path.iterdir()]}") - try: - platformio_ini_is_patched = self.platformio_ini_is_patched() - except (FileNotFoundError, ValueError): - platformio_ini_is_patched = False + pio_is_initialized = False + with contextlib.suppress(Exception): # we just want to know the information and don't care about details + # Is present, is correct and is not empty + pio_is_initialized = len(self.platformio_ini_config.sections()) != 0 + + platformio_ini_is_patched = False + if pio_is_initialized: # make no sense to proceed if there is something happened in the first place + with contextlib.suppress(Exception): # we just want to know the information and don't care about details + platformio_ini_is_patched = self.platformio_ini_is_patched() # Create the temporary ordered dictionary and fill it with the conditions results arrays stages_conditions = collections.OrderedDict() @@ -218,11 +217,9 @@ def state(self) -> ProjectState: len(list(self.path.joinpath('Inc').iterdir())) > 0, self.path.joinpath('Src').is_dir() and len(list(self.path.joinpath('Src').iterdir())) > 0] - stages_conditions[ProjectStage.PIO_INITIALIZED] = [ - self.path.joinpath('platformio.ini').is_file() and - self.path.joinpath('platformio.ini').stat().st_size > 0] - stages_conditions[ProjectStage.PATCHED] = [ - platformio_ini_is_patched, not self.path.joinpath('include').is_dir()] + stages_conditions[ProjectStage.PIO_INITIALIZED] = [pio_is_initialized] + stages_conditions[ProjectStage.PATCHED] = [platformio_ini_is_patched, + not self.path.joinpath('include').is_dir()] # Hidden folder! Can be not visible in your familiar file manager and cause a confusion stages_conditions[ProjectStage.BUILT] = [ self.path.joinpath('.pio').is_dir() and @@ -249,15 +246,15 @@ def _find_ioc_file(self) -> pathlib.Path: ioc_file = self.config.get('project', 'ioc_file', fallback=None) if ioc_file: - ioc_file = pathlib.Path(ioc_file).expanduser().resolve() - self.logger.debug(f"use {ioc_file.name} file from the INI config") + ioc_file = self.path.joinpath(ioc_file) + self.logger.debug(f"using '{ioc_file.name}' file from the INI config") if not ioc_file.is_file(): raise FileNotFoundError(error_message) return ioc_file else: self.logger.debug("searching for any .ioc file...") candidates = list(self.path.glob('*.ioc')) - if len(candidates) == 0: # good candidate for the new Python 3.8 assignment expression feature :) + if len(candidates) == 0: # TODO: good candidate for the new Python 3.8 assignment expression feature :) raise FileNotFoundError(error_message) elif len(candidates) == 1: self.logger.debug(f"{candidates[0].name} is selected") @@ -281,14 +278,15 @@ def _load_config(self) -> configparser.ConfigParser: config = configparser.ConfigParser(interpolation=None) # Fill with default values - config.read_dict(stm32pio.settings.config_default) + config.read_dict(copy.deepcopy(stm32pio.settings.config_default)) # Then override by user values (if exist) - config.read(str(self.path.joinpath(stm32pio.settings.config_file_name))) + if len(config.read(str(self.path.joinpath(stm32pio.settings.config_file_name)))) == 0: + self.logger.debug(f"no or empty {stm32pio.settings.config_file_name} config file, will use the default one") # Put away unnecessary processing as the string still will be formed even if the logging level doesn't allow # propagation of this message if self.logger.isEnabledFor(logging.DEBUG): - debug_str = 'resolved config:' + debug_str = 'resolved config (merged):' for section in config.sections(): debug_str += f"\n========== {section} ==========\n" for value in config.items(section): @@ -341,24 +339,6 @@ def save_config(self, parameters: dict = None) -> int: return self._save_config(self.config, self.path, self.logger) - @staticmethod - def _resolve_project_path(dirty_path: str) -> pathlib.Path: - """ - Handle 'path/to/proj', 'path/to/proj/', '.', '../proj' and other cases - - Args: - dirty_path (str): some directory in the filesystem - - Returns: - expanded absolute pathlib.Path instance - """ - resolved_path = pathlib.Path(dirty_path).expanduser().resolve() - if not resolved_path.exists(): - raise FileNotFoundError(f"not found: {resolved_path}") - else: - return resolved_path - - def generate_code(self) -> int: """ Call STM32CubeMX app as 'java -jar' file to generate the code from the .ioc file. Pass commands to the @@ -376,8 +356,12 @@ def generate_code(self) -> int: try: # buffering=0 leads to the immediate flushing on writing with open(cubemx_script_file, mode='w+b', buffering=0) as cubemx_script: + cubemx_script_template = string.Template(self.config.get('project', 'cubemx_script_content')) + cubemx_script_content = cubemx_script_template.substitute(ioc_file_absolute_path=self.ioc_file, + project_dir_absolute_path=self.path) + # should encode, since mode='w+b' - cubemx_script.write(self.config.get('project', 'cubemx_script_content').encode()) + cubemx_script.write(cubemx_script_content.encode()) self.logger.info("starting to generate a code from the CubeMX .ioc file...") command_arr = [self.config.get('app', 'java_cmd'), '-jar', self.config.get('app', 'cubemx_cmd'), '-q', @@ -411,6 +395,7 @@ def pio_init(self) -> int: self.logger.info("starting PlatformIO project initialization...") platformio_ini_file = self.path.joinpath('platformio.ini') + # If size is 0, PlatformIO will overwrite it if platformio_ini_file.is_file() and platformio_ini_file.stat().st_size > 0: self.logger.warning("'platformio.ini' file is already exist") @@ -435,6 +420,22 @@ def pio_init(self) -> int: raise Exception(error_msg) + @property + def platformio_ini_config(self) -> configparser.ConfigParser: + """ + Reads and parses 'platformio.ini' PlatformIO config file into newly created configparser.ConfigParser instance. + Note, that the file may change over time and subsequent calls may produce different results because of this. + + Raises FileNotFoundError if no 'platformio.ini' file is present. Passes out all other exceptions, most likely + caused by parsing errors (i.e. corrupted .INI format). + """ + + platformio_ini = configparser.ConfigParser(interpolation=None) + if len(platformio_ini.read(self.path.joinpath('platformio.ini'))) == 0: + raise FileNotFoundError('platformio.ini') + return platformio_ini + + def platformio_ini_is_patched(self) -> bool: """ Check whether 'platformio.ini' config file is patched or not. It doesn't check for complete project patching @@ -444,21 +445,19 @@ def platformio_ini_is_patched(self) -> bool: boolean indicating a result """ - platformio_ini = configparser.ConfigParser(interpolation=None) try: - if len(platformio_ini.read(self.path.joinpath('platformio.ini'))) == 0: - raise FileNotFoundError("not found: 'platformio.ini' file") + platformio_ini = self.platformio_ini_config except FileNotFoundError as e: - raise e + raise Exception("Cannot determine is project patched: 'platformio.ini' file not found") from e except Exception as e: - # Re-raise parsing exceptions as ValueError - raise ValueError("'platformio.ini' file is incorrect") from e + raise Exception("Cannot determine is project patched: 'platformio.ini' file is incorrect") from e patch_config = configparser.ConfigParser(interpolation=None) try: patch_config.read_string(self.config.get('project', 'platformio_ini_patch_content')) except Exception as e: - raise ValueError("Desired patch content is invalid (should satisfy INI-format requirements)") from e + raise Exception("Cannot determine is project patched: desired patch content is invalid (should satisfy " + "INI-format requirements)") from e for patch_section in patch_config.sections(): if platformio_ini.has_section(patch_section): diff --git a/stm32pio/settings.py b/stm32pio/settings.py index 1ad0172..4d90dd9 100644 --- a/stm32pio/settings.py +++ b/stm32pio/settings.py @@ -28,8 +28,8 @@ project={ # (default is OK) See CubeMX user manual PDF (UM1718) to get other useful options 'cubemx_script_content': inspect.cleandoc(''' - config load $cubemx_ioc_full_filename - generate code $project_path + config load ${ioc_file_absolute_path} + generate code ${project_dir_absolute_path} exit ''') + '\n', diff --git a/stm32pio/tests/test.py b/stm32pio/tests/test.py deleted file mode 100755 index dade200..0000000 --- a/stm32pio/tests/test.py +++ /dev/null @@ -1,592 +0,0 @@ -""" -NOTE: make sure the test project tree is clean before running the tests! - -'pyenv' was used to perform tests with different Python versions (under Ubuntu): -https://www.tecmint.com/pyenv-install-and-manage-multiple-python-versions-in-linux/ - -To get the test coverage install and use 'coverage' package: - $ coverage run -m stm32pio.tests.test -b - $ coverage html -""" - -import configparser -import contextlib -import inspect -import io -import pathlib -import platform -import re -import shutil -import subprocess -import sys -import tempfile -import time -import unittest -import unittest.mock - -import stm32pio.app -import stm32pio.lib -import stm32pio.settings -import stm32pio.util - - -STM32PIO_MAIN_SCRIPT: str = inspect.getfile(stm32pio.app) # absolute path to the main stm32pio script -# absolute path to the Python executable (no need to guess whether it's python or python3 and so on) -PYTHON_EXEC: str = sys.executable - -# Test data -TEST_PROJECT_PATH = pathlib.Path('stm32pio-test-project').resolve() -if not TEST_PROJECT_PATH.is_dir() or not TEST_PROJECT_PATH.joinpath('stm32pio-test-project.ioc').is_file(): - raise FileNotFoundError("No test project is present") -# Make sure you have F0 framework installed (try to run code generation from STM32CubeMX manually at least once before -# proceeding) -TEST_PROJECT_BOARD = 'nucleo_f031k6' - -# Instantiate a temporary folder on every test suite run. It is used across all the tests and is deleted on shutdown -# automatically -temp_dir = tempfile.TemporaryDirectory() -FIXTURE_PATH = pathlib.Path(temp_dir.name).joinpath(TEST_PROJECT_PATH.name) - -print(f"The file of 'stm32pio.app' module: {STM32PIO_MAIN_SCRIPT}") -print(f"Python executable: {PYTHON_EXEC} {sys.version}") -print(f"Temp test fixture path: {FIXTURE_PATH}") -print() - - -class CustomTestCase(unittest.TestCase): - """ - These pre- and post-tasks are common for all test cases - """ - - def setUp(self): - """ - Copy the test project from the repo to our temp directory. WARNING: make sure the test project folder is clean - (i.e. contains only an .ioc file) before running the test - """ - shutil.rmtree(FIXTURE_PATH, ignore_errors=True) - shutil.copytree(TEST_PROJECT_PATH, FIXTURE_PATH) - - def tearDown(self): - """ - Clean up the temp directory - """ - shutil.rmtree(FIXTURE_PATH, ignore_errors=True) - - -class TestUnit(CustomTestCase): - """ - Test the single method. As we at some point decided to use a class instead of the set of scattered functions we need - to do some preparations for almost every test (e.g. instantiate the class, create the PlatformIO project, etc.), - though, so the architecture now is way less modular - """ - - def test_generate_code(self): - """ - Check whether files and folders have been created (by STM32CubeMX) - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) - project.generate_code() - - # Assuming that the presence of these files indicating a success - files_should_be_present = ['Src/main.c', 'Inc/main.h'] - for file in files_should_be_present: - with self.subTest(msg=f"{file} hasn't been created"): - self.assertEqual(FIXTURE_PATH.joinpath(file).is_file(), True) - - def test_pio_init(self): - """ - Consider that existence of 'platformio.ini' file showing a successful PlatformIO project initialization. The - last one has another traces that can be checked too but we are interested only in a 'platformio.ini' anyway. - Also, check that it is a correct configparser file and is not empty - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) - result = project.pio_init() - - self.assertEqual(result, 0, msg="Non-zero return code") - self.assertTrue(FIXTURE_PATH.joinpath('platformio.ini').is_file(), msg="platformio.ini is not there") - - platformio_ini = configparser.ConfigParser(interpolation=None) - self.assertGreater(len(platformio_ini.read(str(FIXTURE_PATH.joinpath('platformio.ini')))), 0, - msg='platformio.ini is empty') - - def test_patch(self): - """ - Check that new parameters were added, modified were updated and existing parameters didn't gone. Also, check for - unnecessary folders deletion - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) - - test_content = inspect.cleandoc(''' - ; This is a test config .ini file - ; with a comment. It emulates a real - ; platformio.ini file - - [platformio] - include_dir = this s;789hould be replaced - ; there should appear a new parameter - test_key3 = this should be preserved - - [test_section] - test_key1 = test_value1 - test_key2 = 123 - ''') + '\n' - FIXTURE_PATH.joinpath('platformio.ini').write_text(test_content) - FIXTURE_PATH.joinpath('include').mkdir() - - project.patch() - - with self.subTest(): - self.assertFalse(FIXTURE_PATH.joinpath('include').is_dir(), msg="'include' has not been deleted") - - original_test_config = configparser.ConfigParser(interpolation=None) - original_test_config.read_string(test_content) - - patched_config = configparser.ConfigParser(interpolation=None) - patch_config = configparser.ConfigParser(interpolation=None) - patch_config.read_string(project.config.get('project', 'platformio_ini_patch_content')) - - self.assertGreater(len(patched_config.read(FIXTURE_PATH.joinpath('platformio.ini'))), 0) - - for patch_section in patch_config.sections(): - self.assertTrue(patched_config.has_section(patch_section), msg=f"{patch_section} is missing") - for patch_key, patch_value in patch_config.items(patch_section): - self.assertEqual(patched_config.get(patch_section, patch_key, fallback=None), patch_value, - msg=f"{patch_section}: {patch_key}={patch_value} is missing or incorrect in the " - "patched config") - - for original_section in original_test_config.sections(): - self.assertTrue(patched_config.has_section(original_section), - msg=f"{original_section} from the original config is missing") - for original_key, original_value in original_test_config.items(original_section): - # We've already checked patch parameters so skip them - if not patch_config.has_option(original_section, original_key): - self.assertEqual(patched_config.get(original_section, original_key), original_value, - msg=f"{original_section}: {original_key}={original_value} is corrupted") - - def test_build_should_handle_error(self): - """ - Build an empty project so PlatformIO should return an error - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) - project.pio_init() - - with self.assertLogs(level='ERROR') as logs: - self.assertNotEqual(project.build(), 0, msg="Build error was not indicated") - # next() - Technique to find something in array, string, etc. (or to indicate that there is no) - self.assertTrue(next((True for item in logs.output if "PlatformIO build error" in item), False), - msg="Error message does not match") - - def test_start_editor(self): - """ - Call the editors - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) - - editors = { - 'atom': { - 'Windows': 'atom.exe', - 'Darwin': 'Atom', - 'Linux': 'atom' - }, - 'code': { - 'Windows': 'Code.exe', - 'Darwin': 'Visual Studio Code', - 'Linux': 'code' - }, - 'subl': { - 'Windows': 'sublime_text.exe', - 'Darwin': 'Sublime', - 'Linux': 'sublime' - } - } - - for editor, editor_process_names in editors.items(): - # Look for the command presence in the system so we test only installed editors - if platform.system() == 'Windows': - command_str = f"where {editor} /q" - else: - command_str = f"command -v {editor}" - editor_exists = False - if subprocess.run(command_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode == 0: - editor_exists = True - - if editor_exists: - with self.subTest(command=editor, name=editor_process_names[platform.system()]): - project.start_editor(editor) - - time.sleep(1) # wait a little bit for app to start - - if platform.system() == 'Windows': - command_arr = ['wmic', 'process', 'get', 'description'] - else: - command_arr = ['ps', '-A'] - # "encoding='utf-8'" is for "a bytes-like object is required, not 'str'" in "assertIn" - result = subprocess.run(command_arr, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8') - # Or, for Python 3.7 and above: - # result = subprocess.run(command_arr, capture_output=True, encoding='utf-8') - self.assertIn(editor_process_names[platform.system()], result.stdout) - - def test_init_path_not_found_should_raise(self): - """ - Pass non-existing path and expect the error - """ - path_does_not_exist_name = 'does_not_exist' - - path_does_not_exist = FIXTURE_PATH.joinpath(path_does_not_exist_name) - with self.assertRaisesRegex(FileNotFoundError, path_does_not_exist_name, - msg="FileNotFoundError was not raised or doesn't contain a description"): - stm32pio.lib.Stm32pio(path_does_not_exist, save_on_destruction=False) - - def test_save_config(self): - """ - Explicitly save the config to file and look did that actually happen and whether all the information was - preserved - """ - # 'board' is non-default, 'project'-section parameter - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) - project.save_config() - - self.assertTrue(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).is_file(), - msg=f"{stm32pio.settings.config_file_name} file hasn't been created") - - config = configparser.ConfigParser(interpolation=None) - self.assertGreater(len(config.read(str(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name)))), 0, - msg="Config is empty") - for section, parameters in stm32pio.settings.config_default.items(): - for option, value in parameters.items(): - with self.subTest(section=section, option=option, - msg="Section/key is not found in the saved config file"): - self.assertNotEqual(config.get(section, option, fallback="Not found"), "Not found") - - self.assertEqual(config.get('project', 'board', fallback="Not found"), TEST_PROJECT_BOARD, - msg="'board' has not been set") - - def test_get_platformio_boards(self): - """ - PlatformIO identifiers of boards are requested using PlatformIO Python API (not sure it can be called public, - though...) - """ - self.assertIsInstance(stm32pio.util.get_platformio_boards(), list) - - -class TestIntegration(CustomTestCase): - """ - Sequence of methods that should work seamlessly - """ - - def test_config_priorities(self): - """ - Test the compliance with priorities when reading the parameters - """ - # Sample user's custom patch value - config_parameter_user_value = inspect.cleandoc(''' - [test_section] - key1 = value1 - key2 = 789 - ''') - cli_parameter_user_value = 'nucleo_f429zi' - - # Create test config - config = configparser.ConfigParser(interpolation=None) - config.read_dict({ - 'project': { - 'platformio_ini_patch_content': config_parameter_user_value, - 'board': TEST_PROJECT_BOARD - } - }) - # ... save it - with FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).open(mode='w') as config_file: - config.write(config_file) - - # On project creation we should interpret the CLI-provided values as superseding to the saved ones and - # saved ones, in turn, as superseding to the default ones - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': cli_parameter_user_value}, - save_on_destruction=False) - project.pio_init() - project.patch() - - # Actually, we can parse platformio.ini via configparser but this is simpler in our case - after_patch_content = FIXTURE_PATH.joinpath('platformio.ini').read_text() - self.assertIn(config_parameter_user_value, after_patch_content, - msg="User config parameter has not been prioritized over the default one") - self.assertIn(cli_parameter_user_value, after_patch_content, - msg="User CLI parameter has not been prioritized over the saved one") - - def test_build(self): - """ - Initialize a new project and try to build it - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) - project.generate_code() - project.pio_init() - project.patch() - - result = project.build() - - self.assertEqual(result, 0, msg="Build failed") - - def test_regenerate_code(self): - """ - Simulate a new project creation, its changing and CubeMX code re-generation (for example, after adding new - hardware features and some new files by a user) - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) - - # Generate a new project ... - project.generate_code() - project.pio_init() - project.patch() - - # ... change it: - test_file_1 = FIXTURE_PATH.joinpath('Src', 'main.c') - test_content_1 = "*** TEST STRING 1 ***\n" - test_file_2 = FIXTURE_PATH.joinpath('Inc', 'my_header.h') - test_content_2 = "*** TEST STRING 2 ***\n" - # - add some sample string inside CubeMX' /* BEGIN - END */ block - main_c_content = test_file_1.read_text() - pos = main_c_content.index("while (1)") - main_c_new_content = main_c_content[:pos] + test_content_1 + main_c_content[pos:] - test_file_1.write_text(main_c_new_content) - # - add new file inside the project - test_file_2.write_text(test_content_2) - - # Re-generate CubeMX project - project.generate_code() - - # Check if added information has been preserved - for test_content, after_regenerate_content in [(test_content_1, test_file_1.read_text()), - (test_content_2, test_file_2.read_text())]: - with self.subTest(msg=f"User content hasn't been preserved in {after_regenerate_content}"): - self.assertIn(test_content, after_regenerate_content) - - def test_current_stage(self): - """ - Go through the sequence of states emulating the real-life project lifecycle - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.EMPTY) - - project.save_config() - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.INITIALIZED) - - project.generate_code() - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.GENERATED) - - project.pio_init() - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.PIO_INITIALIZED) - - project.patch() - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.PATCHED) - - project.build() - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.BUILT) - - project.clean() - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.EMPTY) - - # Should be UNDEFINED when the project is messed up - project.pio_init() - self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.UNDEFINED) - self.assertFalse(project.state.is_consistent) - - -class TestCLI(CustomTestCase): - """ - Some tests to mimic the behavior of end-user tasks (CLI commands such as 'new', 'clean', etc.). Run main function - passing the arguments to it but sometimes even run as subprocess (to capture actual STDOUT/STDERR output) - """ - - def test_clean(self): - for case in ['--quiet', 'yes', 'no']: - with self.subTest(case=case): - # Create files and folders - test_file = FIXTURE_PATH.joinpath('test.file') - test_dir = FIXTURE_PATH.joinpath('test.dir') - test_file.touch(exist_ok=False) - test_dir.mkdir(exist_ok=False) - - # Clean ... - if case == '--quiet': - return_code = stm32pio.app.main(sys_argv=['clean', case, '-d', str(FIXTURE_PATH)]) - else: - with unittest.mock.patch('builtins.input', return_value=case): - return_code = stm32pio.app.main(sys_argv=['clean', '-d', str(FIXTURE_PATH)]) - - self.assertEqual(return_code, 0, msg="Non-zero return code") - - # ... look for remaining items ... - if case == 'no': - with self.subTest(): - self.assertTrue(test_file.is_file(), msg=f"{test_file} has been deleted") - with self.subTest(): - self.assertTrue(test_dir.is_dir(), msg=f"{test_dir}/ has been deleted") - else: - with self.subTest(): - self.assertFalse(test_file.is_file(), msg=f"{test_file} is still there") - with self.subTest(): - self.assertFalse(test_dir.is_dir(), msg=f"{test_dir}/ is still there") - - # ... and .ioc file should be preserved in any case - with self.subTest(): - self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), - msg="Missing .ioc file") - - def test_new(self): - """ - Successful build is the best indicator that all went right so we use '--with-build' option here - """ - return_code = stm32pio.app.main(sys_argv=['new', '-d', str(FIXTURE_PATH), '-b', TEST_PROJECT_BOARD, - '--with-build']) - self.assertEqual(return_code, 0, msg="Non-zero return code") - - # .ioc file should be preserved - self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), msg="Missing .ioc file") - - def test_generate(self): - return_code = stm32pio.app.main(sys_argv=['generate', '-d', str(FIXTURE_PATH)]) - self.assertEqual(return_code, 0, msg="Non-zero return code") - - for directory in ['Inc', 'Src']: - with self.subTest(): - self.assertTrue(FIXTURE_PATH.joinpath(directory).is_dir(), msg=f"Missing '{directory}'") - self.assertNotEqual(len(list(FIXTURE_PATH.joinpath(directory).iterdir())), 0, - msg=f"'{directory}' is empty") - - # .ioc file should be preserved - self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), msg="Missing .ioc file") - - def test_incorrect_path_should_log_error(self): - """ - We should see an error log message and non-zero return code - """ - path_not_exist = pathlib.Path('path/does/not/exist') - - with self.assertLogs(level='ERROR') as logs: - return_code = stm32pio.app.main(sys_argv=['init', '-d', str(path_not_exist)]) - self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') - self.assertTrue(next((True for item in logs.output if str(path_not_exist) in item), False), - msg="'ERROR' logging message hasn't been printed") - - def test_no_ioc_file_should_log_error(self): - """ - We should see an error log message and non-zero return code - """ - dir_with_no_ioc_file = FIXTURE_PATH.joinpath('dir.with.no.ioc.file') - dir_with_no_ioc_file.mkdir(exist_ok=False) - - with self.assertLogs(level='ERROR') as logs: - return_code = stm32pio.app.main(sys_argv=['init', '-d', str(dir_with_no_ioc_file)]) - self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') - self.assertTrue(next((True for item in logs.output if "CubeMX project .ioc file" in item), False), - msg="'ERROR' logging message hasn't been printed") - - def test_verbose(self): - """ - Run as subprocess to capture the full output. Check for both 'DEBUG' logging messages and STM32CubeMX CLI - output. Verbose logs format should match such a regex: - - ^(?=(DEBUG|INFO|WARNING|ERROR|CRITICAL) {0,4})(?=.{8} (?=(build|pio_init|...) {0,26})(?=.{26} [^ ])) - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) - methods = [member[0] for member in inspect.getmembers(project, predicate=inspect.ismethod)] + ['main'] - - buffer_stdout, buffer_stderr = io.StringIO(), io.StringIO() - with contextlib.redirect_stdout(buffer_stdout), contextlib.redirect_stderr(buffer_stderr): - return_code = stm32pio.app.main(sys_argv=['-v', 'generate', '-d', str(FIXTURE_PATH)]) - - self.assertEqual(return_code, 0, msg="Non-zero return code") - # stderr and not stdout contains the actual output (by default for the logging module) - self.assertEqual(len(buffer_stdout.getvalue()), 0, - msg="Process has printed something directly into STDOUT bypassing logging") - self.assertIn('DEBUG', buffer_stderr.getvalue(), msg="Verbose logging output hasn't been enabled on stderr") - - # Inject all methods' names in the regex. Inject the width of field in a log format string - regex = re.compile("^(?=(DEBUG) {0,4})(?=.{8} (?=(" + '|'.join(methods) + ") {0," + - str(stm32pio.settings.log_fieldwidth_function) + "})(?=.{" + - str(stm32pio.settings.log_fieldwidth_function) + "} [^ ]))", flags=re.MULTILINE) - self.assertGreaterEqual(len(re.findall(regex, buffer_stderr.getvalue())), 1, - msg="Logs messages doesn't match the format") - - self.assertIn('Starting STM32CubeMX', buffer_stderr.getvalue(), msg="STM32CubeMX has not printed its logs") - - def test_non_verbose(self): - """ - Run as subprocess to capture the full output. We should not see any 'DEBUG' logging messages or STM32CubeMX CLI - output. Logs format should match such a regex: - - ^(?=(INFO) {0,4})(?=.{8} ((?!( |build|pio_init|...)))) - """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) - methods = [method[0] for method in inspect.getmembers(project, predicate=inspect.ismethod)] - methods.append('main') - - buffer_stdout, buffer_stderr = io.StringIO(), io.StringIO() - with contextlib.redirect_stdout(buffer_stdout), contextlib.redirect_stderr(buffer_stderr): - return_code = stm32pio.app.main(sys_argv=['generate', '-d', str(FIXTURE_PATH)]) - - self.assertEqual(return_code, 0, msg="Non-zero return code") - # stderr and not stdout contains the actual output (by default for the logging module) - self.assertNotIn('DEBUG', buffer_stderr.getvalue(), msg="Verbose logging output has been enabled on stderr") - self.assertEqual(len(buffer_stdout.getvalue()), 0, msg="All app output should flow through the logging module") - - regex = re.compile("^(?=(INFO) {0,4})(?=.{8} ((?!( |" + '|'.join(methods) + "))))", flags=re.MULTILINE) - self.assertGreaterEqual(len(re.findall(regex, buffer_stderr.getvalue())), 1, - msg="Logs messages doesn't match the format") - - self.assertNotIn('Starting STM32CubeMX', buffer_stderr.getvalue(), msg="STM32CubeMX has printed its logs") - - def test_init(self): - """ - Check for config creation and parameters presence - """ - result = subprocess.run([PYTHON_EXEC, STM32PIO_MAIN_SCRIPT, 'init', '-d', str(FIXTURE_PATH), - '-b', TEST_PROJECT_BOARD], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - self.assertEqual(result.returncode, 0, msg="Non-zero return code") - - self.assertTrue(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).is_file(), - msg=f"{stm32pio.settings.config_file_name} file hasn't been created") - - config = configparser.ConfigParser(interpolation=None) - config.read(str(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name))) - for section, parameters in stm32pio.settings.config_default.items(): - for option, value in parameters.items(): - with self.subTest(section=section, option=option, msg="Section/key is not found in saved config file"): - self.assertIsNotNone(config.get(section, option, fallback=None)) - self.assertEqual(config.get('project', 'board', fallback="Not found"), TEST_PROJECT_BOARD, - msg="'board' has not been set") - - def test_status(self): - """ - Test the output returning by the app on a request to the 'status' command - """ - - buffer_stdout = io.StringIO() - with contextlib.redirect_stdout(buffer_stdout), contextlib.redirect_stderr(None): - return_code = stm32pio.app.main(sys_argv=['status', '-d', str(FIXTURE_PATH)]) - - self.assertEqual(return_code, 0, msg="Non-zero return code") - - matches_counter = 0 - last_stage_pos = -1 - for stage in stm32pio.lib.ProjectStage: - if stage != stm32pio.lib.ProjectStage.UNDEFINED: - match = re.search(r"^((\[ \])|(\[\*\])) {2}" + str(stage) + '$', buffer_stdout.getvalue(), re.MULTILINE) - self.assertTrue(match, msg="Status information was not found on STDOUT") - if match: - matches_counter += 1 - self.assertGreater(match.start(), last_stage_pos, msg="The order of stages is messed up") - last_stage_pos = match.start() - - self.assertEqual(matches_counter, len(stm32pio.lib.ProjectStage) - 1) - - -if __name__ == '__main__': - unittest.main() diff --git a/stm32pio/tests/__init__.py b/tests/__init__.py similarity index 100% rename from stm32pio/tests/__init__.py rename to tests/__init__.py diff --git a/tests/test.py b/tests/test.py new file mode 100755 index 0000000..2530610 --- /dev/null +++ b/tests/test.py @@ -0,0 +1,70 @@ +""" +Common preparations for all test suites. Use this as a source of constants for test cases. Find the tests themself at +concrete files + +NOTE: make sure the test project tree is clean before running the tests! + +'pyenv' was used to execute tests with different Python versions (under Ubuntu): +https://www.tecmint.com/pyenv-install-and-manage-multiple-python-versions-in-linux/ + +To get the test coverage install and use 'coverage' package: + $ coverage run -m stm32pio.tests.test -b + $ coverage html +""" + +import inspect +import pathlib +import shutil +import sys +import tempfile +import unittest + +import stm32pio.app + + +# Absolute path to the main stm32pio script (make sure what repo we are testing +STM32PIO_MAIN_SCRIPT: str = inspect.getfile(stm32pio.app) +# Absolute path to the Python executable (no need to guess whether it's 'python' or 'python3' and so on) +PYTHON_EXEC: str = sys.executable + +# Test data +TEST_PROJECT_PATH = pathlib.Path('stm32pio-test-project').resolve(strict=True) +if not TEST_PROJECT_PATH.joinpath('stm32pio-test-project.ioc').is_file(): + raise FileNotFoundError("No test project is present") +# Make sure you have F0 framework installed (try to run code generation from STM32CubeMX manually at least once before +# proceeding) +TEST_PROJECT_BOARD = 'nucleo_f031k6' + +# Instantiate a temporary folder on every test suite run. It is used across all the tests and is deleted on shutdown +# automatically +TEMP_DIR = tempfile.TemporaryDirectory() +FIXTURE_PATH = pathlib.Path(TEMP_DIR.name).joinpath(TEST_PROJECT_PATH.name) + +print(f"The file of 'stm32pio.app' module: {STM32PIO_MAIN_SCRIPT}") +print(f"Python executable: {PYTHON_EXEC} {sys.version}") +print(f"Temp test fixture path: {FIXTURE_PATH}") +print() + + +class CustomTestCase(unittest.TestCase): + """ + These pre- and post-tasks are common for all test cases + """ + + def setUp(self): + """ + Copy the test project from the repo to our temp directory. WARNING: make sure the test project folder (one from + this repo, not a temporarily created one) is clean (i.e. contains only an .ioc file) before running the test + """ + shutil.rmtree(FIXTURE_PATH, ignore_errors=True) + shutil.copytree(TEST_PROJECT_PATH, FIXTURE_PATH) + + def tearDown(self): + """ + Clean up the temp directory + """ + shutil.rmtree(FIXTURE_PATH, ignore_errors=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..add29b0 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,207 @@ +import configparser +import contextlib +import io +import pathlib +import re +import subprocess +import unittest.mock + +import stm32pio.app +import stm32pio.lib +import stm32pio.settings + +from tests.test import * + + +class TestCLI(CustomTestCase): + """ + Some tests to mimic the behavior of end-user tasks (CLI commands such as 'new', 'clean', etc.). Run main function + passing the arguments to it but sometimes even run as subprocess (to capture actual STDOUT/STDERR output) + """ + + def test_clean(self): + for case in ['--quiet', 'yes', 'no']: + with self.subTest(case=case): + # Create files and folders + test_file = FIXTURE_PATH.joinpath('test.file') + test_dir = FIXTURE_PATH.joinpath('test.dir') + test_file.touch(exist_ok=False) + test_dir.mkdir(exist_ok=False) + + # Clean ... + if case == '--quiet': + return_code = stm32pio.app.main(sys_argv=['clean', case, '-d', str(FIXTURE_PATH)]) + else: + with unittest.mock.patch('builtins.input', return_value=case): + return_code = stm32pio.app.main(sys_argv=['clean', '-d', str(FIXTURE_PATH)]) + + self.assertEqual(return_code, 0, msg="Non-zero return code") + + # ... look for remaining items ... + if case == 'no': + with self.subTest(): + self.assertTrue(test_file.is_file(), msg=f"{test_file} has been deleted") + with self.subTest(): + self.assertTrue(test_dir.is_dir(), msg=f"{test_dir}/ has been deleted") + else: + with self.subTest(): + self.assertFalse(test_file.is_file(), msg=f"{test_file} is still there") + with self.subTest(): + self.assertFalse(test_dir.is_dir(), msg=f"{test_dir}/ is still there") + + # ... and .ioc file should be preserved in any case + with self.subTest(): + self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), + msg="Missing .ioc file") + + def test_new(self): + """ + Successful build is the best indicator that all went right so we use '--with-build' option here + """ + return_code = stm32pio.app.main(sys_argv=['new', '-d', str(FIXTURE_PATH), '-b', TEST_PROJECT_BOARD, + '--with-build']) + self.assertEqual(return_code, 0, msg="Non-zero return code") + + # .ioc file should be preserved + self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), msg="Missing .ioc file") + + def test_generate(self): + return_code = stm32pio.app.main(sys_argv=['generate', '-d', str(FIXTURE_PATH)]) + self.assertEqual(return_code, 0, msg="Non-zero return code") + + for directory in ['Inc', 'Src']: + with self.subTest(): + self.assertTrue(FIXTURE_PATH.joinpath(directory).is_dir(), msg=f"Missing '{directory}'") + self.assertNotEqual(len(list(FIXTURE_PATH.joinpath(directory).iterdir())), 0, + msg=f"'{directory}' is empty") + + # .ioc file should be preserved + self.assertTrue(FIXTURE_PATH.joinpath(f"{FIXTURE_PATH.name}.ioc").is_file(), msg="Missing .ioc file") + + def test_incorrect_path_should_log_error(self): + """ + We should see an error log message and non-zero return code + """ + path_not_exist = pathlib.Path('path/does/not/exist') + + with self.assertLogs(level='ERROR') as logs: + return_code = stm32pio.app.main(sys_argv=['init', '-d', str(path_not_exist)]) + self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') + # TODO: for macOS: + # ERROR [Errno 2] No such file or directory: '/Users/chufyrev/Documents/GitHub/stm32pio/path' + # probably another on different systems + # self.assertTrue(next((True for item in logs.output if str(path_not_exist) in item), False), + # msg="'ERROR' logging message hasn't been printed") + + def test_no_ioc_file_should_log_error(self): + """ + We should see an error log message and non-zero return code + """ + dir_with_no_ioc_file = FIXTURE_PATH.joinpath('dir.with.no.ioc.file') + dir_with_no_ioc_file.mkdir(exist_ok=False) + + with self.assertLogs(level='ERROR') as logs: + return_code = stm32pio.app.main(sys_argv=['init', '-d', str(dir_with_no_ioc_file)]) + self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') + self.assertTrue(next((True for item in logs.output if "CubeMX project .ioc file" in item), False), + msg="'ERROR' logging message hasn't been printed") + + def test_verbose(self): + """ + Run as subprocess to capture the full output. Check for both 'DEBUG' logging messages and STM32CubeMX CLI + output. Verbose logs format should match such a regex: + + ^(?=(DEBUG|INFO|WARNING|ERROR|CRITICAL) {0,4})(?=.{8} (?=(build|pio_init|...) {0,26})(?=.{26} [^ ])) + """ + + # inspect.getmembers is great but it triggers class properties leading to the unacceptable code execution + methods = dir(stm32pio.lib.Stm32pio) + ['main'] + + buffer_stdout, buffer_stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(buffer_stdout), contextlib.redirect_stderr(buffer_stderr): + return_code = stm32pio.app.main(sys_argv=['-v', 'new', '-d', str(FIXTURE_PATH), '-b', TEST_PROJECT_BOARD]) + + self.assertEqual(return_code, 0, msg="Non-zero return code") + # stderr and not stdout contains the actual output (by default for the logging module) + self.assertEqual(len(buffer_stdout.getvalue()), 0, + msg="Process has printed something directly into STDOUT bypassing logging") + self.assertIn('DEBUG', buffer_stderr.getvalue(), msg="Verbose logging output hasn't been enabled on stderr") + + # Inject all methods' names in the regex. Inject the width of field in a log format string + regex = re.compile("^(?=(DEBUG) {0,4})(?=.{8} (?=(" + '|'.join(methods) + ") {0," + + str(stm32pio.settings.log_fieldwidth_function) + "})(?=.{" + + str(stm32pio.settings.log_fieldwidth_function) + "} [^ ]))", flags=re.MULTILINE) + self.assertGreaterEqual(len(re.findall(regex, buffer_stderr.getvalue())), 1, + msg="Logs messages doesn't match the format") + + self.assertIn('Starting STM32CubeMX', buffer_stderr.getvalue(), msg="STM32CubeMX has not printed its logs") + + def test_non_verbose(self): + """ + Run as subprocess to capture the full output. We should not see any 'DEBUG' logging messages or STM32CubeMX CLI + output. Logs format should match such a regex: + + ^(?=(INFO) {0,4})(?=.{8} ((?!( |build|pio_init|...)))) + """ + + # inspect.getmembers is great but it triggers class properties leading to the unacceptable code execution + methods = dir(stm32pio.lib.Stm32pio) + ['main'] + + buffer_stdout, buffer_stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(buffer_stdout), contextlib.redirect_stderr(buffer_stderr): + return_code = stm32pio.app.main(sys_argv=['generate', '-d', str(FIXTURE_PATH)]) + + self.assertEqual(return_code, 0, msg="Non-zero return code") + # stderr and not stdout contains the actual output (by default for the logging module) + self.assertNotIn('DEBUG', buffer_stderr.getvalue(), msg="Verbose logging output has been enabled on stderr") + self.assertEqual(len(buffer_stdout.getvalue()), 0, msg="All app output should flow through the logging module") + + regex = re.compile("^(?=(INFO) {0,4})(?=.{8} ((?!( |" + '|'.join(methods) + "))))", flags=re.MULTILINE) + self.assertGreaterEqual(len(re.findall(regex, buffer_stderr.getvalue())), 1, + msg="Logs messages doesn't match the format") + + self.assertNotIn('Starting STM32CubeMX', buffer_stderr.getvalue(), msg="STM32CubeMX has printed its logs") + + def test_init(self): + """ + Check for config creation and parameters presence + """ + result = subprocess.run([PYTHON_EXEC, STM32PIO_MAIN_SCRIPT, 'init', '-d', str(FIXTURE_PATH), + '-b', TEST_PROJECT_BOARD], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.assertEqual(result.returncode, 0, msg="Non-zero return code") + + self.assertTrue(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).is_file(), + msg=f"{stm32pio.settings.config_file_name} file hasn't been created") + + config = configparser.ConfigParser(interpolation=None) + config.read(str(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name))) + for section, parameters in stm32pio.settings.config_default.items(): + for option, value in parameters.items(): + with self.subTest(section=section, option=option, msg="Section/key is not found in saved config file"): + self.assertIsNotNone(config.get(section, option, fallback=None)) + self.assertEqual(config.get('project', 'board', fallback="Not found"), TEST_PROJECT_BOARD, + msg="'board' has not been set") + + def test_status(self): + """ + Test the output returning by the app on a request to the 'status' command + """ + + buffer_stdout = io.StringIO() + with contextlib.redirect_stdout(buffer_stdout), contextlib.redirect_stderr(None): + return_code = stm32pio.app.main(sys_argv=['status', '-d', str(FIXTURE_PATH)]) + + self.assertEqual(return_code, 0, msg="Non-zero return code") + + matches_counter = 0 + last_stage_pos = -1 + for stage in stm32pio.lib.ProjectStage: + if stage != stm32pio.lib.ProjectStage.UNDEFINED: + match = re.search(r"^((\[ \])|(\[\*\])) {2}" + str(stage) + '$', buffer_stdout.getvalue(), re.MULTILINE) + self.assertTrue(match, msg="Status information was not found on STDOUT") + if match: + matches_counter += 1 + self.assertGreater(match.start(), last_stage_pos, msg="The order of stages is messed up") + last_stage_pos = match.start() + + self.assertEqual(matches_counter, len(stm32pio.lib.ProjectStage) - 1) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..3e9ceb2 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,151 @@ +import configparser +import inspect +import shutil + +import stm32pio.lib +import stm32pio.settings + +from tests.test import * + + +class TestIntegration(CustomTestCase): + """ + Sequence of methods that should work seamlessly + """ + + def test_rebase_project(self): + """ + Test the portability of projects: they should stay totally valid after moving to another path (same as renaming + the parent part of the path). If we will not meet any exceptions, we should consider the test passed. + """ + project_before = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + project_before.save_config() + + new_path = f'{project_before.path}-moved' + shutil.move(str(project_before.path), new_path) + + project_after = stm32pio.lib.Stm32pio(new_path, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + project_after.generate_code() + project_after.pio_init() + project_after.patch() + project_after.build() + + def test_config_priorities(self): + """ + Test the compliance with priorities when reading the parameters + """ + # Sample user's custom patch value + config_parameter_user_value = inspect.cleandoc(''' + [test_section] + key1 = value1 + key2 = 789 + ''') + cli_parameter_user_value = 'nucleo_f429zi' + + # Create test config + config = configparser.ConfigParser(interpolation=None) + config.read_dict({ + 'project': { + 'platformio_ini_patch_content': config_parameter_user_value, + 'board': TEST_PROJECT_BOARD + } + }) + # ... save it + with FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).open(mode='w') as config_file: + config.write(config_file) + + # On project creation we should interpret the CLI-provided values as superseding to the saved ones and + # saved ones, in turn, as superseding to the default ones + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': cli_parameter_user_value}, + save_on_destruction=False) + project.pio_init() + project.patch() + + # Actually, we can parse platformio.ini via configparser but this is simpler in our case + after_patch_content = FIXTURE_PATH.joinpath('platformio.ini').read_text() + self.assertIn(config_parameter_user_value, after_patch_content, + msg="User config parameter has not been prioritized over the default one") + self.assertIn(cli_parameter_user_value, after_patch_content, + msg="User CLI parameter has not been prioritized over the saved one") + + def test_build(self): + """ + Initialize a new project and try to build it + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + project.generate_code() + project.pio_init() + project.patch() + + result = project.build() + + self.assertEqual(result, 0, msg="Build failed") + + def test_regenerate_code(self): + """ + Simulate a new project creation, its changing and CubeMX code re-generation (for example, after adding new + hardware features and some new files by a user) + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + + # Generate a new project ... + project.generate_code() + project.pio_init() + project.patch() + + # ... change it: + test_file_1 = FIXTURE_PATH.joinpath('Src', 'main.c') + test_content_1 = "*** TEST STRING 1 ***\n" + test_file_2 = FIXTURE_PATH.joinpath('Inc', 'my_header.h') + test_content_2 = "*** TEST STRING 2 ***\n" + # - add some sample string inside CubeMX' /* BEGIN - END */ block + main_c_content = test_file_1.read_text() + pos = main_c_content.index("while (1)") + main_c_new_content = main_c_content[:pos] + test_content_1 + main_c_content[pos:] + test_file_1.write_text(main_c_new_content) + # - add new file inside the project + test_file_2.write_text(test_content_2) + + # Re-generate CubeMX project + project.generate_code() + + # Check if added information has been preserved + for test_content, after_regenerate_content in [(test_content_1, test_file_1.read_text()), + (test_content_2, test_file_2.read_text())]: + with self.subTest(msg=f"User content hasn't been preserved in {after_regenerate_content}"): + self.assertIn(test_content, after_regenerate_content) + + def test_current_stage(self): + """ + Go through the sequence of states emulating the real-life project lifecycle + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.EMPTY) + + project.save_config() + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.INITIALIZED) + + project.generate_code() + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.GENERATED) + + project.pio_init() + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.PIO_INITIALIZED) + + project.patch() + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.PATCHED) + + project.build() + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.BUILT) + + project.clean() + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.EMPTY) + + # Should be UNDEFINED when the project is messed up + project.pio_init() + self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.UNDEFINED) + self.assertFalse(project.state.is_consistent) diff --git a/tests/test_unit.py b/tests/test_unit.py new file mode 100644 index 0000000..d3a0a19 --- /dev/null +++ b/tests/test_unit.py @@ -0,0 +1,213 @@ +import configparser +import inspect +import pathlib +import platform +import subprocess +import time + +import stm32pio.lib +import stm32pio.settings +import stm32pio.util + +from tests.test import * + + +class TestUnit(CustomTestCase): + """ + Test the single method. As we at some point decided to use a class instead of the set of scattered functions we need + to do some preparations for almost every test (e.g. instantiate the class, create the PlatformIO project, etc.), + though, so the architecture now is way less modular + """ + + def test_generate_code(self): + """ + Check whether files and folders have been created (by STM32CubeMX) + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + project.generate_code() + + # Assuming that the presence of these files indicating a success + files_should_be_present = ['Src/main.c', 'Inc/main.h'] + for file in files_should_be_present: + with self.subTest(msg=f"{file} hasn't been created"): + self.assertEqual(FIXTURE_PATH.joinpath(file).is_file(), True) + + def test_pio_init(self): + """ + Consider that existence of 'platformio.ini' file showing a successful PlatformIO project initialization. The + last one has another traces that can be checked too but we are interested only in a 'platformio.ini' anyway. + Also, check that it is a correct configparser file and is not empty + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + result = project.pio_init() + + self.assertEqual(result, 0, msg="Non-zero return code") + self.assertTrue(FIXTURE_PATH.joinpath('platformio.ini').is_file(), msg="platformio.ini is not there") + + platformio_ini = configparser.ConfigParser(interpolation=None) + self.assertGreater(len(platformio_ini.read(str(FIXTURE_PATH.joinpath('platformio.ini')))), 0, + msg='platformio.ini is empty') + + def test_patch(self): + """ + Check that new parameters were added, modified were updated and existing parameters didn't gone. Also, check for + unnecessary folders deletion + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) + + test_content = inspect.cleandoc(''' + ; This is a test config .ini file + ; with a comment. It emulates a real + ; platformio.ini file + + [platformio] + include_dir = this s;789hould be replaced + ; there should appear a new parameter + test_key3 = this should be preserved + + [test_section] + test_key1 = test_value1 + test_key2 = 123 + ''') + '\n' + FIXTURE_PATH.joinpath('platformio.ini').write_text(test_content) + FIXTURE_PATH.joinpath('include').mkdir() + + project.patch() + + with self.subTest(): + self.assertFalse(FIXTURE_PATH.joinpath('include').is_dir(), msg="'include' has not been deleted") + + original_test_config = configparser.ConfigParser(interpolation=None) + original_test_config.read_string(test_content) + + patched_config = configparser.ConfigParser(interpolation=None) + patch_config = configparser.ConfigParser(interpolation=None) + patch_config.read_string(project.config.get('project', 'platformio_ini_patch_content')) + + self.assertGreater(len(patched_config.read(FIXTURE_PATH.joinpath('platformio.ini'))), 0) + + for patch_section in patch_config.sections(): + self.assertTrue(patched_config.has_section(patch_section), msg=f"{patch_section} is missing") + for patch_key, patch_value in patch_config.items(patch_section): + self.assertEqual(patched_config.get(patch_section, patch_key, fallback=None), patch_value, + msg=f"{patch_section}: {patch_key}={patch_value} is missing or incorrect in the " + "patched config") + + for original_section in original_test_config.sections(): + self.assertTrue(patched_config.has_section(original_section), + msg=f"{original_section} from the original config is missing") + for original_key, original_value in original_test_config.items(original_section): + # We've already checked patch parameters so skip them + if not patch_config.has_option(original_section, original_key): + self.assertEqual(patched_config.get(original_section, original_key), original_value, + msg=f"{original_section}: {original_key}={original_value} is corrupted") + + def test_build_should_handle_error(self): + """ + Build an empty project so PlatformIO should return an error + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + project.pio_init() + + with self.assertLogs(level='ERROR') as logs: + self.assertNotEqual(project.build(), 0, msg="Build error was not indicated") + # next() - Technique to find something in array, string, etc. (or to indicate that there is no) + self.assertTrue(next((True for item in logs.output if "PlatformIO build error" in item), False), + msg="Error message does not match") + + def test_start_editor(self): + """ + Call the editors + """ + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) + + editors = { + 'atom': { + 'Windows': 'atom.exe', + 'Darwin': 'Atom', + 'Linux': 'atom' + }, + 'code': { + 'Windows': 'Code.exe', + 'Darwin': 'Visual Studio Code', + 'Linux': 'code' + }, + 'subl': { + 'Windows': 'sublime_text.exe', + 'Darwin': 'Sublime', + 'Linux': 'sublime' + } + } + + for editor, editor_process_names in editors.items(): + # Look for the command presence in the system so we test only installed editors + if platform.system() == 'Windows': + command_str = f"where {editor} /q" + else: + command_str = f"command -v {editor}" + editor_exists = False + if subprocess.run(command_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode == 0: + editor_exists = True + + if editor_exists: + with self.subTest(command=editor, name=editor_process_names[platform.system()]): + project.start_editor(editor) + + time.sleep(1) # wait a little bit for app to start + + if platform.system() == 'Windows': + command_arr = ['wmic', 'process', 'get', 'description'] + else: + command_arr = ['ps', '-A'] + # "encoding='utf-8'" is for "a bytes-like object is required, not 'str'" in "assertIn" + result = subprocess.run(command_arr, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + encoding='utf-8') + # Or, for Python 3.7 and above: + # result = subprocess.run(command_arr, capture_output=True, encoding='utf-8') + self.assertIn(editor_process_names[platform.system()], result.stdout) + + def test_init_path_not_found_should_raise(self): + """ + Pass non-existing path and expect the error + """ + path_does_not_exist_name = 'does_not_exist' + + path_does_not_exist = FIXTURE_PATH.joinpath(path_does_not_exist_name) + with self.assertRaisesRegex(FileNotFoundError, path_does_not_exist_name, + msg="FileNotFoundError was not raised or doesn't contain a description"): + stm32pio.lib.Stm32pio(path_does_not_exist, save_on_destruction=False) + + def test_save_config(self): + """ + Explicitly save the config to file and look did that actually happen and whether all the information was + preserved + """ + # 'board' is non-default, 'project'-section parameter + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, + save_on_destruction=False) + project.save_config() + + self.assertTrue(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).is_file(), + msg=f"{stm32pio.settings.config_file_name} file hasn't been created") + + config = configparser.ConfigParser(interpolation=None) + self.assertGreater(len(config.read(str(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name)))), 0, + msg="Config is empty") + for section, parameters in stm32pio.settings.config_default.items(): + for option, value in parameters.items(): + with self.subTest(section=section, option=option, + msg="Section/key is not found in the saved config file"): + self.assertNotEqual(config.get(section, option, fallback="Not found"), "Not found") + + self.assertEqual(config.get('project', 'board', fallback="Not found"), TEST_PROJECT_BOARD, + msg="'board' has not been set") + + def test_get_platformio_boards(self): + """ + PlatformIO identifiers of boards are requested using PlatformIO Python API (not sure it can be called public, + though...) + """ + self.assertIsInstance(stm32pio.util.get_platformio_boards(), list) From f94425c1a47727174e777afca0e2cd383c52261d Mon Sep 17 00:00:00 2001 From: ussserrr Date: Mon, 9 Mar 2020 23:43:17 +0300 Subject: [PATCH 3/7] notify the user that the test project tree is not clean. Analyze CubeMX output to catch errors --- TODO.md | 9 ++++++--- stm32pio/lib.py | 23 ++++++++++++++++------- stm32pio/settings.py | 4 +++- stm32pio/util.py | 21 ++++++++++++++++----- tests/test.py | 17 +++++++++++------ tests/test_cli.py | 9 +++------ 6 files changed, 55 insertions(+), 28 deletions(-) diff --git a/TODO.md b/TODO.md index cadad04..cca5561 100644 --- a/TODO.md +++ b/TODO.md @@ -27,14 +27,17 @@ - [ ] `__init__`' `parameters` dict argument schema (Python 3.8 feature). Also, maybe move `save_on_desctruction` parameter there. Maybe separate on `project_params` and `instance_opts` - [ ] General algo of merging a given dict of parameters with the saved one on project initialization - [x] parse `platformio.ini` to check its correctness in state getter - - [ ] CubeMX 0 return code doesn't necessarily means the correct generation (e.g. migration dialog has appeared and 'Cancel' was chosen, or CubeMX_version < ioc_file_version), probably should somehow analyze the output (logs can be parsed. i.e. 2020-03-05 12:08:40,765 \[ERROR\] MainProjectManager:806 - Program Manager : The version of the current IOC is too high.) + - [x] CubeMX 0 return code doesn't necessarily means the correct generation (e.g. migration dialog has appeared and 'Cancel' was chosen, or CubeMX_version < ioc_file_version), probably should somehow analyze the output (logs can be parsed. i.e. 2020-03-05 12:08:40,765 \[ERROR\] MainProjectManager:806 - Program Manager : The version of the current IOC is too high.) - [x] Dispatch tests on several files (too many code actually) - [x] Do not store absolute paths in config file and make a project portable (use configparser parameters interpolation). Handle renaming - - [ ] See https://docs.python.org/3/howto/logging-cookbook.html#context-info to maybe replace current scheme + - [ ] See https://docs.python.org/3/howto/logging-cookbook.html#context-info to maybe remade current logging schema - [ ] UML diagrams (core, GUI back- and front-ends) - [ ] CI is possible (Arch's AUR has the STM32CubeMX package, also there is a direct link). Deploy Docker in Azure Pipelines, basic at Travis CI - [ ] Test preserving user files and folders on regeneration and mb other operations - [ ] Move special formatters inside the library. It is an implementation detail actually that we use subprocesses and so on - - [ ] Mb clean the test project tree before running the tests + - [x] Mb clean the test project tree before running the tests - [x] README table of contents - [ ] Mb store the last occurred exception traceback in .ini file and show on some CLI command (so we don't need to turn on verbose mode) + - [ ] 'verbose' and 'non-verbose' tests as `subTest` (also 'should_log_error_...') + - [ ] turn off all possible dialogs in STM32CubeMX + - [ ] invoke PlatformIO as a library (get rid of subprocess for pio entirely) diff --git a/stm32pio/lib.py b/stm32pio/lib.py index f1c171c..4687236 100644 --- a/stm32pio/lib.py +++ b/stm32pio/lib.py @@ -352,7 +352,7 @@ def generate_code(self) -> int: # more details) cubemx_script_file, cubemx_script_name = tempfile.mkstemp() - # We should necessarily remove the temp directory, so do not let any exception break our plans + # We must remove the temp directory, so do not let any exception break our plans try: # buffering=0 leads to the immediate flushing on writing with open(cubemx_script_file, mode='w+b', buffering=0) as cubemx_script: @@ -367,20 +367,29 @@ def generate_code(self) -> int: command_arr = [self.config.get('app', 'java_cmd'), '-jar', self.config.get('app', 'cubemx_cmd'), '-q', cubemx_script_name, '-s'] # -q: read the commands from the file, -s: silent performance # Redirect the output of the subprocess into the logging module (with DEBUG level) - with stm32pio.util.LogPipe(self.logger, logging.DEBUG) as log_pipe: - result = subprocess.run(command_arr, stdout=log_pipe, stderr=log_pipe) + with stm32pio.util.LogPipe(self.logger, logging.DEBUG) as log: + result = subprocess.run(command_arr, stdout=log.pipe, stderr=log.pipe) + result_output = log.value + except Exception as e: raise e # re-raise an exception after the 'finally' block finally: pathlib.Path(cubemx_script_name).unlink() + error_msg = "code generation error" if result.returncode == 0: + # CubeMX 0 return code doesn't necessarily means the correct generation (e.g. migration dialog has appeared + # and 'Cancel' was chosen, or CubeMX_version < ioc_file_version), should analyze the output + error_lines = [line for line in result_output.splitlines() if '[ERROR]' in line] + if len(error_lines): + self.logger.error('\n'.join(error_lines)) + raise Exception(error_msg) self.logger.info("successful code generation") return result.returncode else: self.logger.error(f"code generation error (CubeMX return code is {result.returncode}).\n" "Enable a verbose output or try to generate a code from the CubeMX itself.") - raise Exception("code generation error") + raise Exception(error_msg) def pio_init(self) -> int: @@ -411,7 +420,7 @@ def pio_init(self) -> int: # PlatformIO returns 0 even on some errors (e.g. no '--board' argument) if 'error' in result.stdout.lower(): self.logger.error(result.stdout) - raise Exception('\n' + error_msg) + raise Exception(error_msg) self.logger.debug(result.stdout, 'from_subprocess') self.logger.info("successful PlatformIO project initialization") return result.returncode @@ -567,8 +576,8 @@ def build(self) -> int: if not self.logger.isEnabledFor(logging.DEBUG): command_arr.append('--silent') - with stm32pio.util.LogPipe(self.logger, logging.DEBUG) as log_pipe: - result = subprocess.run(command_arr, stdout=log_pipe, stderr=log_pipe) + with stm32pio.util.LogPipe(self.logger, logging.DEBUG) as log: + result = subprocess.run(command_arr, stdout=log.pipe, stderr=log.pipe) if result.returncode == 0: self.logger.info("successful PlatformIO build") else: diff --git a/stm32pio/settings.py b/stm32pio/settings.py index 4d90dd9..06bcdba 100644 --- a/stm32pio/settings.py +++ b/stm32pio/settings.py @@ -50,4 +50,6 @@ config_file_name = 'stm32pio.ini' -log_fieldwidth_function = 26 # TODO: can be calculated actually (longest name +# Longest name (not necessarily method so a little bit tricky...) +# log_fieldwidth_function = max([len(member) for member in dir(stm32pio.lib.Stm32pio)]) + 1 +log_fieldwidth_function = 25 + 1 diff --git a/stm32pio/util.py b/stm32pio/util.py index cfe481c..36e2235 100644 --- a/stm32pio/util.py +++ b/stm32pio/util.py @@ -74,11 +74,20 @@ def format(self, record: logging.LogRecord) -> str: return super().format(record) +class LogPipeRC: + """ + Small class suitable for passing to the caller when the LogPipe context manager is invoked + """ + value = '' # string accumulating all incoming messages + def __init__(self, fd: int): + self.pipe = fd # writable half of os.pipe + class LogPipe(threading.Thread): """ The thread combined with a context manager to provide a nice way to temporarily redirect something's stream output into logging module. The most straightforward application is to suppress subprocess STDOUT and/or STDERR streams and - wrap them in the logging mechanism as it is for now for any other message in your app. + wrap them in the logging mechanism as it is now for any other message in your app. Also, store the incoming messages + in the string """ def __init__(self, logger: logging.Logger, level: int, *args, **kwargs): @@ -90,21 +99,23 @@ def __init__(self, logger: logging.Logger, level: int, *args, **kwargs): self.fd_read, self.fd_write = os.pipe() # create 2 ends of the pipe and setup the reading one self.pipe_reader = os.fdopen(self.fd_read) - def __enter__(self) -> int: + self.rc = LogPipeRC(self.fd_write) # "remote control" + + def __enter__(self) -> LogPipeRC: """ Activate the thread and return the consuming end of the pipe so the invoking code can use it to feed its messages from now on """ self.start() - return self.fd_write + return self.rc def run(self): """ Routine of the thread, logging everything """ - for line in iter(self.pipe_reader.readline, ''): + for line in iter(self.pipe_reader.readline, ''): # stops the iterator when empty string will occur + self.rc.value += line # accumulate the string self.logger.log(self.level, line.strip('\n'), 'from_subprocess') # mark the message origin - self.pipe_reader.close() def __exit__(self, exc_type, exc_val, exc_tb): diff --git a/tests/test.py b/tests/test.py index 2530610..589a16e 100755 --- a/tests/test.py +++ b/tests/test.py @@ -22,15 +22,15 @@ import stm32pio.app -# Absolute path to the main stm32pio script (make sure what repo we are testing -STM32PIO_MAIN_SCRIPT: str = inspect.getfile(stm32pio.app) -# Absolute path to the Python executable (no need to guess whether it's 'python' or 'python3' and so on) -PYTHON_EXEC: str = sys.executable - -# Test data TEST_PROJECT_PATH = pathlib.Path('stm32pio-test-project').resolve(strict=True) if not TEST_PROJECT_PATH.joinpath('stm32pio-test-project.ioc').is_file(): raise FileNotFoundError("No test project is present") + +# Gently ask a user running tests to remove all irrelevant files from the TEST_PROJECT_PATH +if len(list(TEST_PROJECT_PATH.iterdir())) > 1: + raise Warning(f"There are extrinsic files in the test project directory '{TEST_PROJECT_PATH}'. Please persist only " + "the .ioc file") + # Make sure you have F0 framework installed (try to run code generation from STM32CubeMX manually at least once before # proceeding) TEST_PROJECT_BOARD = 'nucleo_f031k6' @@ -40,6 +40,11 @@ TEMP_DIR = tempfile.TemporaryDirectory() FIXTURE_PATH = pathlib.Path(TEMP_DIR.name).joinpath(TEST_PROJECT_PATH.name) +# Absolute path to the main stm32pio script (make sure what repo we are testing) +STM32PIO_MAIN_SCRIPT: str = inspect.getfile(stm32pio.app) +# Absolute path to the Python executable (no need to guess whether it's 'python' or 'python3' and so on) +PYTHON_EXEC: str = sys.executable + print(f"The file of 'stm32pio.app' module: {STM32PIO_MAIN_SCRIPT}") print(f"Python executable: {PYTHON_EXEC} {sys.version}") print(f"Temp test fixture path: {FIXTURE_PATH}") diff --git a/tests/test_cli.py b/tests/test_cli.py index add29b0..6ad4620 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -87,11 +87,8 @@ def test_incorrect_path_should_log_error(self): with self.assertLogs(level='ERROR') as logs: return_code = stm32pio.app.main(sys_argv=['init', '-d', str(path_not_exist)]) self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') - # TODO: for macOS: - # ERROR [Errno 2] No such file or directory: '/Users/chufyrev/Documents/GitHub/stm32pio/path' - # probably another on different systems - # self.assertTrue(next((True for item in logs.output if str(path_not_exist) in item), False), - # msg="'ERROR' logging message hasn't been printed") + self.assertTrue(next((True for message in logs.output if "No such file or directory" in message), False), + msg="'ERROR' logging message hasn't been printed") def test_no_ioc_file_should_log_error(self): """ @@ -103,7 +100,7 @@ def test_no_ioc_file_should_log_error(self): with self.assertLogs(level='ERROR') as logs: return_code = stm32pio.app.main(sys_argv=['init', '-d', str(dir_with_no_ioc_file)]) self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') - self.assertTrue(next((True for item in logs.output if "CubeMX project .ioc file" in item), False), + self.assertTrue(next((True for message in logs.output if "CubeMX project .ioc file" in message), False), msg="'ERROR' logging message hasn't been printed") def test_verbose(self): From e5359f4884ee63f639ec5bc4f0ed6905b6c458cb Mon Sep 17 00:00:00 2001 From: ussserrr Date: Wed, 11 Mar 2020 19:29:59 +0300 Subject: [PATCH 4/7] revert back to PlatformIO CLI instead of Python import --- README.md | 1 - TODO.md | 5 ++--- setup.py | 3 --- stm32pio-gui/app.py | 2 +- stm32pio/app.py | 2 +- stm32pio/lib.py | 9 ++++++++- stm32pio/util.py | 17 ++++++++++------- tests/test_cli.py | 5 +++-- tests/test_unit.py | 5 ++--- 9 files changed, 27 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1b416df..2baa0bf 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ It uses STM32CubeMX to generate a HAL-framework-based code and alongside creates ## Requirements: - For this app: - Python 3.6 and above - - `platformio` - For usage: - macOS, Linux, Windows - STM32CubeMX with desired downloaded frameworks (F0, F1, etc.) diff --git a/TODO.md b/TODO.md index cca5561..abccea1 100644 --- a/TODO.md +++ b/TODO.md @@ -16,7 +16,6 @@ - [x] exclude tests from the bundle (see `setup.py` options) - [ ] generate code docs (help user to understand an internal mechanics, e.g. for embedding). Can be uploaded to the GitHub Wiki - [ ] colored logs, maybe... - - [ ] if we require `platformio` package as a dependency we probably can rely on its dependencies too - [ ] check logging work when embed stm32pio lib in third-party stuff (no logging setup at all) - [ ] merge subprocess pipes to one where suitable (i.e. `stdout` and `stderr`) - [ ] redirect subprocess pipes to `DEVNULL` where suitable to suppress output @@ -30,7 +29,7 @@ - [x] CubeMX 0 return code doesn't necessarily means the correct generation (e.g. migration dialog has appeared and 'Cancel' was chosen, or CubeMX_version < ioc_file_version), probably should somehow analyze the output (logs can be parsed. i.e. 2020-03-05 12:08:40,765 \[ERROR\] MainProjectManager:806 - Program Manager : The version of the current IOC is too high.) - [x] Dispatch tests on several files (too many code actually) - [x] Do not store absolute paths in config file and make a project portable (use configparser parameters interpolation). Handle renaming - - [ ] See https://docs.python.org/3/howto/logging-cookbook.html#context-info to maybe remade current logging schema + - [ ] See https://docs.python.org/3/howto/logging-cookbook.html#context-info to maybe remade current logging schema (current is, perhaps, a cause of the strange error while testing (in the logging thread)) - [ ] UML diagrams (core, GUI back- and front-ends) - [ ] CI is possible (Arch's AUR has the STM32CubeMX package, also there is a direct link). Deploy Docker in Azure Pipelines, basic at Travis CI - [ ] Test preserving user files and folders on regeneration and mb other operations @@ -40,4 +39,4 @@ - [ ] Mb store the last occurred exception traceback in .ini file and show on some CLI command (so we don't need to turn on verbose mode) - [ ] 'verbose' and 'non-verbose' tests as `subTest` (also 'should_log_error_...') - [ ] turn off all possible dialogs in STM32CubeMX - - [ ] invoke PlatformIO as a library (get rid of subprocess for pio entirely) + - [ ] test (at least manually) when the tools are not present diff --git a/setup.py b/setup.py index 1bc08a5..9018c8a 100644 --- a/setup.py +++ b/setup.py @@ -48,9 +48,6 @@ setup_requires=[ 'wheel' ], - install_requires=[ - 'platformio' - ], include_package_data=True, entry_points={ 'console_scripts': [ diff --git a/stm32pio-gui/app.py b/stm32pio-gui/app.py index 92e22c0..dd9f9db 100644 --- a/stm32pio-gui/app.py +++ b/stm32pio-gui/app.py @@ -483,7 +483,7 @@ def set(self, key, value): def loading(): global boards - boards = ['None'] + stm32pio.util.get_platformio_boards() + boards = ['None'] + stm32pio.util.get_platformio_boards('platformio') def on_loading(_, success): # TODO: somehow handle an initialization error diff --git a/stm32pio/app.py b/stm32pio/app.py index a87fc0e..721508e 100755 --- a/stm32pio/app.py +++ b/stm32pio/app.py @@ -165,5 +165,5 @@ def main(sys_argv=None) -> int: if __name__ == '__main__': - sys.path.insert(0, str(pathlib.Path(sys.path[0]).parent)) # hack to be able to run the app as 'python3 app.py' + # sys.path.append(str(pathlib.Path(sys.path[0]).parent)) # hack to be able to run the app as 'python3 app.py' sys.exit(main()) diff --git a/stm32pio/lib.py b/stm32pio/lib.py index 4687236..d2e507e 100644 --- a/stm32pio/lib.py +++ b/stm32pio/lib.py @@ -172,7 +172,13 @@ def __init__(self, dirty_path: str, parameters: dict = None, save_on_destruction # General rule: given parameter takes precedence over the saved one board = '' if 'board' in parameters and parameters['board'] is not None: - if parameters['board'] in stm32pio.util.get_platformio_boards(): + try: + boards = stm32pio.util.get_platformio_boards(self.config.get('app', 'platformio_cmd')) + except Exception as e: + self.logger.warning(f"There was an error while obtaining possible PlatformIO boards: {e}", + exc_info=self.logger.isEnabledFor(logging.DEBUG)) + boards = [] + if parameters['board'] in boards: board = parameters['board'] else: self.logger.warning(f"'{parameters['board']}' was not found in PlatformIO. " @@ -578,6 +584,7 @@ def build(self) -> int: with stm32pio.util.LogPipe(self.logger, logging.DEBUG) as log: result = subprocess.run(command_arr, stdout=log.pipe, stderr=log.pipe) + if result.returncode == 0: self.logger.info("successful PlatformIO build") else: diff --git a/stm32pio/util.py b/stm32pio/util.py index 36e2235..38118b1 100644 --- a/stm32pio/util.py +++ b/stm32pio/util.py @@ -2,13 +2,13 @@ Some auxiliary entities not falling into other categories """ +import json import logging import os +import subprocess import threading from typing import List -from platformio.managers.platform import PlatformManager - module_logger = logging.getLogger(__name__) @@ -127,14 +127,17 @@ def __exit__(self, exc_type, exc_val, exc_tb): -def get_platformio_boards() -> List[str]: +def get_platformio_boards(platformio_cmd) -> List[str]: """ - Use PlatformIO Python sources to obtain the boards list. As we interested only in STM32 ones, cut off all the - others. + Obtain the PlatformIO boards list. As we interested only in STM32 ones, cut off all the others. IMPORTANT NOTE: The inner implementation can go to the Internet from time to time when it decides that its cache is out of date. So it can take a long time to execute. """ - pm = PlatformManager() - return [board['id'] for board in pm.get_all_boards() if 'stm32cube' in board['frameworks']] + # Windows 7, as usual, correctly works only with shell=True... + result = subprocess.run(f"{platformio_cmd} boards --json-output stm32cube", + encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True) + + boards = json.loads(result.stdout) + return [board['id'] for board in boards] diff --git a/tests/test_cli.py b/tests/test_cli.py index 6ad4620..cf55357 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -82,12 +82,13 @@ def test_incorrect_path_should_log_error(self): """ We should see an error log message and non-zero return code """ - path_not_exist = pathlib.Path('path/does/not/exist') + path_not_exist = pathlib.Path('path_some_uniq_name/does/not/exist') with self.assertLogs(level='ERROR') as logs: return_code = stm32pio.app.main(sys_argv=['init', '-d', str(path_not_exist)]) self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') - self.assertTrue(next((True for message in logs.output if "No such file or directory" in message), False), + # Actual text may vary and depends on OS and system language + self.assertTrue(next((True for message in logs.output if 'path_some_uniq_name' in message.lower()), False), msg="'ERROR' logging message hasn't been printed") def test_no_ioc_file_should_log_error(self): diff --git a/tests/test_unit.py b/tests/test_unit.py index d3a0a19..0a03d57 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -207,7 +207,6 @@ def test_save_config(self): def test_get_platformio_boards(self): """ - PlatformIO identifiers of boards are requested using PlatformIO Python API (not sure it can be called public, - though...) + PlatformIO identifiers of boards are requested using PlatformIO CLI in JSON format """ - self.assertIsInstance(stm32pio.util.get_platformio_boards(), list) + self.assertIsInstance(stm32pio.util.get_platformio_boards(platformio_cmd='platformio'), list) From bc8291b134e5cb2466bde9fac05b760dcf017120 Mon Sep 17 00:00:00 2001 From: ussserrr Date: Sat, 14 Mar 2020 21:01:14 +0300 Subject: [PATCH 5/7] new mechanics for project parameters and instance options that are passed in the project constructor --- TODO.md | 16 ++++--- stm32pio-gui/app.py | 24 ++++++++--- stm32pio/app.py | 35 +++++++-------- stm32pio/lib.py | 89 +++++++++++++++++++++------------------ stm32pio/settings.py | 6 ++- tests/test_cli.py | 26 +++++++----- tests/test_integration.py | 25 +++++------ tests/test_unit.py | 26 ++++++------ 8 files changed, 140 insertions(+), 107 deletions(-) diff --git a/TODO.md b/TODO.md index abccea1..277b4e5 100644 --- a/TODO.md +++ b/TODO.md @@ -7,12 +7,14 @@ - [ ] GUI. Reduce number of calls to 'state' (many IO operations) - [ ] GUI. Drag and drop the new folder into the app window - [ ] GUI. Implement some other methods for Qt abstract models - - [ ] VSCode plugin + - [ ] GUI. Warning on 'Clean' action + - [ ] GUI. On 'Clean' clean the log too + - [ ] Create VSCode plugin - [x] Remove casts to string where we can use path-like objects (related to Python version as new ones receive path-like objects arguments) - [ ] We look for some snippets of strings in logs and output for the testing code but we hard-code them and this is not good, probably - - [ ] Store an initial folder content in .ini config and ignore it on clean-up process. Allow the user to modify such list + - [ ] Store a folder initial content in .ini config and ignore it on clean-up process. Allow the user to modify such list (i.e. list of exclusion) - [x] Ask the confirmation of a user by-defualt for `clean` and add additional option for quiet performance - - [ ] check for all tools (CubeMX, ...) to be present in the system (both CLI and GUI) + - [ ] at some point check for all tools (CubeMX, ...) to be present in the system (both CLI and GUI) (global `--check` command (as `--version`)) - [x] exclude tests from the bundle (see `setup.py` options) - [ ] generate code docs (help user to understand an internal mechanics, e.g. for embedding). Can be uploaded to the GitHub Wiki - [ ] colored logs, maybe... @@ -23,8 +25,9 @@ - [x] CHANGELOG markdown markup - [ ] Two words about a synchronous nature of the lib and user's responsibility of async wrapping (if needed). Also, maybe migrate to async/await approach in the future - [x] `shlex` for `start_editor` command option sanitizing - - [ ] `__init__`' `parameters` dict argument schema (Python 3.8 feature). Also, maybe move `save_on_desctruction` parameter there. Maybe separate on `project_params` and `instance_opts` - - [ ] General algo of merging a given dict of parameters with the saved one on project initialization + - [ ] `__init__`' `parameters` dict argument schema (Python 3.8 feature). + - [x] Maybe separate on `project_params` and `instance_opts` + - [x] General algo of merging a given dict of parameters with the saved one on project initialization - [x] parse `platformio.ini` to check its correctness in state getter - [x] CubeMX 0 return code doesn't necessarily means the correct generation (e.g. migration dialog has appeared and 'Cancel' was chosen, or CubeMX_version < ioc_file_version), probably should somehow analyze the output (logs can be parsed. i.e. 2020-03-05 12:08:40,765 \[ERROR\] MainProjectManager:806 - Program Manager : The version of the current IOC is too high.) - [x] Dispatch tests on several files (too many code actually) @@ -36,7 +39,8 @@ - [ ] Move special formatters inside the library. It is an implementation detail actually that we use subprocesses and so on - [x] Mb clean the test project tree before running the tests - [x] README table of contents - - [ ] Mb store the last occurred exception traceback in .ini file and show on some CLI command (so we don't need to turn on verbose mode) + - [ ] Mb store the last occurred exception traceback in .ini file and show on some CLI command (so we don't necessarily need to turn on the verbose mode). And, in general, we should show the error reason right off - [ ] 'verbose' and 'non-verbose' tests as `subTest` (also 'should_log_error_...') - [ ] turn off all possible dialogs in STM32CubeMX - [ ] test (at least manually) when the tools are not present + - [ ] minimal example of the lib usage for the README diff --git a/stm32pio-gui/app.py b/stm32pio-gui/app.py index dd9f9db..7f3b875 100644 --- a/stm32pio-gui/app.py +++ b/stm32pio-gui/app.py @@ -107,6 +107,11 @@ class ProjectListItem(QObject): def __init__(self, project_args: list = None, project_kwargs: dict = None, parent: QObject = None): super().__init__(parent=parent) + if project_args is None: + project_args = [] + if project_kwargs is None: + project_kwargs = {} + self.logger = logging.getLogger(f"{stm32pio.lib.__name__}.{id(self)}") self.logger.setLevel(logging.DEBUG if settings.get('verbose') else logging.INFO) self.logging_worker = LoggingWorker(self.logger) @@ -128,8 +133,12 @@ def __init__(self, project_args: list = None, project_kwargs: dict = None, paren self._finalizer = weakref.finalize(self, self.at_exit) # register some kind of deconstruction handler if project_args is not None: - if 'logger' not in project_kwargs: - project_kwargs['logger'] = self.logger + if 'instance_options' not in project_kwargs: + project_kwargs['instance_options'] = { + 'logger': self.logger + } + elif 'logger' not in project_kwargs['instance_options']: + project_kwargs['instance_options']['logger'] = self.logger # Start the Stm32pio part initialization right after. It can take some time so we schedule it in a dedicated # thread @@ -309,7 +318,7 @@ def addProjectByPath(self, path: QUrl): path: QUrl path to the project folder (absolute by default) """ self.beginInsertRows(QModelIndex(), self.rowCount(), self.rowCount()) - project = ProjectListItem(project_args=[path.toLocalFile()], project_kwargs=dict(save_on_destruction=False), parent=self) + project = ProjectListItem(project_args=[path.toLocalFile()], project_kwargs=dict(instance_options={'save_on_destruction': False}), parent=self) self.projects.append(project) settings.beginGroup('app') @@ -488,8 +497,13 @@ def loading(): def on_loading(_, success): # TODO: somehow handle an initialization error boards_model.setStringList(boards) - projects = [ProjectListItem(project_args=[path], project_kwargs=dict(save_on_destruction=False), parent=projects_model) - for path in projects_paths] + projects = [ProjectListItem( + project_args=[path], + project_kwargs=dict( + instance_options={'save_on_destruction': False} + ), + parent=projects_model + ) for path in projects_paths] for p in projects: projects_model.addProject(p) main_window.backendLoaded.emit() # inform the GUI diff --git a/stm32pio/app.py b/stm32pio/app.py index 721508e..b2a1597 100755 --- a/stm32pio/app.py +++ b/stm32pio/app.py @@ -7,6 +7,7 @@ import logging import pathlib import sys +import traceback from typing import Optional @@ -27,7 +28,7 @@ def parse_args(args: list) -> Optional[argparse.Namespace]: "and other tools (if defaults doesn't work for you)") # Global arguments (there is also an automatically added '-h, --help' option) parser.add_argument('--version', action='version', version=f"stm32pio v{__version__}") - parser.add_argument('-v', '--verbose', help="enable verbose output (default: INFO)", action='count', required=False) + parser.add_argument('-v', '--verbose', help="enable verbose output (default: INFO)", action='count') subparsers = parser.add_subparsers(dest='subcommand', title='subcommands', description="valid subcommands", help="modes of operation") @@ -42,17 +43,17 @@ def parse_args(args: list) -> Optional[argparse.Namespace]: # Common subparsers options for p in [parser_init, parser_new, parser_generate, parser_status, parser_clean]: - p.add_argument('-d', '--directory', dest='project_path', default=pathlib.Path.cwd(), required=False, + p.add_argument('-d', '--directory', dest='project_path', default=pathlib.Path.cwd(), help="path to the project (current directory, if not given)") for p in [parser_init, parser_new]: - p.add_argument('-b', '--board', dest='board', required=False, help="PlatformIO name of the board") + p.add_argument('-b', '--board', dest='board', default='', help="PlatformIO name of the board") for p in [parser_init, parser_new, parser_generate]: - p.add_argument('--start-editor', dest='editor', required=False, + p.add_argument('--start-editor', dest='editor', help="use specified editor to open the PlatformIO project (e.g. subl, code, atom, etc.)") for p in [parser_new, parser_generate]: - p.add_argument('--with-build', action='store_true', required=False, help="build the project after generation") + p.add_argument('--with-build', action='store_true', help="build the project after generation") - parser_clean.add_argument('-q', '--quiet', action='store_true', required=False, + parser_clean.add_argument('-q', '--quiet', action='store_true', help="suppress the caution about the content removal") if len(args) == 0: @@ -101,15 +102,13 @@ def main(sys_argv=None) -> int: handler.setFormatter(stm32pio.util.DispatchingFormatter("%(levelname)-8s %(message)s", special=stm32pio.util.special_formatters)) else: - logger.setLevel(logging.INFO) - handler.setFormatter(logging.Formatter("%(message)s")) - logger.info("\nNo arguments were given, exiting...") + print("\nNo arguments were given, exiting...") return 0 # Main routine try: if args.subcommand == 'init': - project = stm32pio.lib.Stm32pio(args.project_path, parameters={'board': args.board}) + project = stm32pio.lib.Stm32pio(args.project_path, parameters={'project': {'board': args.board}}) if not args.board: logger.warning("STM32 PlatformIO board is not specified, it will be needed on PlatformIO project " "creation") @@ -118,7 +117,7 @@ def main(sys_argv=None) -> int: project.start_editor(args.editor) elif args.subcommand == 'new': - project = stm32pio.lib.Stm32pio(args.project_path, parameters={'board': args.board}) + project = stm32pio.lib.Stm32pio(args.project_path, parameters={'project': {'board': args.board}}) if project.config.get('project', 'board') == '': raise Exception("STM32 PlatformIO board is not specified, it is needed for PlatformIO project creation") project.generate_code() @@ -130,7 +129,7 @@ def main(sys_argv=None) -> int: project.start_editor(args.editor) elif args.subcommand == 'generate': - project = stm32pio.lib.Stm32pio(args.project_path, save_on_destruction=False) + project = stm32pio.lib.Stm32pio(args.project_path, instance_options={'save_on_destruction': False}) project.generate_code() if args.with_build: project.build() @@ -138,11 +137,11 @@ def main(sys_argv=None) -> int: project.start_editor(args.editor) elif args.subcommand == 'status': - project = stm32pio.lib.Stm32pio(args.project_path, save_on_destruction=False) + project = stm32pio.lib.Stm32pio(args.project_path, instance_options={'save_on_destruction': False}) print(project.state) elif args.subcommand == 'clean': - project = stm32pio.lib.Stm32pio(args.project_path, save_on_destruction=False) + project = stm32pio.lib.Stm32pio(args.project_path, instance_options={'save_on_destruction': False}) if args.quiet: project.clean() else: @@ -157,13 +156,15 @@ def main(sys_argv=None) -> int: break # Library is designed to throw the exception in bad cases so we catch here globally - except Exception as e: - logger.exception(e, exc_info=logger.isEnabledFor(logging.DEBUG)) + except Exception: + # ExceptionName: message + logger.exception(traceback.format_exception_only(*(sys.exc_info()[:2]))[-1], + exc_info=logger.isEnabledFor(logging.DEBUG)) return -1 return 0 if __name__ == '__main__': - # sys.path.append(str(pathlib.Path(sys.path[0]).parent)) # hack to be able to run the app as 'python3 app.py' + sys.path.append(str(pathlib.Path(sys.path[0]).parent)) # hack to be able to run the app as 'python3 app.py' sys.exit(main()) diff --git a/stm32pio/lib.py b/stm32pio/lib.py index d2e507e..f519755 100644 --- a/stm32pio/lib.py +++ b/stm32pio/lib.py @@ -140,23 +140,30 @@ class Stm32pio: of the project directory except the main .ioc file. Args: - dirty_path (str): path to the project - parameters (dict): additional parameters to set on initialization stage - save_on_destruction (bool): register or not the finalizer that saves the config to file - logger (logging.Logger): if an external logger is given, it will be used, otherwise the new one will be created - (unique for every instance) + dirty_path (str): path to the project (required) + parameters (dict): additional parameters to set on initialization stage (format is same as for project' config + configparser.ConfigParser (see settings.py), values are merging) + instance_options (dict): some parameters, related more to the instance itself than to the project: + save_on_destruction (bool=True): register or not the finalizer that saves the config to file + logger (logging.Logger=None): if an external logger is given, it will be used, otherwise the new one will be created + (unique for every instance) """ - def __init__(self, dirty_path: str, parameters: dict = None, save_on_destruction: bool = True, - logger: logging.Logger = None): + def __init__(self, dirty_path: str, parameters: dict = None, instance_options: dict = None): if parameters is None: parameters = {} + if instance_options is None: # TODO: use Python 3.8 feature - dict schemas + instance_options = { + 'save_on_destruction': True, + 'logger': None + } + # The individual loggers for every single project allow to fine-tune the output when multiple projects are # created by the third-party code. - if logger is not None: - self.logger = logger + if 'logger' in instance_options and instance_options['logger'] is not None: + self.logger = instance_options['logger'] else: self.logger = logging.getLogger(f"{__name__}.{id(self)}") # use id() as uniqueness guarantee @@ -164,13 +171,11 @@ def __init__(self, dirty_path: str, parameters: dict = None, save_on_destruction # make the path absolute and check for existence self.path = pathlib.Path(dirty_path).expanduser().resolve(strict=True) - self.config = self._load_config() + self.config = self._load_config(parameters) self.ioc_file = self._find_ioc_file() self.config.set('project', 'ioc_file', self.ioc_file.name) - # General rule: given parameter takes precedence over the saved one - board = '' if 'board' in parameters and parameters['board'] is not None: try: boards = stm32pio.util.get_platformio_boards(self.config.get('app', 'platformio_cmd')) @@ -178,16 +183,11 @@ def __init__(self, dirty_path: str, parameters: dict = None, save_on_destruction self.logger.warning(f"There was an error while obtaining possible PlatformIO boards: {e}", exc_info=self.logger.isEnabledFor(logging.DEBUG)) boards = [] - if parameters['board'] in boards: - board = parameters['board'] - else: + if parameters['board'] not in boards: self.logger.warning(f"'{parameters['board']}' was not found in PlatformIO. " "Run 'platformio boards' for possible names") - self.config.set('project', 'board', board) - elif self.config.get('project', 'board', fallback=None) is None: - self.config.set('project', 'board', board) - if save_on_destruction: + if 'save_on_destruction' in instance_options and instance_options['save_on_destruction']: # Save the config on an instance destruction self._finalizer = weakref.finalize(self, self._save_config, self.config, self.path, self.logger) @@ -202,7 +202,7 @@ def state(self) -> ProjectState: Constructing and returning the current state of the project (tweaked dict, see ProjectState docs) """ - self.logger.debug(f"project content: {[item.name for item in self.path.iterdir()]}") + # self.logger.debug(f"project content: {[item.name for item in self.path.iterdir()]}") pio_is_initialized = False with contextlib.suppress(Exception): # we just want to know the information and don't care about details @@ -248,20 +248,16 @@ def _find_ioc_file(self) -> pathlib.Path: absolute path to the .ioc file """ - error_message = "not found: CubeMX project .ioc file" - ioc_file = self.config.get('project', 'ioc_file', fallback=None) if ioc_file: - ioc_file = self.path.joinpath(ioc_file) + ioc_file = self.path.joinpath(ioc_file).resolve(strict=True) self.logger.debug(f"using '{ioc_file.name}' file from the INI config") - if not ioc_file.is_file(): - raise FileNotFoundError(error_message) return ioc_file else: self.logger.debug("searching for any .ioc file...") candidates = list(self.path.glob('*.ioc')) if len(candidates) == 0: # TODO: good candidate for the new Python 3.8 assignment expression feature :) - raise FileNotFoundError(error_message) + raise FileNotFoundError("CubeMX project .ioc file") elif len(candidates) == 1: self.logger.debug(f"{candidates[0].name} is selected") return candidates[0] @@ -270,25 +266,33 @@ def _find_ioc_file(self) -> pathlib.Path: return candidates[0] - def _load_config(self) -> configparser.ConfigParser: + def _load_config(self, runtime_parameters: dict = None) -> configparser.ConfigParser: """ - Prepare ConfigParser config for the project. First, read the default config and then mask these values with user - ones. + Prepare ConfigParser config for the project. Order of getting values (masking) (higher levels overwrites lower): + + default dict (settings module) => config file stm32pio.ini => user-given (runtime) values + (via CLI or another way) Returns: new configparser.ConfigParser instance """ - self.logger.debug(f"searching for {stm32pio.settings.config_file_name}...") + if runtime_parameters is None: + runtime_parameters = {} config = configparser.ConfigParser(interpolation=None) - # Fill with default values + # Fill with default values ... config.read_dict(copy.deepcopy(stm32pio.settings.config_default)) - # Then override by user values (if exist) + + # ... then merge with user's config file values (if exist) ... + self.logger.debug(f"searching for {stm32pio.settings.config_file_name}...") if len(config.read(str(self.path.joinpath(stm32pio.settings.config_file_name)))) == 0: self.logger.debug(f"no or empty {stm32pio.settings.config_file_name} config file, will use the default one") + # ... finally merge with the given in this session CLI parameters + config.read_dict(runtime_parameters) + # Put away unnecessary processing as the string still will be formed even if the logging level doesn't allow # propagation of this message if self.logger.isEnabledFor(logging.DEBUG): @@ -317,7 +321,7 @@ def _save_config(config: configparser.ConfigParser, path: pathlib.Path, logger: try: with path.joinpath(stm32pio.settings.config_file_name).open(mode='w') as config_file: config.write(config_file) - logger.debug("stm32pio.ini config file has been saved") + logger.debug(f"{stm32pio.settings.config_file_name} config file has been saved") return 0 except Exception as e: logger.warning(f"cannot save the config: {e}", exc_info=logger.isEnabledFor(logging.DEBUG)) @@ -336,12 +340,13 @@ def save_config(self, parameters: dict = None) -> int: } Returns: - passes forward _save_config result + passes forward the _save_config() result """ - if parameters is not None: - for section_name, section_value in parameters.items(): - for key, value in section_value.items(): - self.config.set(section_name, key, value) + + if parameters is None: + parameters = {} + + self.config.read_dict(parameters) return self._save_config(self.config, self.path, self.logger) @@ -525,14 +530,14 @@ def patch(self) -> None: try: shutil.rmtree(self.path.joinpath('include')) self.logger.debug("'include' folder has been removed") - except: + except Exception: self.logger.info("cannot delete 'include' folder", exc_info=self.logger.isEnabledFor(logging.DEBUG)) # Remove 'src' directory too but on case-sensitive file systems 'Src' == 'src' == 'SRC' so we need to check if not self.path.joinpath('SRC').is_dir(): try: shutil.rmtree(self.path.joinpath('src')) self.logger.debug("'src' folder has been removed") - except: + except Exception: self.logger.info("cannot delete 'src' folder", exc_info=self.logger.isEnabledFor(logging.DEBUG)) self.logger.info("project has been patched") @@ -582,8 +587,8 @@ def build(self) -> int: if not self.logger.isEnabledFor(logging.DEBUG): command_arr.append('--silent') - with stm32pio.util.LogPipe(self.logger, logging.DEBUG) as log: - result = subprocess.run(command_arr, stdout=log.pipe, stderr=log.pipe) + with stm32pio.util.LogPipe(self.logger, logging.DEBUG if self.logger.isEnabledFor(logging.DEBUG) else logging.WARNING) as log: + result = subprocess.run(command_arr, stdout=log.pipe, stderr=log.pipe) # TODO: stderr is hidden if result.returncode == 0: self.logger.info("successful PlatformIO build") diff --git a/stm32pio/settings.py b/stm32pio/settings.py index 06bcdba..5ad21d3 100644 --- a/stm32pio/settings.py +++ b/stm32pio/settings.py @@ -44,7 +44,11 @@ [platformio] include_dir = Inc src_dir = Src - ''') + '\n' + ''') + '\n', + + # Runtime-determined values + 'board': '', + 'ioc_file': '' # required } ) diff --git a/tests/test_cli.py b/tests/test_cli.py index cf55357..d95a5cc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,6 +10,7 @@ import stm32pio.lib import stm32pio.settings +# Provides test constants from tests.test import * @@ -86,8 +87,8 @@ def test_incorrect_path_should_log_error(self): with self.assertLogs(level='ERROR') as logs: return_code = stm32pio.app.main(sys_argv=['init', '-d', str(path_not_exist)]) - self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') - # Actual text may vary and depends on OS and system language + self.assertNotEqual(return_code, 0, msg="Return code should be non-zero") + # Actual text may vary and depends on OS and system language so we check only for a part of path string self.assertTrue(next((True for message in logs.output if 'path_some_uniq_name' in message.lower()), False), msg="'ERROR' logging message hasn't been printed") @@ -100,19 +101,20 @@ def test_no_ioc_file_should_log_error(self): with self.assertLogs(level='ERROR') as logs: return_code = stm32pio.app.main(sys_argv=['init', '-d', str(dir_with_no_ioc_file)]) - self.assertNotEqual(return_code, 0, msg='Return code should be non-zero') - self.assertTrue(next((True for message in logs.output if "CubeMX project .ioc file" in message), False), + self.assertNotEqual(return_code, 0, msg="Return code should be non-zero") + self.assertTrue(next((True for message in logs.output if FileNotFoundError.__name__ in message), False), msg="'ERROR' logging message hasn't been printed") def test_verbose(self): """ - Run as subprocess to capture the full output. Check for both 'DEBUG' logging messages and STM32CubeMX CLI - output. Verbose logs format should match such a regex: + Capture the full output. Check for both 'DEBUG' logging messages and STM32CubeMX CLI output. Verbose logs format + should match such a regex: ^(?=(DEBUG|INFO|WARNING|ERROR|CRITICAL) {0,4})(?=.{8} (?=(build|pio_init|...) {0,26})(?=.{26} [^ ])) """ - # inspect.getmembers is great but it triggers class properties leading to the unacceptable code execution + # inspect.getmembers() is great but it triggers class properties to execute leading to the unwanted code + # execution methods = dir(stm32pio.lib.Stm32pio) + ['main'] buffer_stdout, buffer_stderr = io.StringIO(), io.StringIO() @@ -123,7 +125,7 @@ def test_verbose(self): # stderr and not stdout contains the actual output (by default for the logging module) self.assertEqual(len(buffer_stdout.getvalue()), 0, msg="Process has printed something directly into STDOUT bypassing logging") - self.assertIn('DEBUG', buffer_stderr.getvalue(), msg="Verbose logging output hasn't been enabled on stderr") + self.assertIn('DEBUG', buffer_stderr.getvalue(), msg="Verbose logging output hasn't been enabled on STDERR") # Inject all methods' names in the regex. Inject the width of field in a log format string regex = re.compile("^(?=(DEBUG) {0,4})(?=.{8} (?=(" + '|'.join(methods) + ") {0," + @@ -132,12 +134,13 @@ def test_verbose(self): self.assertGreaterEqual(len(re.findall(regex, buffer_stderr.getvalue())), 1, msg="Logs messages doesn't match the format") - self.assertIn('Starting STM32CubeMX', buffer_stderr.getvalue(), msg="STM32CubeMX has not printed its logs") + # The snippet of the actual STM32CubeMX output + self.assertIn("Starting STM32CubeMX", buffer_stderr.getvalue(), msg="STM32CubeMX has not printed its logs") def test_non_verbose(self): """ - Run as subprocess to capture the full output. We should not see any 'DEBUG' logging messages or STM32CubeMX CLI - output. Logs format should match such a regex: + Capture the full output. We should not see any 'DEBUG' logging messages or STM32CubeMX CLI output. Logs format + should match such a regex: ^(?=(INFO) {0,4})(?=.{8} ((?!( |build|pio_init|...)))) """ @@ -158,6 +161,7 @@ def test_non_verbose(self): self.assertGreaterEqual(len(re.findall(regex, buffer_stderr.getvalue())), 1, msg="Logs messages doesn't match the format") + # The snippet of the actual STM32CubeMX output self.assertNotIn('Starting STM32CubeMX', buffer_stderr.getvalue(), msg="STM32CubeMX has printed its logs") def test_init(self): diff --git a/tests/test_integration.py b/tests/test_integration.py index 3e9ceb2..7f1e9b6 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -5,6 +5,7 @@ import stm32pio.lib import stm32pio.settings +# Provides test constants from tests.test import * @@ -18,15 +19,15 @@ def test_rebase_project(self): Test the portability of projects: they should stay totally valid after moving to another path (same as renaming the parent part of the path). If we will not meet any exceptions, we should consider the test passed. """ - project_before = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project_before = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) project_before.save_config() new_path = f'{project_before.path}-moved' shutil.move(str(project_before.path), new_path) - project_after = stm32pio.lib.Stm32pio(new_path, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project_after = stm32pio.lib.Stm32pio(new_path, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) project_after.generate_code() project_after.pio_init() project_after.patch() @@ -58,8 +59,8 @@ def test_config_priorities(self): # On project creation we should interpret the CLI-provided values as superseding to the saved ones and # saved ones, in turn, as superseding to the default ones - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': cli_parameter_user_value}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': cli_parameter_user_value}}, + instance_options={'save_on_destruction': False}) project.pio_init() project.patch() @@ -74,8 +75,8 @@ def test_build(self): """ Initialize a new project and try to build it """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) project.generate_code() project.pio_init() project.patch() @@ -89,8 +90,8 @@ def test_regenerate_code(self): Simulate a new project creation, its changing and CubeMX code re-generation (for example, after adding new hardware features and some new files by a user) """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) # Generate a new project ... project.generate_code() @@ -123,8 +124,8 @@ def test_current_stage(self): """ Go through the sequence of states emulating the real-life project lifecycle """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.EMPTY) project.save_config() diff --git a/tests/test_unit.py b/tests/test_unit.py index 0a03d57..b872c4f 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -1,6 +1,5 @@ import configparser import inspect -import pathlib import platform import subprocess import time @@ -9,6 +8,7 @@ import stm32pio.settings import stm32pio.util +# Provides test constants from tests.test import * @@ -23,8 +23,8 @@ def test_generate_code(self): """ Check whether files and folders have been created (by STM32CubeMX) """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) project.generate_code() # Assuming that the presence of these files indicating a success @@ -39,8 +39,8 @@ def test_pio_init(self): last one has another traces that can be checked too but we are interested only in a 'platformio.ini' anyway. Also, check that it is a correct configparser file and is not empty """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) result = project.pio_init() self.assertEqual(result, 0, msg="Non-zero return code") @@ -55,7 +55,7 @@ def test_patch(self): Check that new parameters were added, modified were updated and existing parameters didn't gone. Also, check for unnecessary folders deletion """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, instance_options={'save_on_destruction': False}) test_content = inspect.cleandoc(''' ; This is a test config .ini file @@ -108,8 +108,8 @@ def test_build_should_handle_error(self): """ Build an empty project so PlatformIO should return an error """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) project.pio_init() with self.assertLogs(level='ERROR') as logs: @@ -120,9 +120,9 @@ def test_build_should_handle_error(self): def test_start_editor(self): """ - Call the editors + Call the editors. Use subprocess shell=True as it works on all OSes """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, instance_options={'save_on_destruction': False}) editors = { 'atom': { @@ -178,7 +178,7 @@ def test_init_path_not_found_should_raise(self): path_does_not_exist = FIXTURE_PATH.joinpath(path_does_not_exist_name) with self.assertRaisesRegex(FileNotFoundError, path_does_not_exist_name, msg="FileNotFoundError was not raised or doesn't contain a description"): - stm32pio.lib.Stm32pio(path_does_not_exist, save_on_destruction=False) + stm32pio.lib.Stm32pio(path_does_not_exist, instance_options={'save_on_destruction': False}) def test_save_config(self): """ @@ -186,8 +186,8 @@ def test_save_config(self): preserved """ # 'board' is non-default, 'project'-section parameter - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'board': TEST_PROJECT_BOARD}, - save_on_destruction=False) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, + instance_options={'save_on_destruction': False}) project.save_config() self.assertTrue(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).is_file(), From cc4c1c6941f6edd9ac2c585e13a93476e87adb49 Mon Sep 17 00:00:00 2001 From: ussserrr Date: Sat, 14 Mar 2020 23:13:15 +0300 Subject: [PATCH 6/7] 'save_on_destruction' is now False by default --- TODO.md | 7 +++---- stm32pio-gui/app.py | 6 +++--- stm32pio/app.py | 14 ++++++++------ stm32pio/lib.py | 4 ++-- tests/test_integration.py | 18 ++++++------------ tests/test_unit.py | 18 +++++++----------- 6 files changed, 29 insertions(+), 38 deletions(-) diff --git a/TODO.md b/TODO.md index 277b4e5..ebe5fda 100644 --- a/TODO.md +++ b/TODO.md @@ -11,17 +11,16 @@ - [ ] GUI. On 'Clean' clean the log too - [ ] Create VSCode plugin - [x] Remove casts to string where we can use path-like objects (related to Python version as new ones receive path-like objects arguments) - - [ ] We look for some snippets of strings in logs and output for the testing code but we hard-code them and this is not good, probably + - [ ] We look for some snippets of strings in logs and output for the testing code but we hard-code them and this is not good, probably (e.g. 'DEBUG') - [ ] Store a folder initial content in .ini config and ignore it on clean-up process. Allow the user to modify such list (i.e. list of exclusion) - [x] Ask the confirmation of a user by-defualt for `clean` and add additional option for quiet performance - [ ] at some point check for all tools (CubeMX, ...) to be present in the system (both CLI and GUI) (global `--check` command (as `--version`)) - [x] exclude tests from the bundle (see `setup.py` options) - [ ] generate code docs (help user to understand an internal mechanics, e.g. for embedding). Can be uploaded to the GitHub Wiki - [ ] colored logs, maybe... - - [ ] check logging work when embed stm32pio lib in third-party stuff (no logging setup at all) + - [ ] check logging work when embed stm32pio lib in a third-party stuff (no logging setup at all) - [ ] merge subprocess pipes to one where suitable (i.e. `stdout` and `stderr`) - [ ] redirect subprocess pipes to `DEVNULL` where suitable to suppress output - - [ ] some `stm32pio.ini` config file validation - [x] CHANGELOG markdown markup - [ ] Two words about a synchronous nature of the lib and user's responsibility of async wrapping (if needed). Also, maybe migrate to async/await approach in the future - [x] `shlex` for `start_editor` command option sanitizing @@ -43,4 +42,4 @@ - [ ] 'verbose' and 'non-verbose' tests as `subTest` (also 'should_log_error_...') - [ ] turn off all possible dialogs in STM32CubeMX - [ ] test (at least manually) when the tools are not present - - [ ] minimal example of the lib usage for the README + - [x] minimal example of the lib usage for the README diff --git a/stm32pio-gui/app.py b/stm32pio-gui/app.py index 7f3b875..af4fed6 100644 --- a/stm32pio-gui/app.py +++ b/stm32pio-gui/app.py @@ -138,7 +138,7 @@ def __init__(self, project_args: list = None, project_kwargs: dict = None, paren 'logger': self.logger } elif 'logger' not in project_kwargs['instance_options']: - project_kwargs['instance_options']['logger'] = self.logger + project_kwargs['instance_options']['logger'] = self.logger # Start the Stm32pio part initialization right after. It can take some time so we schedule it in a dedicated # thread @@ -155,7 +155,7 @@ def init_project(self, *args, **kwargs) -> None: **kwargs: keyword arguments of the Stm32pio constructor """ try: - self.project = stm32pio.lib.Stm32pio(*args, **kwargs) # our slightly tweaked subclass + self.project = stm32pio.lib.Stm32pio(*args, **kwargs) except Exception as e: # Error during the initialization self.logger.exception(e, exc_info=self.logger.isEnabledFor(logging.DEBUG)) @@ -318,7 +318,7 @@ def addProjectByPath(self, path: QUrl): path: QUrl path to the project folder (absolute by default) """ self.beginInsertRows(QModelIndex(), self.rowCount(), self.rowCount()) - project = ProjectListItem(project_args=[path.toLocalFile()], project_kwargs=dict(instance_options={'save_on_destruction': False}), parent=self) + project = ProjectListItem(project_args=[path.toLocalFile()], parent=self) self.projects.append(project) settings.beginGroup('app') diff --git a/stm32pio/app.py b/stm32pio/app.py index b2a1597..fb4f484 100755 --- a/stm32pio/app.py +++ b/stm32pio/app.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -__version__ = '1.0' +__version__ = '1.10' import argparse import logging @@ -108,7 +108,8 @@ def main(sys_argv=None) -> int: # Main routine try: if args.subcommand == 'init': - project = stm32pio.lib.Stm32pio(args.project_path, parameters={'project': {'board': args.board}}) + project = stm32pio.lib.Stm32pio(args.project_path, parameters={'project': {'board': args.board}}, + instance_options={'save_on_destruction': True}) if not args.board: logger.warning("STM32 PlatformIO board is not specified, it will be needed on PlatformIO project " "creation") @@ -117,7 +118,8 @@ def main(sys_argv=None) -> int: project.start_editor(args.editor) elif args.subcommand == 'new': - project = stm32pio.lib.Stm32pio(args.project_path, parameters={'project': {'board': args.board}}) + project = stm32pio.lib.Stm32pio(args.project_path, parameters={'project': {'board': args.board}}, + instance_options={'save_on_destruction': True}) if project.config.get('project', 'board') == '': raise Exception("STM32 PlatformIO board is not specified, it is needed for PlatformIO project creation") project.generate_code() @@ -129,7 +131,7 @@ def main(sys_argv=None) -> int: project.start_editor(args.editor) elif args.subcommand == 'generate': - project = stm32pio.lib.Stm32pio(args.project_path, instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(args.project_path) project.generate_code() if args.with_build: project.build() @@ -137,11 +139,11 @@ def main(sys_argv=None) -> int: project.start_editor(args.editor) elif args.subcommand == 'status': - project = stm32pio.lib.Stm32pio(args.project_path, instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(args.project_path) print(project.state) elif args.subcommand == 'clean': - project = stm32pio.lib.Stm32pio(args.project_path, instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(args.project_path) if args.quiet: project.clean() else: diff --git a/stm32pio/lib.py b/stm32pio/lib.py index f519755..59fc38f 100644 --- a/stm32pio/lib.py +++ b/stm32pio/lib.py @@ -154,9 +154,9 @@ def __init__(self, dirty_path: str, parameters: dict = None, instance_options: d if parameters is None: parameters = {} - if instance_options is None: # TODO: use Python 3.8 feature - dict schemas + if instance_options is None: # TODO: use Python 3.8 TypedDict instance_options = { - 'save_on_destruction': True, + 'save_on_destruction': False, 'logger': None } diff --git a/tests/test_integration.py b/tests/test_integration.py index 7f1e9b6..eaf98ac 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -19,15 +19,13 @@ def test_rebase_project(self): Test the portability of projects: they should stay totally valid after moving to another path (same as renaming the parent part of the path). If we will not meet any exceptions, we should consider the test passed. """ - project_before = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project_before = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) project_before.save_config() new_path = f'{project_before.path}-moved' shutil.move(str(project_before.path), new_path) - project_after = stm32pio.lib.Stm32pio(new_path, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project_after = stm32pio.lib.Stm32pio(new_path, parameters={'project': {'board': TEST_PROJECT_BOARD}}) project_after.generate_code() project_after.pio_init() project_after.patch() @@ -59,8 +57,7 @@ def test_config_priorities(self): # On project creation we should interpret the CLI-provided values as superseding to the saved ones and # saved ones, in turn, as superseding to the default ones - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': cli_parameter_user_value}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': cli_parameter_user_value}}) project.pio_init() project.patch() @@ -75,8 +72,7 @@ def test_build(self): """ Initialize a new project and try to build it """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) project.generate_code() project.pio_init() project.patch() @@ -90,8 +86,7 @@ def test_regenerate_code(self): Simulate a new project creation, its changing and CubeMX code re-generation (for example, after adding new hardware features and some new files by a user) """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) # Generate a new project ... project.generate_code() @@ -124,8 +119,7 @@ def test_current_stage(self): """ Go through the sequence of states emulating the real-life project lifecycle """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) self.assertEqual(project.state.current_stage, stm32pio.lib.ProjectStage.EMPTY) project.save_config() diff --git a/tests/test_unit.py b/tests/test_unit.py index b872c4f..92308d1 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -23,8 +23,7 @@ def test_generate_code(self): """ Check whether files and folders have been created (by STM32CubeMX) """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) project.generate_code() # Assuming that the presence of these files indicating a success @@ -39,8 +38,7 @@ def test_pio_init(self): last one has another traces that can be checked too but we are interested only in a 'platformio.ini' anyway. Also, check that it is a correct configparser file and is not empty """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) result = project.pio_init() self.assertEqual(result, 0, msg="Non-zero return code") @@ -55,7 +53,7 @@ def test_patch(self): Check that new parameters were added, modified were updated and existing parameters didn't gone. Also, check for unnecessary folders deletion """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH) test_content = inspect.cleandoc(''' ; This is a test config .ini file @@ -108,8 +106,7 @@ def test_build_should_handle_error(self): """ Build an empty project so PlatformIO should return an error """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) project.pio_init() with self.assertLogs(level='ERROR') as logs: @@ -122,7 +119,7 @@ def test_start_editor(self): """ Call the editors. Use subprocess shell=True as it works on all OSes """ - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH) editors = { 'atom': { @@ -178,7 +175,7 @@ def test_init_path_not_found_should_raise(self): path_does_not_exist = FIXTURE_PATH.joinpath(path_does_not_exist_name) with self.assertRaisesRegex(FileNotFoundError, path_does_not_exist_name, msg="FileNotFoundError was not raised or doesn't contain a description"): - stm32pio.lib.Stm32pio(path_does_not_exist, instance_options={'save_on_destruction': False}) + stm32pio.lib.Stm32pio(path_does_not_exist) def test_save_config(self): """ @@ -186,8 +183,7 @@ def test_save_config(self): preserved """ # 'board' is non-default, 'project'-section parameter - project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}, - instance_options={'save_on_destruction': False}) + project = stm32pio.lib.Stm32pio(FIXTURE_PATH, parameters={'project': {'board': TEST_PROJECT_BOARD}}) project.save_config() self.assertTrue(FIXTURE_PATH.joinpath(stm32pio.settings.config_file_name).is_file(), From 7e882a131587e3a15b4945c519d35803165a7954 Mon Sep 17 00:00:00 2001 From: ussserrr Date: Sun, 15 Mar 2020 21:24:42 +0300 Subject: [PATCH 7/7] CHANGELOG, fine tuning --- CHANGELOG.md | 22 ++++++++++++++++++++++ README.md | 20 +++++++++----------- TODO.md | 23 +++++------------------ setup.py | 2 ++ stm32pio/app.py | 6 +++--- stm32pio/lib.py | 11 ++++++----- stm32pio/util.py | 3 +++ tests/test.py | 3 ++- 8 files changed, 52 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a920d5f..c1fdc70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -173,3 +173,25 @@ - Changed: slightly tuned exceptions (more specific ones where it make sense) - Changed: rename `project_path` -> `path` - Changed: actualized tests, more broad usage of the `app.main()` function versus `subprocess.run()` + +## ver. 1.10 (15.03.20) + - New: table of contents for the README + - New: GitHub project wiki + - New: `-q/--quiet` option for the `clean` CLI command. The command now by default warns the user about the content deletion + - New: embedding example (minimal reproducible code snippet, easier than the full CLI or GUI versions) + - New: show the exception name too when the global error has been caught (`stm32pio/app.py`) + - New: sanitize `--start-editor` option value using `shlex.quote()` + - New: parse `platformio.ini` to establish its correctness when checking for project states (`ProjectStage.PIO_INITIALIZED`, `ProjectStage.PATCHED`) + - New: projects are now portable. The user specifying paths relatively to the project folder and using variables (we still don't use `configparser` interpolation but there is no need in it). The backwards compatibility with the old-style config format has been preserved though those projects still will be non-portable unless you manually edit a config + - New: analyze STM32CubeMX output to detect errors on execution. This utility does not necessarily returns non-zero code when some error was happened (e.g. `.ioc` and app versions mismatch and so on), and just shows a dialog + - New: `platformio_ini_config` `Stm32pio` instance property returning current `platformio.ini` parsed `ConfigParser` value. Used in some internal routines such as correctness determination and doesn't have to be used by the library user + - New: `LogPipe` now returns "remote control" `LogPipeRC` - small utility class holding the writable stream and the reference to the string accumulating all incoming messages. It can be accessed later, in the end of the context manager, to store and analyze all the output + - New: some new tests, I think, but I do not remember as all the tests are now moved to the new files :) + - Fixed: warnings appearing during the `pio_build()` execution were suppressed + - Changed: tests are moved out to the root of the repo and excluded from the distribution bundle + - Changed: went back to the PlatformIO CLI as a single point to interact with PlatformIO (remove `platformio` package imports and dependencies) (the reason is crushes when the pio is not isolated in a separated subprocess). Use PlatformIO JSON format output to get and filter boards + - Changed: remove `required=False` from `argparse` commands as it is a default (and even recommended) value anyway + - Changed: remove the unnecessary logging setup when no arguments were given to the program (CLI version) + - Changed: separate `Stm32pio` arguments onto 2 categories: project parameters and instance options and use dictionaries for them. First one has now the same form as the project config `configparser.ConfigParser` and merging into the default and file settings on the project creation. Instance options are more related to the programmatic instance itself and contains currently 2 options - `logger` and `save_on_destruction` + - Changed: use `append()` instead of `insert()` to modify `sys.path` + - Changed: when raising the exceptions use more elegant expressions (e.g. `raise FileNotFoundError(file)` instead of `raise FileNotFoundError("file FILE was not found")`). Use `pathlib.Path().resolve(strict=True)` where appropriate to shorten the code diff --git a/README.md b/README.md index 2baa0bf..ae30518 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ It uses STM32CubeMX to generate a HAL-framework-based code and alongside creates ## Features - Start the new complete project in a single directory using only an `.ioc` file - Update an existing project after changing hardware options in CubeMX - - Clean-up the project (WARNING: it deletes ALL content of project path except the `.ioc` file!) + - Clean-up the project - Get the status information - *[optional]* Automatically run your favorite editor in the end - *[optional]* Automatically make an initial build of the project @@ -47,14 +47,13 @@ stm32pio-repo/ $ python3 stm32pio/app.py # or stm32pio-repo/ $ python3 -m stm32pio # or any-path/ $ python3 path/to/stm32pio-repo/stm32pio/app.py ``` - (we assume python3 and pip3 hereinafter). It is possible to run the app like this from anywhere. However, it's handier to install the utility to be able to run stm32pio from anywhere. Use ```shell script stm32pio-repo/ $ pip install . ``` -command to launch the setup process. Now you can simply type 'stm32pio' in the terminal to run the utility in any directory. +command to launch the setup process. Now you can simply type `stm32pio` in the terminal to run the utility in any directory. Finally, the PyPI distribution (starting from v0.95) is available: ```shell script @@ -87,9 +86,9 @@ It may be useful to tweak some parameters before proceeding. The structure of th You can always run ```shell script -$ python3 app.py --help +$ python app.py --help ``` -to see help on available commands. +to see help on available commands. Find the copy of its output on the [project wiki](https://github.com/ussserrr/stm32pio/wiki/stm32pio-help) page, also. ### Project patching @@ -99,7 +98,7 @@ For those who want to modify the patch (default one is at [`settings.py`](/stm32 ### Embedding -You can also use stm32pio as an ordinary Python package and embed it in your own application. Take a look at the CLI ([`app.py`](/stm32pio/app.py)) or GUI versions to see some possible ways of implementing this. Basically, you need to import `stm32pio.lib` module (where the main `Stm32pio` class resides), (optionally) set up a logger and you are good to go. If you prefer higher-level API similar to the CLI version, use `main()` function in `app.py` passing the same CLI arguments to it (except the actual script name). +You can also use stm32pio as an ordinary Python package and embed it in your own application. Find the minimal example at the [project wiki](https://github.com/ussserrr/stm32pio/wiki/Embedding-example) page to see some possible ways of implementing this. Basically, you need to import `stm32pio.lib` module (where the main `Stm32pio` class resides), (optionally) set up a logger and you are good to go. If you prefer higher-level API similar to the CLI version, use `main()` function in `app.py` passing the same CLI arguments to it (except the actual script name). Also, take a look at the CLI ([`app.py`](/stm32pio/app.py)) or GUI versions. ## Example @@ -133,18 +132,18 @@ You can also use stm32pio as an ordinary Python package and embed it in your own ## Testing There are some tests in file [`test.py`](/stm32pio/tests/test.py) (based on the unittest module). Run ```shell script -stm32pio-repo/ $ python3 -m unittest -b -v +stm32pio-repo/ $ python -m unittest -b -v ``` or ```shell script -stm32pio-repo/ $ python3 -m stm32pio.tests.test -b -v +stm32pio-repo/ $ python -m stm32pio.tests.test -b -v ``` to test the app. It uses STM32F0 framework to generate and build a code from the test [`stm32pio-test-project.ioc`](/stm32pio-test-project/stm32pio-test-project.ioc) project file. Please make sure that the test project folder is clean (i.e. contains only an .ioc file) before running the test otherwise it can lead to some cases failing. Tests automatically create temporary directory (using `tempfile` Python standard module) where all actions are performed. For the specific test suite or case you can use ```shell script -stm32pio-repo/ $ python3 -m unittest stm32pio.tests.test.TestIntegration -b -v -stm32pio-repo/ $ python3 -m unittest stm32pio.tests.test.TestCLI.test_verbose -b -v +stm32pio-repo/ $ python -m unittest stm32pio.tests.test.TestIntegration -b -v +stm32pio-repo/ $ python -m unittest stm32pio.tests.test.TestCLI.test_verbose -b -v ``` @@ -155,4 +154,3 @@ stm32pio-repo/ $ python3 -m unittest stm32pio.tests.test.TestCLI.test_verbose lib_extra_dirs = Middlewares/Third_Party/FreeRTOS ``` You also need to move all `.c`/`.h` files to the appropriate folders respectively. See PlatformIO documentation for more information. - - The project folder, once instantiated, is not portable i.e. if you move it at some other place and invoke stm32pio it will report you an error. This because `stm32pio.ini` config is currently stores absolute paths instead of relative. diff --git a/TODO.md b/TODO.md index ebe5fda..ab83361 100644 --- a/TODO.md +++ b/TODO.md @@ -9,37 +9,24 @@ - [ ] GUI. Implement some other methods for Qt abstract models - [ ] GUI. Warning on 'Clean' action - [ ] GUI. On 'Clean' clean the log too + - [ ] GUI. Stop the chain of commands if someone drops -1 or an exception - [ ] Create VSCode plugin - [x] Remove casts to string where we can use path-like objects (related to Python version as new ones receive path-like objects arguments) - [ ] We look for some snippets of strings in logs and output for the testing code but we hard-code them and this is not good, probably (e.g. 'DEBUG') - [ ] Store a folder initial content in .ini config and ignore it on clean-up process. Allow the user to modify such list (i.e. list of exclusion) - - [x] Ask the confirmation of a user by-defualt for `clean` and add additional option for quiet performance - - [ ] at some point check for all tools (CubeMX, ...) to be present in the system (both CLI and GUI) (global `--check` command (as `--version`)) - - [x] exclude tests from the bundle (see `setup.py` options) + - [ ] at some point check for all tools (CubeMX, ...) to be present in the system (both CLI and GUI) (global `--check` command (as `--version`), also before execution of the full cycle (no sense to start if some tool doesn't exist)) - [ ] generate code docs (help user to understand an internal mechanics, e.g. for embedding). Can be uploaded to the GitHub Wiki - [ ] colored logs, maybe... - [ ] check logging work when embed stm32pio lib in a third-party stuff (no logging setup at all) - [ ] merge subprocess pipes to one where suitable (i.e. `stdout` and `stderr`) - - [ ] redirect subprocess pipes to `DEVNULL` where suitable to suppress output - - [x] CHANGELOG markdown markup + - [ ] redirect subprocess pipes to `DEVNULL` where suitable to suppress output (tests) - [ ] Two words about a synchronous nature of the lib and user's responsibility of async wrapping (if needed). Also, maybe migrate to async/await approach in the future - - [x] `shlex` for `start_editor` command option sanitizing - [ ] `__init__`' `parameters` dict argument schema (Python 3.8 feature). - - [x] Maybe separate on `project_params` and `instance_opts` - - [x] General algo of merging a given dict of parameters with the saved one on project initialization - - [x] parse `platformio.ini` to check its correctness in state getter - - [x] CubeMX 0 return code doesn't necessarily means the correct generation (e.g. migration dialog has appeared and 'Cancel' was chosen, or CubeMX_version < ioc_file_version), probably should somehow analyze the output (logs can be parsed. i.e. 2020-03-05 12:08:40,765 \[ERROR\] MainProjectManager:806 - Program Manager : The version of the current IOC is too high.) - - [x] Dispatch tests on several files (too many code actually) - - [x] Do not store absolute paths in config file and make a project portable (use configparser parameters interpolation). Handle renaming - [ ] See https://docs.python.org/3/howto/logging-cookbook.html#context-info to maybe remade current logging schema (current is, perhaps, a cause of the strange error while testing (in the logging thread)) - [ ] UML diagrams (core, GUI back- and front-ends) - [ ] CI is possible (Arch's AUR has the STM32CubeMX package, also there is a direct link). Deploy Docker in Azure Pipelines, basic at Travis CI - [ ] Test preserving user files and folders on regeneration and mb other operations - [ ] Move special formatters inside the library. It is an implementation detail actually that we use subprocesses and so on - - [x] Mb clean the test project tree before running the tests - - [x] README table of contents - [ ] Mb store the last occurred exception traceback in .ini file and show on some CLI command (so we don't necessarily need to turn on the verbose mode). And, in general, we should show the error reason right off - - [ ] 'verbose' and 'non-verbose' tests as `subTest` (also 'should_log_error_...') - - [ ] turn off all possible dialogs in STM32CubeMX - - [ ] test (at least manually) when the tools are not present - - [x] minimal example of the lib usage for the README + - [ ] 'verbose' and 'non-verbose' tests as `subTest` (also `should_log_error_...`) + - [ ] the lib sometimes raising, sometimes returning the code and it is not consistent. While the reasons behind such behaviour are clear, would be great to always return a result code and raise the exceptions in the outer scope, if there is need to diff --git a/setup.py b/setup.py index 9018c8a..7ce4b28 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,11 @@ import stm32pio.app + with open('README.md', 'r') as readme: long_description = readme.read() + setuptools.setup( name='stm32pio', version=stm32pio.app.__version__, diff --git a/stm32pio/app.py b/stm32pio/app.py index fb4f484..e135dfa 100755 --- a/stm32pio/app.py +++ b/stm32pio/app.py @@ -38,8 +38,8 @@ def parse_args(args: list) -> Optional[argparse.Namespace]: parser_new = subparsers.add_parser('new', help="generate CubeMX code, create PlatformIO project") parser_generate = subparsers.add_parser('generate', help="generate CubeMX code only") parser_status = subparsers.add_parser('status', help="get the description of the current project state") - parser_clean = subparsers.add_parser('clean', help="clean-up the project (WARNING: it deletes ALL content of " - "'path' except the .ioc file)") + parser_clean = subparsers.add_parser('clean', help="clean-up the project (delete ALL content of 'path' " + "except the .ioc file)") # Common subparsers options for p in [parser_init, parser_new, parser_generate, parser_status, parser_clean]: @@ -54,7 +54,7 @@ def parse_args(args: list) -> Optional[argparse.Namespace]: p.add_argument('--with-build', action='store_true', help="build the project after generation") parser_clean.add_argument('-q', '--quiet', action='store_true', - help="suppress the caution about the content removal") + help="suppress the caution about the content removal (be sure of what you are doing!)") if len(args) == 0: parser.print_help() diff --git a/stm32pio/lib.py b/stm32pio/lib.py index 59fc38f..49c2f08 100644 --- a/stm32pio/lib.py +++ b/stm32pio/lib.py @@ -398,15 +398,15 @@ def generate_code(self) -> int: self.logger.info("successful code generation") return result.returncode else: - self.logger.error(f"code generation error (CubeMX return code is {result.returncode}).\n" - "Enable a verbose output or try to generate a code from the CubeMX itself.") + # Probably 'java' error (e.g. no CubeMX is present) + self.logger.error(f"return code is {result.returncode}\n\n{result_output}") raise Exception(error_msg) def pio_init(self) -> int: """ Call PlatformIO CLI to initialize a new project. It uses parameters (path, board) collected before so the - confirmation of the data presence is lying on the invoking code + confirmation about the data presence is lying on the invoking code Returns: return code of the PlatformIO on success, raises an exception otherwise @@ -587,8 +587,9 @@ def build(self) -> int: if not self.logger.isEnabledFor(logging.DEBUG): command_arr.append('--silent') - with stm32pio.util.LogPipe(self.logger, logging.DEBUG if self.logger.isEnabledFor(logging.DEBUG) else logging.WARNING) as log: - result = subprocess.run(command_arr, stdout=log.pipe, stderr=log.pipe) # TODO: stderr is hidden + log_level = logging.DEBUG if self.logger.isEnabledFor(logging.DEBUG) else logging.WARNING + with stm32pio.util.LogPipe(self.logger, log_level) as log: + result = subprocess.run(command_arr, stdout=log.pipe, stderr=log.pipe) if result.returncode == 0: self.logger.info("successful PlatformIO build") diff --git a/stm32pio/util.py b/stm32pio/util.py index 38118b1..d253168 100644 --- a/stm32pio/util.py +++ b/stm32pio/util.py @@ -78,10 +78,13 @@ class LogPipeRC: """ Small class suitable for passing to the caller when the LogPipe context manager is invoked """ + value = '' # string accumulating all incoming messages + def __init__(self, fd: int): self.pipe = fd # writable half of os.pipe + class LogPipe(threading.Thread): """ The thread combined with a context manager to provide a nice way to temporarily redirect something's stream output diff --git a/tests/test.py b/tests/test.py index 589a16e..d449e18 100755 --- a/tests/test.py +++ b/tests/test.py @@ -4,7 +4,8 @@ NOTE: make sure the test project tree is clean before running the tests! -'pyenv' was used to execute tests with different Python versions (under Ubuntu): +'pyenv' was used to execute tests with different Python versions (under Linux): +https://github.com/pyenv/pyenv https://www.tecmint.com/pyenv-install-and-manage-multiple-python-versions-in-linux/ To get the test coverage install and use 'coverage' package: