diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5fd082b4..ba9c8008 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,8 +16,7 @@ "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", - "ms-python.flake8", - "ms-python.autopep8" + "ms-python.pylint" ] } }, diff --git a/.github/verify.py b/.github/verify.py index db9ab0bd..04b9bcc6 100755 --- a/.github/verify.py +++ b/.github/verify.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Command-line helper for running verification builds. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a8d0a62..163e5ec7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ jobs: release: if: ${{ github.event_name == 'release' && !github.event.release.prerelease }} runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36b99958..a6d398ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: jobs: test: runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 steps: - uses: actions/checkout@v3 with: @@ -95,7 +95,7 @@ jobs: compat-test-python3-windows-and-mac: strategy: matrix: - python3-version: ['10','11'] + python3-version: ['11','12'] python3-platform: ['windows-latest', 'macos-latest'] runs-on: ${{ matrix.python3-platform }} needs: test @@ -115,9 +115,9 @@ jobs: compat-test-python3-ubuntu: strategy: matrix: - python3-version: ['7', '8', '9', '10', '11'] + python3-version: ['7', '8', '9', '10', '11', '12'] runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 needs: test steps: - uses: actions/checkout@v3 @@ -129,7 +129,7 @@ jobs: language-verification-c: runs-on: ubuntu-latest needs: test - container: ghcr.io/opencyphal/toolshed:ts22.4.1 + container: ghcr.io/opencyphal/toolshed:ts22.4.2 strategy: matrix: build_type: [Debug, Release, MinSizeRel] @@ -223,7 +223,7 @@ jobs: language-verification-python: runs-on: ubuntu-latest needs: test - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 steps: - uses: actions/checkout@v3 with: diff --git a/.gitignore b/.gitignore index 298af685..2ebdd67f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__ *.egg-info *.patch .tox +.nox .coverage* .venv coverage.xml @@ -26,6 +27,7 @@ dist .DS_Store prof out +venv # Eclipse .metadata diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c70118b6..b47700b5 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,6 +3,8 @@ build: os: ubuntu-22.04 tools: python: "3.11" +sphinx: + configuration: conf.py python: install: - requirements: requirements.txt diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 8152e0c8..e45ff5b8 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,7 +8,6 @@ "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", - "ms-python.flake8", - "ms-python.autopep8" + "ms-python.pylint" ] } diff --git a/.vscode/launch.json b/.vscode/launch.json index 6f2412d0..b7419f4c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,7 +6,7 @@ "configurations": [ { "name": "Python: nnvg c++", - "type": "python", + "type": "debugpy", "request": "launch", "module": "nunavut", "cwd": "${workspaceFolder}/src", @@ -18,22 +18,33 @@ }, { "name": "Pytest: current test", - "type": "python", + "type": "debugpy", "request": "launch", "module": "pytest", "args": [ "--keep-generated", + "--rootdir=${workspaceFolder}", "${file}" ], - "console": "internalConsole", + "cwd": "${workspaceFolder}" + }, + { + "name": "Pytest: all doc tests", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": [ + "--keep-generated", + "--rootdir=${workspaceFolder}", + "${workspaceFolder}/src" + ], "cwd": "${workspaceFolder}" }, { "name": "Pytest: all tests", - "type": "python", + "type": "debugpy", "request": "launch", "module": "pytest", - "console": "internalConsole", "cwd": "${workspaceFolder}" }, { diff --git a/.vscode/nunavut-words.txt b/.vscode/nunavut-words.txt new file mode 100644 index 00000000..6c8090df --- /dev/null +++ b/.vscode/nunavut-words.txt @@ -0,0 +1,52 @@ +addoption +allclose +astype +autouse +behaviour +bitorder +bools +builtins +Bxxx +caplog +CDEF +codegen +doctests +dryrun +dtype +EDCB +elementwise +emptylines +endianness +errstate +fillvalue +finalizer +fpid +frombuffer +functor +functors +htmlcov +itemsize +lctx +markupsafe +maxsplit +nbytes +ndarray +ndim +nnvg +noxfile +outdir +packbits +postprocessor +postprocessors +roadmap +rtype +Sriram +tobytes +transcompilation +typecheck +Unionant +unpackbits +unseparate +unstropped +WKCV +Xlang diff --git a/.vscode/settings.json b/.vscode/settings.json index c8527680..2132ab87 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,12 +3,18 @@ "python.testing.cwd": "${workspaceFolder}", "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, + "pylint.args": [ + "--rcfile=${workspaceFolder}/tox.ini" + ], "black-formatter.args": [ "--line-length=120" ], "mypy-type-checker.args": [ "--config-file=${workspaceFolder}/tox.ini" ], + "flake8.args": [ + "--line-length=120" + ], "files.exclude": { "**/.git": true, "**/.svn": true, @@ -168,56 +174,20 @@ "environment": "", "sourceFileMap": "${sourceFileMapObj}" }, - "cSpell.words": [ - "allclose", - "astype", - "autouse", - "bitorder", - "bools", - "builtins", - "Bxxx", - "caplog", - "CDEF", - "codegen", - "Cyphal", - "doctests", - "DSDL", - "dtype", - "EDCB", - "elementwise", - "emptylines", - "endianness", - "errstate", - "fillvalue", - "fpid", - "frombuffer", - "htmlcov", - "itemsize", - "Kirienko", - "maxsplit", - "nbytes", - "ndarray", - "ndim", - "nnvg", - "noxfile", - "opencyphal", - "outdir", - "packbits", - "Pavel", - "postprocessor", - "postprocessors", - "pycyphal", - "pydsdl", - "roadmap", - "Sriram", - "tobytes", - "transcompilation", - "typecheck", - "uavcan", - "unpackbits", - "unseparate", - "unstropped", - "WKCV", - "Unionant" - ], + "cSpell.allowCompoundWords": true, + "cSpell.caseSensitive": false, + "cSpell.customDictionaries": { + "cyphal" : { + "name": "Cyphal-words", + "path": "${workspaceRoot}/cyphal-words.txt", + "description": "Words used in Cyphal and UAVCAN to add to spell checker dictionaries.", + "addWords": true + }, + "nunavut" : { + "name": "Nunavut-words", + "path": "${workspaceRoot}/.vscode/nunavut-words.txt", + "description": "Words used in in the Nunavut codebase to add to spell checker dictionaries.", + "addWords": true + } + } } diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c22bf796..cb715996 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -24,7 +24,7 @@ your dev environment setup. Tools ************************************************ -tox -e local +tox devenv -e local ================================================ I highly recommend using the local tox environment when doing python development. It'll save you hours @@ -33,8 +33,8 @@ global python environment. You can install tox from brew on osx or apt-get on GN recommend the following environment for vscode:: git submodule update --init --recursive - tox -e local - source .tox/local/bin/activate + tox devenv -e local + source venv/bin/activate cmake @@ -58,8 +58,8 @@ Do:: cd path/to/nunavut git submodule update --init --recursive - tox -e local - source .tox/local/bin/activate + tox devenv -e local + source venv/bin/activate code . Then install recommended extensions. @@ -72,16 +72,16 @@ To run the full suite of `tox`_ tests locally you'll need docker. Once you have and running do:: git submodule update --init --recursive - docker pull ghcr.io/opencyphal/toxic:tx22.4.1 - docker run --rm -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.1 tox + docker pull ghcr.io/opencyphal/toxic:tx22.4.2 + docker run --rm -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.2 tox To run a limited suite using only locally available interpreters directly on your host machine, -skip the docker invocations and use ``tox -s``. +skip the docker invocations and use ``tox run -s``. To run the language verification build you'll need to use a different docker container:: - docker pull ghcr.io/opencyphal/toolshed:ts22.4.3 - docker run --rm -it -v $PWD:/workspace ghcr.io/opencyphal/toolshed:ts22.4.3 + docker pull ghcr.io/opencyphal/toolshed:ts22.4.5 + docker run --rm -it -v $PWD:/workspace ghcr.io/opencyphal/toolshed:ts22.4.5 cd /workspace ./.github/verify.py -l c ./.github/verify.py -l cpp @@ -187,7 +187,7 @@ Building The Docs We rely on `read the docs`_ to build our documentation from github but we also verify this build as part of our tox build. This means you can view a local copy after completing a full, successful test run (See `Running The Tests`_) or do -:code:`docker run --rm -t -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.1 /bin/sh -c "tox -e docs"` to build +:code:`docker run --rm -t -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.2 /bin/sh -c "tox run -e docs"` to build the docs target. You can open the index.html under ``.tox/docs/tmp/index.html`` or run a local web-server:: @@ -198,15 +198,6 @@ Of course, you can just use `Visual Studio Code`_ to build and preview the docs :code:`> reStructuredText: Open Preview`. -apidoc -================================================ - -We manually generate the api doc using ``sphinx-apidoc``. To regenerate use ``tox -e gen-apidoc``. - -.. warning:: - - ``tox -e gen-apidoc`` will start by deleting the docs/api directory. - ************************************************ Coverage and Linting Reports ************************************************ diff --git a/LICENSE.rst b/LICENSE.rst index 16cfd48d..d5ee39ee 100644 --- a/LICENSE.rst +++ b/LICENSE.rst @@ -1,5 +1,5 @@ ################################################ -Licence +License ################################################ ************************************* diff --git a/README.rst b/README.rst index d2cfb399..9cbe3845 100644 --- a/README.rst +++ b/README.rst @@ -68,7 +68,7 @@ Nunavut is invoked to generate code for the former. .. code-block:: shell - nnvg --target-language c --target-endianness=little --enable-serialization-asserts public_regulated_data_types/reg --lookup-dir public_regulated_data_types/uavcan + nnvg --target-language c --enable-serialization-asserts public_regulated_data_types/reg --lookup-dir public_regulated_data_types/uavcan Generate HTML documentation pages using the command-line tool ------------------------------------------------------------- diff --git a/conf.py b/conf.py index 5439a007..59aa4092 100644 --- a/conf.py +++ b/conf.py @@ -24,13 +24,13 @@ _version_tuple = nunavut_version.split(".") # The short X.Y version -version = "{}.{}".format(_version_tuple[0], _version_tuple[1]) +version = f"{_version_tuple[0]}.{_version_tuple[1]}" # The full version, including alpha/beta/rc tags release = nunavut_version -exclude_patterns = ["**/test", "**/.nox"] +exclude_patterns = ["**/test", "verification"] -with open(".gitignore", "r") as gif: +with open(".gitignore", "r", encoding="utf-8") as gif: for line in gif: stripped = line.strip() if len(stripped) > 0 and not stripped.startswith("#"): @@ -55,6 +55,7 @@ "sphinxarg.ext", "sphinx.ext.intersphinx", "sphinxemoji.sphinxemoji", + "sphinx_rtd_theme" ] # Add any paths that contain templates here, relative to this directory. diff --git a/conftest.py b/conftest.py index c27056ac..99453557 100644 --- a/conftest.py +++ b/conftest.py @@ -1,9 +1,10 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ -Fixtures for our tests. +Configuration for pytest tests including fixtures and hooks. """ import logging @@ -20,69 +21,24 @@ import pydsdl import pytest from sybil import Sybil - -try: - from sybil.parsers.codeblock import PythonCodeBlockParser -except ImportError: - from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser - -from sybil.parsers.doctest import DocTestParser +from sybil.parsers.rest import DocTestParser, PythonCodeBlockParser from nunavut import Namespace -# +-------------------------------------------------------------------------------------------------------------------+ -# | PYTEST HOOKS -# +-------------------------------------------------------------------------------------------------------------------+ - - -def pytest_configure(config: typing.Any) -> None: - """ - See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks - """ - # pydsdl._dsdl_definition is reeeeeeealy verbose at the INFO level and below. Turn this down to reduce - # scroll-blindness. - logging.getLogger("pydsdl._dsdl_definition").setLevel(logging.WARNING) - # A lot of DEBUG noise in the other loggers so we'll tune this down to INFO and higher. - logging.getLogger("pydsdl._namespace").setLevel(logging.INFO) - logging.getLogger("pydsdl._data_type_builder").setLevel(logging.INFO) - - -def pytest_addoption(parser): # type: ignore - """ - See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks - """ - parser.addoption( - "--keep-generated", - action="store_true", - help=textwrap.dedent( - """ - If set then the temporary directory used to generate files for each test will be left after - the test has completed. Normally this directory is temporary and therefore cleaned up automatically. - - :: WARNING :: - This will leave orphaned files on disk. They won't be big but there will be a lot of them. - - :: WARNING :: - Do not run tests in parallel when using this option. - """ - ), - ) - - # +-------------------------------------------------------------------------------------------------------------------+ # | TEST FIXTURES # +-------------------------------------------------------------------------------------------------------------------+ @pytest.fixture -def run_nnvg(request): # type: ignore +def run_nnvg(request: pytest.FixtureRequest) -> typing.Callable: # pylint: disable=unused-argument """ - Test helper for invoking the nnvg commandline script as part of a unit test. + Test helper for invoking the nnvg command-line script as part of a unit test. """ def _run_nnvg( - gen_paths: typing.Any, + _: typing.Any, args: typing.List[str], check_result: bool = True, env: typing.Optional[typing.Dict[str, str]] = None, @@ -102,8 +58,7 @@ def _run_nnvg( except subprocess.CalledProcessError as e: if raise_called_process_error: raise e - else: - raise AssertionError(e.stderr.decode("utf-8")) + raise AssertionError(e.stderr.decode("utf-8")) from e return _run_nnvg @@ -113,7 +68,7 @@ class GenTestPaths: def __init__(self, test_file: str, keep_temporaries: bool, node_name: str): test_file_path = pathlib.Path(test_file) - self.test_name = "{}_{}".format(test_file_path.parent.stem, node_name) + self.test_name = f"{test_file_path.parent.stem}_{node_name}" self.test_dir = test_file_path.parent search_dir = self.test_dir.resolve() while search_dir.is_dir() and not (search_dir / pathlib.Path("src")).is_dir(): @@ -125,19 +80,25 @@ def __init__(self, test_file: str, keep_temporaries: bool, node_name: str): self.lang_src_dir = self.root_dir / pathlib.Path("src") / pathlib.Path("nunavut") / pathlib.Path("lang") self._keep_temp = keep_temporaries - self._out_dir = None # type: typing.Optional[pathlib.Path] - self._build_dir = None # type: typing.Optional[pathlib.Path] - self._dsdl_dir = None # type: typing.Optional[pathlib.Path] - self._temp_dirs = [] # type: typing.List[tempfile.TemporaryDirectory] - print('Paths for test "{}" under dir {}'.format(self.test_name, self.test_dir)) - print("(root directory: {})".format(self.root_dir)) + self._out_dir: typing.Optional[pathlib.Path] = None + self._build_dir: typing.Optional[pathlib.Path] = None + self._dsdl_dir: typing.Optional[pathlib.Path] = None + self._temp_dirs: typing.List[tempfile.TemporaryDirectory] = [] + print(f'Paths for test "{self.test_name}" under dir {self.test_dir}') + print(f"(root directory: {self.root_dir})") def test_path_finalizer(self) -> None: + """ + Finalizer to clean up any temporary directories created during the test. + """ for temporary_dir in self._temp_dirs: temporary_dir.cleanup() self._temp_dirs.clear() def create_new_temp_dir(self, dir_key: str) -> pathlib.Path: + """ + Create a new temporary directory for the test case. + """ if self._keep_temp: result = self._ensure_dir(self.build_dir / pathlib.Path(dir_key)) else: @@ -157,6 +118,9 @@ def out_dir(self) -> pathlib.Path: @property def build_dir(self) -> pathlib.Path: + """ + The directory to place build artifacts under for this test case. + """ if self._build_dir is None: self._build_dir = self._ensure_dir(self.root_dir / pathlib.Path("build")) return self._build_dir @@ -165,7 +129,10 @@ def build_dir(self) -> pathlib.Path: def find_outfile_in_namespace( typename: str, namespace: Namespace, type_version: pydsdl.Version = None ) -> typing.Optional[str]: - found_outfile = None # type: typing.Optional[str] + """ + Find the output file for a given type in a namespace. + """ + found_outfile: typing.Optional[str] = None for dsdl_type, outfile in namespace.get_all_types(): if dsdl_type.full_name == typename: if type_version is not None: @@ -176,8 +143,8 @@ def find_outfile_in_namespace( # of the type we're looking for. elif found_outfile is not None: raise RuntimeError( - "Type {} had more than one version for this test but no type version argument" - " was provided.".format(typename) + f"Type {typename} had more than one version for this test but no type version argument" + " was provided." ) else: found_outfile = str(outfile) @@ -191,14 +158,14 @@ def _ensure_dir(path_dir: pathlib.Path) -> pathlib.Path: except FileExistsError: pass if not path_dir.exists() or not path_dir.is_dir(): - raise RuntimeWarning('Test directory "{}" was not setup properly. Tests may fail.'.format(path_dir)) + raise RuntimeWarning(f'Test directory "{path_dir}" was not setup properly. Tests may fail.') return path_dir @pytest.fixture(scope="function") -def gen_paths(request): # type: ignore +def gen_paths(request: pytest.FixtureRequest) -> GenTestPaths: """ - Used by the "gentest" unittests in Nunavut to standardize output paths for generated code created as part of + Used by the "gentest" unit tests in Nunavut to standardize output paths for generated code created as part of the tests. Use the --keep-generated argument to disable the auto-clean behaviour this fixture provides by default. """ g = GenTestPaths(str(request.fspath), request.config.option.keep_generated, request.node.name) @@ -206,9 +173,23 @@ def gen_paths(request): # type: ignore return g +@pytest.fixture(scope="module") +def gen_paths_for_module(request: pytest.FixtureRequest) -> GenTestPaths: # pylint: disable=unused-argument + """ + Used by our Sybil doctests in Nunavut to standardize output paths for generated code created as part of + the tests. Use the --keep-generated argument to disable the auto-clean behaviour this fixture provides by default. + + Note: this fixture is different than gen_paths because it is scoped to the module level. This is useful for + Sybil tests that share temporary files across different test blocks within the same document. + """ + g = GenTestPaths(str(request.fspath), request.config.option.keep_generated, request.node.name) + request.addfinalizer(g.test_path_finalizer) + return g + + class _UniqueNameEvaluator: def __init__(self) -> None: - self._found_names = set() # type: typing.Set[str] + self._found_names: typing.Set[str] = set() def __call__(self, expected_pattern: str, actual_value: str) -> None: assert re.match(expected_pattern, actual_value) is not None @@ -217,7 +198,7 @@ def __call__(self, expected_pattern: str, actual_value: str) -> None: @pytest.fixture(scope="function") -def unique_name_evaluator(request): # type: ignore +def unique_name_evaluator(request: pytest.FixtureRequest) -> _UniqueNameEvaluator: # pylint: disable=unused-argument """ Class that defined ``assert_is_expected_and_unique`` allowing assertion that a set of values in a single test adhere to a provided pattern and are unique values (compared to other values @@ -240,11 +221,11 @@ def test_is_unique(unique_name_evaluator) -> None: @pytest.fixture -def assert_language_config_value(request): # type: ignore +def assert_language_config_value(request: pytest.FixtureRequest) -> typing.Callable: # pylint: disable=unused-argument """ Assert that a given configuration value is set for the target language. """ - from nunavut.lang import LanguageContext, LanguageContextBuilder + from nunavut.lang import LanguageContext, LanguageContextBuilder # pylint: disable=import-outside-toplevel def _assert_language_config_value( target_language: typing.Union[str, LanguageContext], @@ -271,7 +252,7 @@ def _assert_language_config_value( @pytest.fixture -def jinja_filter_tester(request): # type: ignore +def jinja_filter_tester(request: pytest.FixtureRequest): # pylint: disable=unused-argument """ Use to create fluent but testable documentation for Jinja filters and tests @@ -311,17 +292,17 @@ def filter_dummy(env, input): jinja_filter_tester(filter_dummy, template, rendered, lctx, I=I) """ - from nunavut.jinja.jinja2 import DictLoader - from nunavut.lang import LanguageContext, LanguageContextBuilder + from nunavut.jinja.jinja2 import DictLoader # pylint: disable=import-outside-toplevel + from nunavut.lang import LanguageContext, LanguageContextBuilder # pylint: disable=import-outside-toplevel def _make_filter_test_template( filter_or_list_of_filters: typing.Union[None, typing.Callable, typing.List[typing.Callable]], body: str, expected: str, target_language_or_language_context: typing.Union[str, LanguageContext], - **globals: typing.Optional[typing.Dict[str, typing.Any]] + **additional_globals: typing.Optional[typing.Dict[str, typing.Any]], ) -> str: - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder # pylint: disable=import-outside-toplevel if isinstance(target_language_or_language_context, LanguageContext): lctx = target_language_or_language_context @@ -333,23 +314,23 @@ def _make_filter_test_template( ) if filter_or_list_of_filters is None: - additional_filters = dict() # type: typing.Optional[typing.Dict[str, typing.Callable]] + additional_filters: typing.Optional[typing.Dict[str, typing.Callable]] = {} elif isinstance(filter_or_list_of_filters, list): - additional_filters = dict() - for filter in filter_or_list_of_filters: - additional_filters[filter.__name__] = filter + additional_filters = {} + for filter_method in filter_or_list_of_filters: + additional_filters[filter_method.__name__] = filter_method else: additional_filters = {filter_or_list_of_filters.__name__: filter_or_list_of_filters} - e = CodeGenEnvironment( - lctx=lctx, - loader=DictLoader({"test": body}), - allow_filter_test_or_use_query_overwrite=True, - additional_filters=additional_filters, - additional_globals=globals, + e = ( + CodeGenEnvironmentBuilder(DictLoader({"test": body}), lctx) + .set_allow_filter_test_or_use_query_overwrite(True) + .add_filters(**additional_filters) + .add_globals(**additional_globals) + .create() ) e.update_nunavut_globals( - *lctx.get_target_language().get_support_module(), is_dryrun=True, omit_serialization_support=True + *lctx.get_target_language().get_support_module(), omit_serialization_support=True, embed_auditing_info=True ) rendered = str(e.get_template("test").render()) @@ -364,18 +345,57 @@ def _make_filter_test_template( @pytest.fixture -def mock_environment(request): # type: ignore +def mock_environment(request: pytest.FixtureRequest) -> typing.Any: # pylint: disable=unused-argument """ A MagicMock that can be used where a jinja environment is needed. """ - from unittest.mock import MagicMock + from unittest.mock import MagicMock # pylint: disable=import-outside-toplevel - mock_environment = MagicMock() + magic_mock_environment = MagicMock() support_mock = MagicMock() - mock_environment.globals = {"nunavut": support_mock} + magic_mock_environment.globals = {"nunavut": support_mock} support_mock.support = {"omit": True} - return mock_environment + return magic_mock_environment + + +# +-------------------------------------------------------------------------------------------------------------------+ +# | PYTEST HOOKS +# +-------------------------------------------------------------------------------------------------------------------+ + + +def pytest_configure(config: typing.Any) -> None: # pylint: disable=unused-argument + """ + See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks + """ + # pydsdl._dsdl_definition is reeeeeeealy verbose at the INFO level and below. Turn this down to reduce + # scroll-blindness. + logging.getLogger("pydsdl._dsdl_definition").setLevel(logging.WARNING) + # A lot of DEBUG noise in the other loggers so we'll tune this down to INFO and higher. + logging.getLogger("pydsdl._namespace").setLevel(logging.INFO) + logging.getLogger("pydsdl._data_type_builder").setLevel(logging.INFO) + + +def pytest_addoption(parser: pytest.Parser) -> None: + """ + See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks + """ + parser.addoption( + "--keep-generated", + action="store_true", + help=textwrap.dedent( + """ + If set then the temporary directory used to generate files for each test will be left after + the test has completed. Normally this directory is temporary and therefore cleaned up automatically. + + :: WARNING :: + This will leave orphaned files on disk. They won't be big but there will be a lot of them. + + :: WARNING :: + Do not run tests in parallel when using this option. + """ + ), + ) # +-------------------------------------------------------------------------------------------------------------------+ @@ -383,7 +403,7 @@ def mock_environment(request): # type: ignore # +-------------------------------------------------------------------------------------------------------------------+ -_sy = Sybil( +pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS), PythonCodeBlockParser(), @@ -404,9 +424,7 @@ def mock_environment(request): # type: ignore fixtures=[ "jinja_filter_tester", "gen_paths", + "gen_paths_for_module", "assert_language_config_value", ], -) - - -pytest_collect_file = _sy.pytest() +).pytest() diff --git a/cyphal-words.txt b/cyphal-words.txt new file mode 100644 index 00000000..83117e7a --- /dev/null +++ b/cyphal-words.txt @@ -0,0 +1,10 @@ +cetl +Cyphal +DSDL +Kirienko +opencyphal +Pavel +pycyphal +pydsdl +roadmap +uavcan diff --git a/docs/templates.rst b/docs/templates.rst index 1f538ae2..b37e9f48 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -237,6 +237,8 @@ Common Filters :noindex: .. autofunction:: nunavut.jinja.DSDLCodeGenerator.filter_bits2bytes_ceil :noindex: +.. autofunction:: nunavut.jinja.DSDLCodeGenerator.filter_text_table + :noindex: Common Tests ------------------------------------------------- @@ -325,6 +327,10 @@ C++ Use Queries ------------------------------------------------- .. autofunction:: nunavut.lang.cpp.uses_std_variant :noindex: +.. autofunction:: nunavut.lang.cpp.uses_cetl + :noindex: +.. autofunction:: nunavut.lang.cpp.uses_pmr + :noindex: Python Filters @@ -342,6 +348,12 @@ Python Filters :noindex: .. autofunction:: nunavut.lang.py.filter_longest_id_length :noindex: +.. autofunction:: nunavut.lang.py.filter_pickle + :noindex: +.. autofunction:: nunavut.lang.py.filter_numpy_scalar_type + :noindex: +.. autofunction:: nunavut.lang.py.filter_newest_minor_version_aliases + :noindex: HTML Filters diff --git a/requirements.txt b/requirements.txt index 3212a6ff..4aa9b534 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ # This file provided for readthedocs.io only. Use tox.ini for all dependencies. . +sphinx_rtd_theme sphinx-argparse sphinxemoji diff --git a/setup.cfg b/setup.cfg index bd714ac0..206f014c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ long_description = file: README.rst long_description_content_type = text/x-rst license = MIT license_files = LICENSE.rst -keywords = uavcan, dsdl, can, can-bus, codegen, cyphal, opencyphal +keywords = uavcan, dsdl, can, can-bus, ethernet, udp, codegen, cyphal, opencyphal classifiers = Development Status :: 3 - Alpha Environment :: Console @@ -21,6 +21,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: 3 :: Only Topic :: Scientific/Engineering Topic :: Software Development :: Embedded Systems @@ -37,7 +38,7 @@ package_dir= packages=find: package_data={"nunavut": ["py.typed"]} install_requires= - pydsdl ~= 1.18 + pydsdl pyyaml importlib-resources diff --git a/src/nunavut/__init__.py b/src/nunavut/__init__.py index 7c6a5a07..b0a7d511 100644 --- a/src/nunavut/__init__.py +++ b/src/nunavut/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2022 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Code generator built on top of pydsdl. @@ -68,28 +68,29 @@ """ import sys as _sys -from ._generators import AbstractGenerator as AbstractGenerator -from ._generators import generate_types as generate_types -from ._namespace import Namespace as Namespace -from ._namespace import build_namespace_tree as build_namespace_tree -from ._utilities import TEMPLATE_SUFFIX as TEMPLATE_SUFFIX +from ._generators import AbstractGenerator +from ._generators import generate_types +from ._namespace import Namespace +from ._namespace import build_namespace_tree +from ._utilities import TEMPLATE_SUFFIX from ._utilities import YesNoDefault -from ._version import __author__ as __author__ -from ._version import __copyright__ as __copyright__ -from ._version import __email__ as __email__ -from ._version import __license__ as __license__ -from ._version import __version__ as __version__ -from .jinja import CodeGenerator as CodeGenerator -from .jinja import DSDLCodeGenerator as DSDLCodeGenerator -from .jinja import SupportGenerator as SupportGenerator -from .lang import Language as Language -from .lang import LanguageContext as LanguageContext -from .lang import LanguageContextBuilder as LanguageContextBuilder -from .lang import UnsupportedLanguageError as UnsupportedLanguageError -from .lang._config import LanguageConfig as LanguageConfig -from ._exceptions import InternalError as InternalError - -if _sys.version_info[:2] < (3, 5): # pragma: no cover +from ._utilities import DefaultValue +from ._version import __author__ +from ._version import __copyright__ +from ._version import __email__ +from ._version import __license__ +from ._version import __version__ +from .jinja import CodeGenerator +from .jinja import DSDLCodeGenerator +from .jinja import SupportGenerator +from .lang import Language +from .lang import LanguageContext +from .lang import LanguageContextBuilder +from .lang import UnsupportedLanguageError +from .lang._config import LanguageConfig +from ._exceptions import InternalError + +if _sys.version_info[:2] < (3, 7): # pragma: no cover print("A newer version of Python is required", file=_sys.stderr) _sys.exit(1) @@ -102,6 +103,7 @@ "DSDLCodeGenerator", "generate_types", "LanguageConfig", + "DefaultValue", "Language", "LanguageContext", "LanguageContextBuilder", diff --git a/src/nunavut/__main__.py b/src/nunavut/__main__.py index 7adcd9f9..c6d0f6ee 100644 --- a/src/nunavut/__main__.py +++ b/src/nunavut/__main__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Command-line entrypoint. diff --git a/src/nunavut/_dependencies.py b/src/nunavut/_dependencies.py index 45fd400d..995ecc98 100644 --- a/src/nunavut/_dependencies.py +++ b/src/nunavut/_dependencies.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Objects and utilities for handling DSDL dependencies when generating code for a given type. diff --git a/src/nunavut/_exceptions.py b/src/nunavut/_exceptions.py index 5a9e517c..8d9552bd 100644 --- a/src/nunavut/_exceptions.py +++ b/src/nunavut/_exceptions.py @@ -1,7 +1,7 @@ # -# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2023 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Exception types thrown by the core library. diff --git a/src/nunavut/_generators.py b/src/nunavut/_generators.py index 9cd9b512..40d804bd 100644 --- a/src/nunavut/_generators.py +++ b/src/nunavut/_generators.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Module containing types and utilities for building generator objects. @@ -78,7 +78,11 @@ def get_templates(self, omit_serialization_support: bool = False) -> typing.Iter @abc.abstractmethod def generate_all( - self, is_dryrun: bool = False, allow_overwrite: bool = True, omit_serialization_support: bool = False + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> typing.Iterable[pathlib.Path]: """ Generates all output for a given :class:`nunavut.Namespace` and using @@ -91,6 +95,9 @@ def generate_all( output file exists and the generation is not a dry-run. :param bool omit_serialization_support: If True then the generator will emit only types without additional serialization and deserialization support and logic. + :param embed_auditing_info: If True then additional information about the inputs and environment used to + generate source will be embedded in the generated files at the cost of build + reproducibility. :return: 0 for success. Non-zero for errors. :raises: PermissionError if :attr:`allow_overwrite` is False and the file exists. """ @@ -108,7 +115,7 @@ def create_default_generators( :return: Tuple with the first item being the code-generator and the second the support-library generator. """ - from nunavut.jinja import DSDLCodeGenerator, SupportGenerator + from nunavut.jinja import DSDLCodeGenerator, SupportGenerator # pylint: disable=import-outside-toplevel return (DSDLCodeGenerator(namespace, **kwargs), SupportGenerator(namespace, **kwargs)) @@ -127,8 +134,9 @@ def generate_types( allow_overwrite: bool = True, lookup_directories: typing.Optional[typing.Iterable[str]] = None, allow_unregulated_fixed_port_id: bool = False, - language_options: typing.Mapping[str, typing.Any] = {}, + language_options: typing.Optional[typing.Mapping[str, typing.Any]] = None, include_experimental_languages: bool = False, + embed_auditing_info: bool = False, ) -> None: """ Helper method that uses default settings and built-in templates to generate types for a given @@ -153,7 +161,12 @@ def generate_types( language objects. The supported arguments and valid values are different depending on the language specified by the `language_key` parameter. :param bool include_experimental_languages: If true then experimental languages will also be available. + :param embed_auditing_info: If True then additional information about the inputs and environment used to + generate source will be embedded in the generated files at the cost of build + reproducibility. """ + if language_options is None: + language_options = {} language_context = ( LanguageContextBuilder(include_experimental_languages=include_experimental_languages) @@ -172,5 +185,5 @@ def generate_types( namespace = build_namespace_tree(type_map, str(root_namespace_dir), str(out_dir), language_context) generator, support_generator = create_default_generators(namespace) - support_generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support) - generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support) + support_generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support, embed_auditing_info) + generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support, embed_auditing_info) diff --git a/src/nunavut/_namespace.py b/src/nunavut/_namespace.py index c4ed79f9..80da5413 100644 --- a/src/nunavut/_namespace.py +++ b/src/nunavut/_namespace.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Namespace object and associated utilities. Nunavut namespaces provide an internal representation of dsdl namespaces @@ -315,6 +315,7 @@ def build_namespace_tree( :param nunavut.LanguageContext language_context: The language context to use when building :class:`nunavut.Namespace` objects. :return: The root :class:`nunavut.Namespace`. + :rtype: nunavut.Namespace """ namespace_index = set() # type: typing.Set[str] diff --git a/src/nunavut/_postprocessors.py b/src/nunavut/_postprocessors.py index 9919b0a7..356e12a5 100644 --- a/src/nunavut/_postprocessors.py +++ b/src/nunavut/_postprocessors.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Module containing post processing logic to run on generated files. diff --git a/src/nunavut/_templates.py b/src/nunavut/_templates.py index 8c89eded..5510e03e 100644 --- a/src/nunavut/_templates.py +++ b/src/nunavut/_templates.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Abstractions around template engine internals. diff --git a/src/nunavut/_utilities.py b/src/nunavut/_utilities.py index 7f4e18a8..ecfdff45 100644 --- a/src/nunavut/_utilities.py +++ b/src/nunavut/_utilities.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ A small collection of common utilities. @@ -17,7 +17,7 @@ import enum import logging import pathlib -from typing import Generator, MutableMapping, cast, TypeVar +from typing import Any, Callable, Generator, MutableMapping, Optional, TypeVar, cast, Generic import importlib_resources @@ -113,7 +113,7 @@ class ResourceType(enum.Enum): @enum.unique class ResourceSearchPolicy(enum.Enum): """ - Generic policy type for controlling the behaviour of things that seach for resources. + Generic policy type for controlling the behaviour of things that search for resources. """ FIND_ALL = 0 @@ -122,12 +122,24 @@ class ResourceSearchPolicy(enum.Enum): def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pathlib.Path, None, None]: """ - >>> from nunavut._utilities import iter_package_resources - >>> rs = [x for x in iter_package_resources("nunavut.lang", ".py") if x.name == "__init__.py"] - >>> len(rs) - 1 - >>> rs[0].name - '__init__.py' + A generator that yields all the resources in a package that match a given suffix filter. + + Example usage: + + .. invisible-code-block: python + + from nunavut._utilities import iter_package_resources + + .. code-block:: python + + for x in iter_package_resources("nunavut.lang", ".py"): + print(x) + + .. invisible-code-block: python + + rs = [x for x in iter_package_resources("nunavut.lang", ".py") if x.name == "__init__.py"] + assert 1 == len(rs) + assert rs[0].name == '__init__.py' """ for resource in importlib_resources.files(pkg_name).iterdir(): @@ -152,17 +164,158 @@ def empty_list_support_files() -> Generator[pathlib.Path, None, None]: yield from () -DeepUpdateType = TypeVar("DeepUpdateType", bound=MutableMapping) +class DefaultValue: + """ + Represents a default value in the language configuration. Use this to differentiate between explicit values and + default values when merging configuration. For example, given the following configuration: + + .. invisible-code-block: python + + from nunavut import DefaultValue + + .. code-block:: python + + collection = { + 'a': DefaultValue(1), + 'b': 2 + } + + overrides = [ + { + 'a': 3, + 'b': DefaultValue(4) + }, + { + 'a': DefaultValue(5), + 'b': 6 + } + ] + + Then the merged configuration should be: + + .. code-block:: python + + merged = { + 'a': 3, + 'b': 6 + } + + .. invisible-code-block: python + + # let's try it + for override in overrides: + collection = deep_update(collection, override) + + assert collection['a'] == merged['a'] + assert collection['b'] == merged['b'] + + Other properties of DefaultValue: + + .. code-block:: python + + assert DefaultValue(1) == 1 + assert DefaultValue(1) != 2 + assert DefaultValue(1) == DefaultValue(1) + assert DefaultValue(1) != DefaultValue(2) + assert eval(repr(DefaultValue(1))) == DefaultValue(1) + assert hash(DefaultValue(1)) == hash(1) + assert bool(DefaultValue(1)) + assert not bool(DefaultValue(None)) + repred = eval(repr(DefaultValue(8))) + assert repred.value == 8 + + """ + + @classmethod + def assign_to_if_not_default(cls, target: MutableMapping[str, Any], key: str, value: Any) -> Any: + """ + Assigns a value to a key in a dictionary unless the key already has a value and the value is not a + `DefaultValue`. The one exception to this is if the value is a `DefaultValue` and the value for the key is + already a `DefaultValue`. In this case the new `DefaultValue` value will be assigned to the key. + + :param target: The dictionary to assign to. + :param key: The key to assign to. + :param value: The value to test and assign. + :return: The value assigned to the key. This is the value of the `value` parameter if it was assigned or the + value of the key in the target dictionary if it was not assigned. + """ + try: + if isinstance(value, DefaultValue) and not isinstance(target[key], DefaultValue): + return target[key] + except KeyError: + pass + target[key] = value + return value + + def __init__(self, value: Any) -> None: + self._value = value + + @property + def value(self) -> Any: + """ + The default value. + """ + return self._value + + def __eq__(self, other: Any) -> bool: + if isinstance(other, DefaultValue): + return bool(self._value == other.value) + return bool(self._value == other) + + def __ne__(self, other: Any) -> bool: + return not self.__eq__(other) + + def __repr__(self) -> str: + return f"DefaultValue({self.value})" + + def __str__(self) -> str: + return f"DefaultValue({self.value})" + + def __hash__(self) -> int: + return hash(self._value) + + def __bool__(self) -> bool: + return bool(self._value) + +def no_default_value(func: Callable[..., Any]) -> Callable[..., Any]: + """ + Decorator to convert a function that may return `DefaultValue`s to a function that returns the value of the any + `DefaultValue`s found. For example: + + .. invisible-code-block: python + + from nunavut._utilities import DefaultValue, no_default_value + + .. code-block:: python -def deep_update(target: DeepUpdateType, source: DeepUpdateType) -> DeepUpdateType: + @no_default_value + def some_function() -> DefaultValue: + return DefaultValue(1) + + assert some_function() == 1 + assert not isinstance(some_function(), DefaultValue) + """ + + def wrapper(*args: Any, **kwargs: Any) -> Any: + result = func(*args, **kwargs) + if isinstance(result, DefaultValue): + return result.value + return result + + return wrapper + + +DeepUpdateT = TypeVar("DeepUpdateT", bound=MutableMapping) + + +def deep_update(target: DeepUpdateT, source: DeepUpdateT) -> DeepUpdateT: """ Helper method to do a recursive update of a map that may contain maps as values. .. invisible-code-block: python from nunavut._utilities import deep_update - import collections.abc .. code-block:: python @@ -186,13 +339,106 @@ def deep_update(target: DeepUpdateType, source: DeepUpdateType) -> DeepUpdateTyp assert "c" in target_map assert target_map["c"] == "see" + Note that this method is `DefaultValue` aware. If a value in the target map is a `DefaultValue` then it will not + overwrite the value in the target map. If the value in the source map is a `DefaultValue` then it will not be + used to update existing values of any type in the target map but will be used to update the target map if the + target map does not have a value for the given key. In such cases the `DefaultValue` will be inserted into the + target map. + + .. code-block:: python + + from nunavut import DefaultValue + target_map = { + "a": { "one": 1, "two": DefaultValue(2) }, + "b": "not a default", + "c": DefaultValue("one default...") + } + update_from = { + "a": { "two": { "i": "this value" }, "three": DefaultValue("that value")}, + "b": DefaultValue("see"), + "c": DefaultValue("...deserves another."), + "d": DefaultValue("This happened.") + } + + target_map = deep_update(target_map, update_from) + + assert target_map["a"]["one"] == 1 + assert target_map["a"]["two"]["i"] == "this value" + assert target_map["a"]["three"] == "that value" + assert target_map["b"] == "not a default" + assert target_map["c"] == "...deserves another." + assert target_map["d"] == "This happened." + """ if isinstance(target, collections.abc.Mapping): for key, value in source.items(): if isinstance(value, collections.abc.Mapping): - target[key] = deep_update(target.get(key, {}), cast(DeepUpdateType, value)) + target[key] = deep_update(target.get(key, {}), cast(DeepUpdateT, value)) else: - target[key] = value + DefaultValue.assign_to_if_not_default(target, key, value) else: target = copy.copy(source) return target + + +PropertyT = TypeVar("PropertyT") + + +class cached_property(Generic[PropertyT]): + """ + Based on `functools.cached_property` (Python Foundation License 2.0, SPDX: PSF-2.0) implementation in Python 3.11, + this is both a backport for older Python versions and a version that omits the problematic lock as documented for + Python 3.12. As such, this version is not thread safe. + + :param func: The function to be wrapped by this decorator. + + .. invisible-code-block: python + + from nunavut._utilities import cached_property + + class Test: + + @classmethod + @cached_property + def cls_test(cls) -> int: + return 1 + + def __init__(self) -> None: + self.calls = 0 + + @cached_property + def test(self) -> int: + self.calls += 1 + return self.calls + + t = Test() + assert t.test == 1 + assert t.test == 1 + assert t.test == 1 + try: + _ = t.cls_test + assert False + except TypeError: + pass + + """ + + _NOT_FOUND = object() + + def __init__(self, func: Callable[..., PropertyT]): + self._func = func + self._attr_name: Optional[str] = None + self.__doc__ = func.__doc__ + + def __set_name__(self, owner: Any, name: str) -> None: + self._attr_name = name + + def __get__(self, instance: Any, owner: Optional[Any] = None) -> PropertyT: + if self._attr_name is None: + raise TypeError("Cannot use cached_property instance without calling __set_name__ on it.") + cache = instance.__dict__ + val = cast(PropertyT, cache.get(self._attr_name, self._NOT_FOUND)) + if val is self._NOT_FOUND: + val = self._func(instance) + cache[self._attr_name] = val + return val diff --git a/src/nunavut/_version.py b/src/nunavut/_version.py index 29d43911..b5c8fde9 100644 --- a/src/nunavut/_version.py +++ b/src/nunavut/_version.py @@ -1,14 +1,14 @@ # -# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2022 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ .. autodata:: __version__ """ -__version__ = "2.3.2.dev0" +__version__ = "2.3.3.dev0" __license__ = "MIT" __author__ = "OpenCyphal" __copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. Copyright (c) 2023 OpenCyphal." diff --git a/src/nunavut/cli/__init__.py b/src/nunavut/cli/__init__.py index 63807a14..ca2c8907 100644 --- a/src/nunavut/cli/__init__.py +++ b/src/nunavut/cli/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Command-line for using nunavut and jinja to generate code @@ -23,6 +23,8 @@ class _LazyVersionAction(argparse._VersionAction): if the --version action is requested. """ + # pylint: disable=protected-access + def __call__( self, parser: argparse.ArgumentParser, @@ -30,6 +32,7 @@ def __call__( values: typing.Any, option_string: typing.Optional[str] = None, ) -> None: + # pylint: disable=import-outside-toplevel from nunavut._version import __version__ parser._print_message(__version__, sys.stdout) @@ -41,9 +44,7 @@ class _NunavutArgumentParser(argparse.ArgumentParser): Specialization of argparse.ArgumentParser to encapsulate inter-argument rules. """ - def parse_known_args( - self, args: typing.Optional[typing.Sequence[str]] = None, namespace: typing.Optional[argparse.Namespace] = None - ) -> typing.Tuple[argparse.Namespace, typing.List[str]]: + def parse_known_args(self, args=None, namespace=None): # type: ignore parsed_args, argv = super().parse_known_args(args, namespace) self._post_process_args(parsed_args) return (parsed_args, argv) @@ -96,7 +97,6 @@ def _make_parser() -> argparse.ArgumentParser: parser.add_argument( "--lookup-dir", "-I", - default=[], action="append", help=textwrap.dedent( """ @@ -280,7 +280,6 @@ def extension_type(raw_arg: str) -> str: parser.add_argument( "--namespace-output-stem", - default=None, help="The name of the file generated when --generate-namespace-types is provided.", ) @@ -355,6 +354,39 @@ def extension_type(raw_arg: str) -> str: ) parser.add_argument( + "--embed-auditing-info", + action="store_true", + help=textwrap.dedent( + """ + + If set, generators are instructed to add additional information in the form of + language-specific comments or meta-data to use when auditing source code generated by + Nunavut. This data may change based on the environment in use which may interfere with + the reproducibility of your builds. For example, paths to input files used to generate + a type may be included with this option where these paths will be different depending + on the server used to run nnvg. + + """ + ).lstrip(), + ) + + # +-----------------------------------------------------------------------+ + # | Post-Processing Options + # +-----------------------------------------------------------------------+ + + ln_pp_group = parser.add_argument_group( + "post-processing options", + description=textwrap.dedent( + """ + + Options that enable various post-generation steps because Pavel Kirienko doesn't + like writing jinja templates. + + """ + ).lstrip(), + ) + + ln_pp_group.add_argument( "--pp-max-emptylines", type=int, help=textwrap.dedent( @@ -371,7 +403,7 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) - parser.add_argument( + ln_pp_group.add_argument( "--pp-trim-trailing-whitespace", action="store_true", help=textwrap.dedent( @@ -388,7 +420,7 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) - parser.add_argument( + ln_pp_group.add_argument( "-pp-rp", "--pp-run-program", help=textwrap.dedent( @@ -408,7 +440,7 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) - parser.add_argument( + ln_pp_group.add_argument( "-pp-rpa", "--pp-run-program-arg", action="append", @@ -422,6 +454,9 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) + # +-----------------------------------------------------------------------+ + # | Language Options + # +-----------------------------------------------------------------------+ ln_opt_group = parser.add_argument_group( "language options", description=textwrap.dedent( @@ -520,7 +555,7 @@ def extension_type(raw_arg: str) -> str: help=textwrap.dedent( """ - There is a set of built-in configuration for Nunvut that provides default falues for known + There is a set of built-in configuration for Nunavut that provides default values for known languages as documented `in the template guide `_. This argument lets you specify override configuration yamls. @@ -547,7 +582,7 @@ def extension_type(raw_arg: str) -> str: def _extra_includes_from_env(env_var_name: str) -> typing.List[str]: try: extra_includes_from_env = os.environ[env_var_name].split(os.pathsep) - logging.info("Additional include directories from {}: {}".format(env_var_name, str(extra_includes_from_env))) + logging.info("Additional include directories from %s: %s", env_var_name, str(extra_includes_from_env)) return extra_includes_from_env except KeyError: return [] @@ -575,13 +610,14 @@ def main() -> int: # # Parse DSDL_INCLUDE_PATH # - extra_includes = args.lookup_dir + extra_includes: typing.List[str] = args.lookup_dir if args.lookup_dir is not None else [] extra_includes_from_env = _extra_includes_from_env("DSDL_INCLUDE_PATH") extra_includes += sorted(extra_includes_from_env) + # pylint: disable=import-outside-toplevel from nunavut.cli.runners import ArgparseRunner - runner = ArgparseRunner(args, extra_includes) + runner = ArgparseRunner(args.root_namespace, args, extra_includes) runner.run() return 0 diff --git a/src/nunavut/cli/runners.py b/src/nunavut/cli/runners.py index f6f8ee58..178034e2 100644 --- a/src/nunavut/cli/runners.py +++ b/src/nunavut/cli/runners.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Objects that utilize command-line inputs to run a program using Nunavut. @@ -23,7 +23,7 @@ SetFileMode, TrimTrailingWhitespace, ) -from nunavut._utilities import YesNoDefault +from nunavut._utilities import DefaultValue, YesNoDefault from nunavut.lang import Language, LanguageContext, LanguageContextBuilder @@ -31,12 +31,18 @@ class ArgparseRunner: """ Runner that uses Python argparse arguments to define a run. - :param argparse.Namespace args: The commandline arguments. + :param root_namespace: The root namespace to generate code for. + :param argparse.Namespace args: The command line arguments. :param typing.Optional[typing.Union[str, typing.List[str]]] extra_includes: A list of paths to additional DSDL root folders. """ - def __init__(self, args: argparse.Namespace, extra_includes: typing.Optional[typing.Union[str, typing.List[str]]]): + def __init__( + self, + root_namespace: pathlib.Path, + args: argparse.Namespace, + extra_includes: typing.Optional[typing.Union[str, typing.List[str]]], + ): self._args = args if extra_includes is None: @@ -53,7 +59,7 @@ def __init__(self, args: argparse.Namespace, extra_includes: typing.Optional[typ if self._args.generate_support != "only" and not self._args.list_configuration: type_map = read_dsdl_namespace( - self._args.root_namespace, + root_namespace, self._extra_includes, allow_unregulated_fixed_port_id=self._args.allow_unregulated_fixed_port_id, ) @@ -61,13 +67,12 @@ def __init__(self, args: argparse.Namespace, extra_includes: typing.Optional[typ type_map = [] self._root_namespace = build_namespace_tree( - type_map, self._args.root_namespace, self._args.outdir, self._language_context + type_map, str(root_namespace), self._args.outdir, self._language_context ) # # nunavut : create generators # - generator_args = { "generate_namespace_types": ( YesNoDefault.YES if self._args.generate_namespace_types else YesNoDefault.DEFAULT @@ -112,8 +117,7 @@ def run(self) -> None: def _should_generate_support(self) -> bool: if self._args.generate_support == "as-needed": return self._args.omit_serialization_support is None or not self._args.omit_serialization_support - else: - return bool(self._args.generate_support == "always" or self._args.generate_support == "only") + return bool(self._args.generate_support in ("always", "only")) def _build_ext_program_postprocessor(self, program: str) -> FilePostProcessor: subprocess_args = [program] @@ -140,12 +144,18 @@ def _build_post_processor_list_from_args(self) -> typing.List[PostProcessor]: return post_processors def _create_language_context(self) -> LanguageContext: - language_options = dict() + language_options = {} if self._args.target_endianness is not None: language_options["target_endianness"] = self._args.target_endianness - language_options["omit_float_serialization_support"] = self._args.omit_float_serialization_support - language_options["enable_serialization_asserts"] = self._args.enable_serialization_asserts - language_options["enable_override_variable_array_capacity"] = self._args.enable_override_variable_array_capacity + language_options["omit_float_serialization_support"] = ( + True if self._args.omit_float_serialization_support else DefaultValue(False) + ) + language_options["enable_serialization_asserts"] = ( + True if self._args.enable_serialization_asserts else DefaultValue(False) + ) + language_options["enable_override_variable_array_capacity"] = ( + True if self._args.enable_override_variable_array_capacity else DefaultValue(False) + ) if self._args.language_standard is not None: language_options["std"] = self._args.language_standard @@ -162,9 +172,7 @@ def _create_language_context(self) -> LanguageContext: include_experimental_languages=self._args.experimental_languages ) builder.set_target_language(target_language_name) - builder.load_default_config(self._args.language_standard) - builder.set_additional_config_files(additional_config_files) - builder.validate_langauge_options() + builder.add_config_files(*additional_config_files) builder.set_target_language_extension(self._args.output_extension) builder.set_target_language_configuration_override( Language.WKCV_NAMESPACE_FILE_STEM, self._args.namespace_output_stem @@ -184,10 +192,10 @@ def _stdout_lister( def _list_outputs_only(self) -> None: if self._args.generate_support != "only": - self._stdout_lister(self._generator.generate_all(is_dryrun=True), lambda p: str(p)) + self._stdout_lister(self._generator.generate_all(is_dryrun=True), str) if self._should_generate_support(): - self._stdout_lister(self._support_generator.generate_all(is_dryrun=True), lambda p: str(p)) + self._stdout_lister(self._support_generator.generate_all(is_dryrun=True), str) def _list_inputs_only(self) -> None: if self._args.generate_support != "only": @@ -216,7 +224,7 @@ def _list_inputs_only(self) -> None: def _list_configuration_only(self) -> None: lctx = self._language_context - import yaml + import yaml # pylint: disable=import-outside-toplevel sys.stdout.write("target_language: '") sys.stdout.write(lctx.get_target_language().name) @@ -230,6 +238,7 @@ def _generate(self) -> None: is_dryrun=self._args.dry_run, allow_overwrite=not self._args.no_overwrite, omit_serialization_support=self._args.omit_serialization_support, + embed_auditing_info=self._args.embed_auditing_info, ) if self._args.generate_support != "only": @@ -237,4 +246,5 @@ def _generate(self) -> None: is_dryrun=self._args.dry_run, allow_overwrite=not self._args.no_overwrite, omit_serialization_support=self._args.omit_serialization_support, + embed_auditing_info=self._args.embed_auditing_info, ) diff --git a/src/nunavut/jinja/__init__.py b/src/nunavut/jinja/__init__.py index ce93bb87..e0eba659 100644 --- a/src/nunavut/jinja/__init__.py +++ b/src/nunavut/jinja/__init__.py @@ -1,12 +1,13 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ jinja-based :class:`~nunavut.generators.AbstractGenerator` implementation. """ +import abc import datetime import io import logging @@ -15,15 +16,16 @@ import shutil import typing -import nunavut._generators -import nunavut.lang -import nunavut._postprocessors import pydsdl -from nunavut._utilities import ResourceType, YesNoDefault, ResourceSearchPolicy, TEMPLATE_SUFFIX from yaml import Dumper as YamlDumper from yaml import dump as yaml_dump -from .environment import CodeGenEnvironment +import nunavut._generators +import nunavut._postprocessors +import nunavut.lang +from nunavut._utilities import TEMPLATE_SUFFIX, ResourceSearchPolicy, ResourceType, YesNoDefault + +from .environment import CodeGenEnvironmentBuilder from .jinja2 import Template from .loaders import DEFAULT_TEMPLATE_PATH, DSDLTemplateLoader @@ -93,7 +95,7 @@ def __augment_post_processors_with_ln_limit_empty_lines( """ Subroutine of _handle_post_processors method. """ - from nunavut._postprocessors import LimitEmptyLines + from nunavut._postprocessors import LimitEmptyLines # pylint: disable=import-outside-toplevel if post_processors is None: post_processors = [LimitEmptyLines(limit_empty_lines)] @@ -114,7 +116,7 @@ def __augment_post_processors_with_ln_trim_trailing_whitespace( """ Subroutine of _handle_post_processors method. """ - from nunavut._postprocessors import TrimTrailingWhitespace + from nunavut._postprocessors import TrimTrailingWhitespace # pylint: disable=import-outside-toplevel if post_processors is None: post_processors = [TrimTrailingWhitespace()] @@ -193,22 +195,32 @@ def __init__( self._post_processors = self._handle_post_processors(target_language, post_processors) - self._env = CodeGenEnvironment( - lctx=language_context, - loader=self._dsdl_template_loader, - lstrip_blocks=lstrip_blocks, - trim_blocks=trim_blocks, - additional_filters=additional_filters, - additional_tests=additional_tests, - additional_globals=additional_globals, + env_builder = ( + CodeGenEnvironmentBuilder(self._dsdl_template_loader, language_context) + .set_trim_blocks(trim_blocks) + .set_lstrip_blocks(lstrip_blocks) ) + if additional_filters is not None: + env_builder.add_filters(**additional_filters) + if additional_tests is not None: + env_builder.add_tests(**additional_tests) + if additional_globals is not None: + env_builder.add_globals(**additional_globals) + + self._env = env_builder.create() @property def dsdl_loader(self) -> DSDLTemplateLoader: + """ + The template loader used by this generator. + """ return self._dsdl_template_loader @property def language_context(self) -> nunavut.lang.LanguageContext: + """ + The language context used by this generator. + """ return self._namespace.get_language_context() # +-----------------------------------------------------------------------+ @@ -219,7 +231,7 @@ def _handle_overwrite(self, output_path: pathlib.Path, allow_overwrite: bool) -> if allow_overwrite: output_path.chmod(output_path.stat().st_mode | 0o220) else: - raise PermissionError("{} exists and allow_overwrite is False.".format(output_path)) + raise PermissionError("{output_path} exists and allow_overwrite is False.") # +-----------------------------------------------------------------------+ # | AbstractGenerator @@ -234,6 +246,16 @@ def get_templates(self, omit_serialization_support: bool = False) -> typing.Iter """ return self._dsdl_template_loader.get_templates() + @abc.abstractmethod + def generate_all( + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, + ) -> typing.Iterable[pathlib.Path]: + raise NotImplementedError() + # +-----------------------------------------------------------------------+ # | PRIVATE # +-----------------------------------------------------------------------+ @@ -314,12 +336,12 @@ def _generate_code( elif isinstance(pp, nunavut._postprocessors.FilePostProcessor): file_pps.append(pp) else: - raise ValueError("PostProcessor type {} is unknown.".format(type(pp))) + raise ValueError(f"PostProcessor type {type(pp)} is unknown.") logger.debug("Using post-processors: %r %r", line_pps, file_pps) self._handle_overwrite(output_path, allow_overwrite) output_path.parent.mkdir(parents=True, exist_ok=True) - with open(str(output_path), "w") as output_file: + with open(str(output_path), "w", encoding="utf-8") as output_file: if len(line_pps) > 0: # The logic gets much more complex when doing line post-processing. self._generate_with_line_buffer(output_file, template_gen, line_pps) @@ -395,7 +417,7 @@ def filter_type_to_template(self, value: typing.Any) -> str: """ result = self.dsdl_loader.type_to_template(type(value)) if result is None: - raise RuntimeError("No template found for type {}".format(type(value))) + raise RuntimeError(f"No template found for type {value}") return result.name def filter_type_to_include_path(self, value: typing.Any, resolve: bool = False) -> str: @@ -462,7 +484,7 @@ def filter_alignment_prefix(offset: pydsdl.BitLengthSet) -> str: # and template = '{{ B | alignment_prefix }}' - # then ('str' is stropped to 'str_' before the version is suffixed) + # outputs rendered = 'aligned' .. invisible-code-block: python @@ -479,7 +501,7 @@ def filter_alignment_prefix(offset: pydsdl.BitLengthSet) -> str: # and template = '{{ B | alignment_prefix }}' - # then ('str' is stropped to 'str_' before the version is suffixed) + # outputs rendered = 'unaligned' .. invisible-code-block: python @@ -493,7 +515,7 @@ def filter_alignment_prefix(offset: pydsdl.BitLengthSet) -> str: if isinstance(offset, pydsdl.BitLengthSet): return "aligned" if offset.is_aligned_at_byte() else "unaligned" else: # pragma: no cover - raise TypeError("Expected BitLengthSet, got {}".format(type(offset).__name__)) + raise TypeError(f"Expected BitLengthSet, got {type(offset).__name__}") @staticmethod def filter_bit_length_set(values: typing.Optional[typing.Union[typing.Iterable[int], int]]) -> pydsdl.BitLengthSet: @@ -521,7 +543,27 @@ def filter_remove_blank_lines(text: str) -> str: from nunavut.jinja import DSDLCodeGenerator import pydsdl - assert DSDLCodeGenerator.filter_remove_blank_lines('123\n \n\n456\n\t\n\v\f\n789') == '123\n456\n789' + .. code-block:: python + + # Given + text = '''123 + + 456 + \t + \v\f + 789''' + + # and + template = '{{ text | remove_blank_lines }}' + + # then the black lines will be removed leaving... + rendered = '''123 + 456 + 789''' + + .. invisible-code-block: python + + jinja_filter_tester(DSDLCodeGenerator.filter_remove_blank_lines, template, rendered, 'c', text=text) """ return re.sub(r"\n([ \t\f\v]*\n)+", r"\n", text) @@ -545,12 +587,56 @@ def filter_bits2bytes_ceil(n_bits: int) -> int: raise ValueError("The number of bits cannot be negative") return (int(n_bits) + 7) // 8 + @staticmethod + def filter_text_table( + data: typing.Dict, start_each_line: str, column_sep: str = " : ", line_end: str = "\n" + ) -> str: + """ + Create a text table from a dictionary of data. + + .. invisible-code-block: python + + from nunavut.jinja import DSDLCodeGenerator + import pydsdl + + .. code-block:: python + + # Given + table = { + "banana": "yellow", + "apple": "red", + "grape": "purple" + } + + # and + template = ''' + {{ table | text_table("// ", " | ", "\\n") }}''' + + # then + rendered = ''' + // banana | yellow + // apple | red + // grape | purple''' + + .. invisible-code-block: python + + jinja_filter_tester(DSDLCodeGenerator.filter_text_table, template, rendered, 'c', table=table) + + """ + # Find the longest key to set the width of the first column + key_width = max(len(key) for key in data.keys()) + + output = [] + for key, value in data.items(): + output.append(f"{start_each_line}{key:<{key_width}}{column_sep}{value}".rstrip()) + return line_end.join(output) + # +-----------------------------------------------------------------------+ # | JINJA : tests # +-----------------------------------------------------------------------+ @staticmethod - def is_None(value: typing.Any) -> bool: + def is_None(value: typing.Any) -> bool: # pylint: disable=invalid-name """ Tests if a value is ``None`` @@ -692,11 +778,17 @@ def __init__(self, namespace: nunavut.Namespace, **kwargs: typing.Any): # +-----------------------------------------------------------------------+ def generate_all( - self, is_dryrun: bool = False, allow_overwrite: bool = True, omit_serialization_support: bool = False + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> typing.Iterable[pathlib.Path]: generated = [] # type: typing.List[pathlib.Path] self._env.update_nunavut_globals( - *self.language_context.get_target_language().get_support_module(), is_dryrun, omit_serialization_support + *self.language_context.get_target_language().get_support_module(), + omit_serialization_support, + embed_auditing_info, ) provider = self.namespace.get_all_types if self.generate_namespace_types else self.namespace.get_all_datatypes for parsed_type, output_path in provider(): @@ -820,10 +912,16 @@ def get_templates(self, omit_serialization_support: bool = False) -> typing.Iter return files def generate_all( - self, is_dryrun: bool = False, allow_overwrite: bool = True, omit_serialization_support: bool = False + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> typing.Iterable[pathlib.Path]: target_language = self.language_context.get_target_language() - self._env.update_nunavut_globals(*target_language.get_support_module(), is_dryrun, omit_serialization_support) + self._env.update_nunavut_globals( + *target_language.get_support_module(), omit_serialization_support, embed_auditing_info + ) target_path = pathlib.Path(self.namespace.get_support_output_folder()) / self._sub_folders line_pps = [] # type: typing.List['nunavut._postprocessors.LinePostProcessor'] @@ -895,8 +993,8 @@ def _copy_header_using_line_pps( target: pathlib.Path, line_pps: typing.List["nunavut._postprocessors.LinePostProcessor"], ) -> None: - with open(str(target), "w") as target_file: - with open(str(resource), "r") as resource_file: + with open(str(target), "w", encoding="utf-8") as target_file: + with open(str(resource), "r", encoding="utf-8") as resource_file: for resource_line in resource_file: if len(resource_line) > 1 and resource_line[-2] == "\r": resource_line_tuple = (resource_line[0:-2], "\r\n") diff --git a/src/nunavut/jinja/environment.py b/src/nunavut/jinja/environment.py index 96ac2c99..9bce2fe4 100644 --- a/src/nunavut/jinja/environment.py +++ b/src/nunavut/jinja/environment.py @@ -1,11 +1,19 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # +# cSpell: words loopcontrols +# +""" +Jinja environment for Nunavut code generation. +""" + import datetime import inspect import logging +import platform +import sys import types import typing @@ -16,7 +24,7 @@ from .jinja2 import BaseLoader, Environment, StrictUndefined, select_autoescape from .jinja2.ext import Extension from .jinja2.ext import do as jinja_do -from .jinja2.ext import loopcontrols as loopcontrols +from .jinja2.ext import loopcontrols from .jinja2.filters import FILTERS as JINJA2_FILTERS logger = logging.getLogger(__name__) @@ -70,8 +78,8 @@ class LanguageTemplateNamespace: """ def __init__(self, **kwargs: typing.Any): - for name in kwargs: - setattr(self, name, kwargs[name]) + for name, value in kwargs.items(): + setattr(self, name, value) def __repr__(self) -> str: type_name = type(self).__name__ @@ -79,12 +87,12 @@ def __repr__(self) -> str: star_args = {} for name, value in self._get_kwargs(): if name.isidentifier(): - arg_strings.append("%s=%r" % (name, value)) + arg_strings.append(f"{name}={repr(value)}") else: star_args[name] = value if star_args: - arg_strings.append("**%s" % repr(star_args)) - return "%s(%s)" % (type_name, ", ".join(arg_strings)) + arg_strings.append(f"**{repr(star_args)}") + return f"{type_name}({','.join(arg_strings)})" def _get_kwargs(self) -> typing.List[typing.Any]: return list(self.__dict__.items()) @@ -98,19 +106,184 @@ def __contains__(self, key: str) -> bool: return key in self.__dict__ def update(self, update_from: typing.Mapping[str, typing.Any]) -> None: + """ + update the namespace with the given values. + """ for key, value in update_from.items(): setattr(self, key, value) def items(self) -> typing.ItemsView[str, typing.Any]: + """ + The items in the namespace. + """ return self.__dict__.items() + def keys(self) -> typing.KeysView[typing.Any]: + """ + The values in the namespace. + """ + return self.__dict__.keys() + def values(self) -> typing.ValuesView[typing.Any]: + """ + The values in the namespace. + """ return self.__dict__.values() # +---------------------------------------------------------------------------+ # | JINJA : CodeGenEnvironment # +---------------------------------------------------------------------------+ +class CodeGenEnvironmentBuilder: + """ + Builder class for creating a CodeGenEnvironment object for code generation. + + :param BaseLoader loader: The loader used to load templates. + :param LanguageContext lctx: The language context used for code generation. + """ + + DEFAULT_JINJA_EXTENSIONS = [jinja_do, loopcontrols, JinjaAssert, UseQuery] + + def __init__(self, loader: BaseLoader, lctx: LanguageContext) -> None: + self._loader = loader + self._lctx = lctx + self._trim_blocks = False + self._lstrip_blocks = False + self._additional_filters: typing.Optional[typing.Dict[str, typing.Callable]] = None + self._additional_tests: typing.Optional[typing.Dict[str, typing.Callable]] = None + self._additional_globals: typing.Optional[typing.Dict[str, typing.Any]] = None + self._extensions = self.DEFAULT_JINJA_EXTENSIONS[:] + self._allow_filter_test_or_use_query_overwrite = False + + @property + def loader(self) -> BaseLoader: + """ + The loader. + + :return: The loader. + :rtype: BaseLoader + """ + return self._loader + + @property + def lctx(self) -> LanguageContext: + """ + The language context. + + :return: The language context. + :rtype: LanguageContext + """ + return self._lctx + + def set_trim_blocks(self, trim_blocks: bool) -> "CodeGenEnvironmentBuilder": + """ + Set the trim blocks flag. + + :param bool trim_blocks: The trim blocks flag. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._trim_blocks = trim_blocks + return self + + def set_lstrip_blocks(self, lstrip_blocks: bool) -> "CodeGenEnvironmentBuilder": + """ + Set the lstrip blocks flag. + + :param bool lstrip_blocks: The lstrip blocks flag. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._lstrip_blocks = lstrip_blocks + return self + + def add_filters(self, **additional_filters: typing.Callable) -> "CodeGenEnvironmentBuilder": + """ + Add filters to the created environment. + + :param typing.Dict[str, typing.Callable] additional_filters: The additional filters. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + if self._additional_filters is None: + self._additional_filters = additional_filters + else: + self._additional_filters.update(additional_filters) + return self + + def add_tests(self, **additional_tests: typing.Callable) -> "CodeGenEnvironmentBuilder": + """ + Add tests to the created environment. + + :param typing.Dict[str, typing.Callable] additional_tests: The additional tests. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + if self._additional_tests is None: + self._additional_tests = additional_tests + else: + self._additional_tests.update(additional_tests) + return self + + def add_globals(self, **additional_globals: typing.Any) -> "CodeGenEnvironmentBuilder": + """ + Add globals so the created environment. + + :param typing.Dict[str, typing.Any] additional_globals: The additional globals. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + if self._additional_globals is None: + self._additional_globals = additional_globals + else: + self._additional_globals.update(additional_globals) + return self + + def set_extensions(self, *extensions: Extension) -> "CodeGenEnvironmentBuilder": + """ + Set the extensions. + + :param typing.List[Extension] extensions: The extensions. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._extensions = list(extensions) + return self + + def set_allow_filter_test_or_use_query_overwrite( + self, allow_filter_test_or_use_query_overwrite: bool + ) -> "CodeGenEnvironmentBuilder": + """ + Allow overwriting of built-in filters, tests, or use queries. + + :param bool allow_filter_test_or_use_query_overwrite: Allow overwrite of built-ins. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._allow_filter_test_or_use_query_overwrite = allow_filter_test_or_use_query_overwrite + return self + + def create(self) -> "CodeGenEnvironment": + """ + Create a CodeGenEnvironment object. + + :return: A CodeGenEnvironment object. + :rtype: CodeGenEnvironment + """ + return CodeGenEnvironment( + self.loader, + self.lctx, + trim_blocks=self._trim_blocks, + lstrip_blocks=self._lstrip_blocks, + additional_filters=self._additional_filters, + additional_tests=self._additional_tests, + additional_globals=self._additional_globals, + extensions=self._extensions, + allow_filter_test_or_use_query_overwrite=self._allow_filter_test_or_use_query_overwrite, + ) + + +# +---------------------------------------------------------------------------+ class CodeGenEnvironment(Environment): @@ -118,11 +291,13 @@ class CodeGenEnvironment(Environment): Jinja Environment optimized for compile-time generation of source code (i.e. as opposed to dynamically generating webpages). + Do not insatiate directly. Use the :class:`CodeGenEnvironmentBuilder` to create an instance. + .. invisible-code-block: python from nunavut.lang import LanguageContext, LanguageContextBuilder from nunavut.lang._language import Language - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader lctx = LanguageContextBuilder().create() @@ -131,7 +306,7 @@ class CodeGenEnvironment(Environment): template = 'Hello World' - e = CodeGenEnvironment(loader=DictLoader({'test': template}), lctx=lctx) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx).create() assert 'Hello World' == e.get_template('test').render() .. warning:: @@ -142,7 +317,11 @@ class CodeGenEnvironment(Environment): .. code-block:: python try: - CodeGenEnvironment(loader=DictLoader({'test': template}), lctx=lctx, additional_globals={'ln': 'bad_ln'}) + ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_globals(ln='bad_ln') + .create() + ) assert False except RuntimeError: pass @@ -152,19 +331,23 @@ class CodeGenEnvironment(Environment): .. code-block:: python try: - CodeGenEnvironment(loader=DictLoader({'test': template}), - lctx=lctx, - additional_filters={'indent': lambda x: x}) + ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_filters(indent=lambda x: x) + .create() + ) assert False except RuntimeError: pass # You can allow overwrite of built-ins using the ``allow_filter_test_or_use_query_overwrite`` # argument. - e = CodeGenEnvironment(loader=DictLoader({'test': template}), - lctx=lctx, - additional_filters={'indent': lambda x: x}, - allow_filter_test_or_use_query_overwrite=True) + e = ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_filters(indent=lambda x: x) + .set_allow_filter_test_or_use_query_overwrite(True) + .create() + ) assert 'foo' == e.filters['indent']('foo') ...or that user-defined filters or redefined. @@ -177,9 +360,11 @@ class MyFilters: def filter_misnamed(name: str) -> str: return name - e = CodeGenEnvironment(loader=DictLoader({'test': template}), - lctx=lctx, - additional_filters={'filter_misnamed': lambda x: x}) + e = ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_filters(filter_misnamed=lambda x: x) + .create() + ) try: e.add_conventional_methods_to_environment(MyFilters()) @@ -202,14 +387,14 @@ def __init__( self, loader: BaseLoader, lctx: LanguageContext, - trim_blocks: bool = False, - lstrip_blocks: bool = False, - additional_filters: typing.Optional[typing.Dict[str, typing.Callable]] = None, - additional_tests: typing.Optional[typing.Dict[str, typing.Callable]] = None, - additional_globals: typing.Optional[typing.Dict[str, typing.Any]] = None, - extensions: typing.List[Extension] = [jinja_do, loopcontrols, JinjaAssert, UseQuery], - allow_filter_test_or_use_query_overwrite: bool = False, - ): + trim_blocks: bool, + lstrip_blocks: bool, + additional_filters: typing.Optional[typing.Dict[str, typing.Callable]], + additional_tests: typing.Optional[typing.Dict[str, typing.Callable]], + additional_globals: typing.Optional[typing.Dict[str, typing.Any]], + extensions: typing.Optional[typing.List[Extension]], + allow_filter_test_or_use_query_overwrite: bool, + ): # pylint: disable=too-many-arguments super().__init__( loader=loader, # nosec extensions=extensions, @@ -226,7 +411,7 @@ def __init__( if additional_globals is not None: for global_name, global_value in additional_globals.items(): if global_name in self.RESERVED_GLOBAL_NAMESPACES or global_name in self.RESERVED_GLOBAL_NAMES: - raise RuntimeError('Additional global "{}" uses a reserved global name'.format(global_name)) + raise RuntimeError(f'Additional global "{global_name}" uses a reserved global name') self.globals[global_name] = global_value self._allow_replacements = allow_filter_test_or_use_query_overwrite @@ -260,6 +445,16 @@ def __init__( self._add_each_to_environment(additional_tests.items(), self.tests, supported_languages=supported_languages) def add_conventional_methods_to_environment(self, obj: typing.Any) -> None: + """ + Adds methods using specific naming conventions to the Jinja environment. For example, methods named `filter_*` + are added to the Jinja environment as filters. + + This method iterates over the methods of the given object and adds them to the Jinja environment. + Only methods that are supported by the specified languages are added. + + :param typing.Any obj: The object to add the methods from. + + """ for name, method in inspect.getmembers(obj, inspect.isroutine): try: self._add_conventional_method_to_environment(method, name, supported_languages=self.supported_languages) @@ -270,11 +465,24 @@ def update_nunavut_globals( self, support_namespace: str = "", support_version: typing.Tuple[int, int, int] = (0, 0, 0), - support_module: typing.Optional["types.ModuleType"] = None, - is_dryrun: bool = False, + support_module: typing.Optional["types.ModuleType"] = None, # pylint: disable=unused-argument omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> None: + """ + Update the global properties available to templates as `nunavut`. + :param support_namespace: The name of a generated namespace for support code. Available as + `nunavut.support.namespace` in templates. + :param support_version: The version to report for supporting code. Available as + `nunavut.support.version` in templates. + :param support_module: The python module containing support for the selected language. + :param omit_serialization_support: Boolean flag set on the support object. Available as + `nunavut.support.omit_serialization_support` in templates. + :param embed_auditing_info: Boolean flag available as `nunavut.embed_auditing_info` in templates. + """ nunavut_namespace = self.nunavut_global + setattr(nunavut_namespace, "embed_auditing_info", embed_auditing_info) + setattr(nunavut_namespace, "platform_version", self._create_platform_version(embed_auditing_info)) setattr( nunavut_namespace, @@ -282,43 +490,88 @@ def update_nunavut_globals( {"omit": omit_serialization_support, "namespace": support_namespace, "version": support_version}, ) - if "version" not in nunavut_namespace: - from nunavut import __version__ as nunavut_version + if "template_sets" not in nunavut_namespace: + # pylint: disable=import-outside-toplevel from nunavut.jinja.loaders import DSDLTemplateLoader - setattr(nunavut_namespace, "version", nunavut_version) - setattr(nunavut_namespace, "platform_version", self._create_platform_version()) - if isinstance(self.loader, DSDLTemplateLoader): setattr(nunavut_namespace, "template_sets", self.loader.get_template_sets()) + if "version" not in nunavut_namespace: + # pylint: disable=import-outside-toplevel + from nunavut import __version__ as nunavut_version + + setattr(nunavut_namespace, "version", nunavut_version) + @property def supported_languages(self) -> typing.ValuesView[Language]: + """ + The supported languages in the environment. + + :return: A view of the supported languages. + :rtype: typing.ValuesView[Language] + """ ln_globals = self.globals["ln"] # type: LanguageTemplateNamespace return ln_globals.values() @property def nunavut_global(self) -> LanguageTemplateNamespace: + """ + The `nunavut` global namespace. + + :return: The `nunavut` global namespace. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["nunavut"]) @property def target_language_uses_queries(self) -> LanguageTemplateNamespace: + """ + All `uses_queries` for the target language. + + :return: The uses queries for the target language. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["uses_queries"]) @property def language_options(self) -> LanguageTemplateNamespace: + """ + The language options. + + :return: The language options. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["options"]) @property def language_support(self) -> LanguageTemplateNamespace: + """ + The language support. + + :return: The language support. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["ln"]) @property def target_language(self) -> Language: + """ + The target language. + + :return: The target language. + :rtype: Language + """ return self._target_language @property def now_utc(self) -> datetime.datetime: + """ + Get or set the current UTC time. + + :return: The current UTC time. + :rtype: datetime.datetime + """ return typing.cast(datetime.datetime, self.globals["now_utc"]) @now_utc.setter @@ -326,6 +579,13 @@ def now_utc(self, utc_time: datetime.datetime) -> None: self.globals["now_utc"] = utc_time def add_test(self, test_name: str, test_callable: typing.Callable) -> None: + """ + Add a test to the environment. + + :param str test_name: The name of the test. + :param typing.Callable test_callable: The test. + :return: None + """ self._add_to_environment(test_name, test_callable, self.tests) # +----------------------------------------------------------------------------------------------------------------+ @@ -337,22 +597,23 @@ def _resolve_collection( method_name: str, collection_maybe: typing.Optional[typing.Union[LanguageTemplateNamespace, typing.Dict[str, typing.Any]]], ) -> typing.Union[LanguageTemplateNamespace, typing.Dict[str, typing.Any]]: + """ + Resolve the collection to add the item to. If collection_maybe is not None then it is returned otherwise the + collection is resolved based on the method name. + """ if collection_maybe is not None: return collection_maybe if LanguageEnvironment.is_test_name(conventional_method_prefix): return typing.cast(typing.Dict[str, typing.Any], self.tests) - elif LanguageEnvironment.is_filter_name(conventional_method_prefix): + if LanguageEnvironment.is_filter_name(conventional_method_prefix): return typing.cast(typing.Dict[str, typing.Any], self.filters) - elif LanguageEnvironment.is_uses_query_name(conventional_method_prefix): + if LanguageEnvironment.is_uses_query_name(conventional_method_prefix): uses_queries = self.globals["uses_queries"] return typing.cast(LanguageTemplateNamespace, uses_queries) - else: - raise TypeError( - "Tried to add an item {} to the template environment but we don't know what the item is.".format( - method_name - ) - ) + raise TypeError( + f"Tried to add an item {method_name} to the template environment but we don't know what the item is." + ) def _add_to_environment( self, @@ -362,13 +623,13 @@ def _add_to_environment( ) -> None: if item_name in collection: if not self._allow_replacements: - raise RuntimeError("{} was already defined.".format(item_name)) - elif item_name in JINJA2_FILTERS: - logger.info("Replacing Jinja built-in {}".format(item_name)) + raise RuntimeError(f"{item_name} was already defined.") + if item_name in JINJA2_FILTERS: + logger.info("Replacing Jinja built-in %s", item_name) else: - logger.info('Replacing "{}" which was already defined for this environment.'.format(item_name)) + logger.info('Replacing "%s" which was already defined for this environment.', item_name) else: - logger.debug("Adding {} to environment".format(item_name)) + logger.debug("Adding %s to environment", item_name) if isinstance(collection, LanguageTemplateNamespace): setattr(collection, item_name, item) else: @@ -384,16 +645,20 @@ def _add_conventional_method_to_environment( is_target: bool = False, ) -> None: """ + Add a method using specific naming conventions to the Jinja environment. For example, methods named `filter_*` + are added to the Jinja environment as filters. - :param str callable_name: The name of the callable to use in a template. - :param typing.Callable[..., bool] callable: The named callable. - :param typing.Optional[str] callable_namespace: If provided the namespace to prefix to the callable name. - :return: tuple of name and the callable which might be prepared as a partial function based on decorators. - :raises: RuntimeWarning if the callable requested resources that were not available in this environment. + :param typing.Callable[..., bool] method: The named method. + :param str method_name: The name of the callable to use in a template. + :param typing.Optional[typing.Union[LanguageTemplateNamespace, typing.Dict[str, typing.Any]]] collection_maybe: + The collection to add the method to. If None then the collection is resolved based on the method name. + :param typing.Optional[typing.ValuesView[Language]] supported_languages: The supported languages. + :param typing.Optional[Language] method_language: The language of the method. + :param bool is_target: Whether the method is for the target language. .. invisible-code-block: python - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader from nunavut._templates import template_language_test from unittest.mock import MagicMock @@ -409,10 +674,10 @@ def _add_conventional_method_to_environment( def test_test(language): return True - e = CodeGenEnvironment( - loader=DictLoader({'test': 'hello world'}), - additional_tests={'foo': test_test}, - lctx=lctx + e = ( + CodeGenEnvironmentBuilder(DictLoader({'test': 'hello world'}), lctx) + .add_tests(foo=test_test) + .create() ) assert test_test == e.tests['foo'].func assert e.tests['foo']() @@ -422,7 +687,7 @@ def test_test(language): collection = self._resolve_collection(result[0], method_name, collection_maybe) if method_language is not None: - self._add_to_environment("ln.{}.{}".format(method_language.name, result[1]), result[2], collection) + self._add_to_environment(f"ln.{method_language.name}.{result[1]}", result[2], collection) else: self._add_to_environment(result[1], result[2], collection) if is_target: @@ -447,25 +712,25 @@ def _add_each_to_environment( ) @classmethod - def _create_platform_version(cls) -> typing.Dict[str, typing.Any]: - import platform - import sys + def _create_platform_version(cls, embed_auditing_info: bool) -> typing.Dict[str, typing.Any]: platform_version = {} # type: typing.Dict[str, typing.Any] - platform_version["python_implementation"] = platform.python_implementation() platform_version["python_version"] = platform.python_version() - platform_version["python_release_level"] = sys.version_info[3] - platform_version["python_build"] = platform.python_build() - platform_version["python_compiler"] = platform.python_compiler() - platform_version["python_revision"] = platform.python_revision() + if embed_auditing_info: + platform_version["python_implementation"] = platform.python_implementation() + platform_version["python_release_level"] = sys.version_info[3] + platform_version["python_build"] = platform.python_build() + platform_version["python_compiler"] = platform.python_compiler() + platform_version["python_revision"] = platform.python_revision() - try: - platform_version["python_xoptions"] = sys._xoptions - except AttributeError: # pragma: no cover - platform_version["python_xoptions"] = {} + try: + # pylint: disable=protected-access + platform_version["python_xoptions"] = sys._xoptions + except AttributeError: # pragma: no cover + platform_version["python_xoptions"] = {} - platform_version["runtime_platform"] = platform.platform() + platform_version["runtime_platform"] = platform.platform() return platform_version diff --git a/src/nunavut/jinja/extensions.py b/src/nunavut/jinja/extensions.py index 5d208168..38acf439 100644 --- a/src/nunavut/jinja/extensions.py +++ b/src/nunavut/jinja/extensions.py @@ -1,12 +1,14 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # - +""" +Jinja2 extensions for use with the Nunavut code generator. +""" import typing -from nunavut.jinja.jinja2 import Environment, TemplateAssertionError, UndefinedError, nodes +from nunavut.jinja.jinja2 import TemplateAssertionError, UndefinedError, nodes from nunavut.jinja.jinja2.ext import Extension from nunavut.jinja.jinja2.parser import Parser @@ -23,14 +25,14 @@ class JinjaAssert(Extension): .. invisible-code-block: python from nunavut.jinja.jinja2.exceptions import TemplateAssertionError - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader from nunavut.jinja.extensions import JinjaAssert from nunavut.lang import LanguageContextBuilder - e = CodeGenEnvironment(lctx=LanguageContextBuilder().create(), - loader=DictLoader({'test': template}), - extensions=[JinjaAssert]) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), LanguageContextBuilder().create()) \ + .set_extensions(JinjaAssert) \ + .create() try: e.get_template('test').render() # huh. This should have raised a TemplateAssertionError @@ -38,7 +40,7 @@ class JinjaAssert(Extension): except TemplateAssertionError: pass - This extension also support provding an assertion message: + This extension also support providing an assertion message: .. code-block:: python @@ -46,9 +48,9 @@ class JinjaAssert(Extension): .. invisible-code-block: python - e = CodeGenEnvironment(lctx=LanguageContextBuilder().create(), - loader=DictLoader({'test': template}), - extensions=[JinjaAssert]) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), LanguageContextBuilder().create())\ + .set_extensions(JinjaAssert)\ + .create() try: e.get_template('test').render() # huh. This should have raised a TemplateAssertionError @@ -60,9 +62,6 @@ class JinjaAssert(Extension): tags = set(["assert"]) - def __init__(self, environment: Environment): - super().__init__(environment) - def parse(self, parser: Parser) -> nodes.Node: """ See http://jinja.pocoo.org/docs/2.10/extensions/ for help writing @@ -115,7 +114,7 @@ class UseQuery(Extension): .. invisible-code-block: python from nunavut.jinja.jinja2.exceptions import TemplateAssertionError - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader from nunavut.jinja.extensions import UseQuery from nunavut.lang import LanguageClassLoader @@ -129,9 +128,9 @@ class UseQuery(Extension): lctx.get_target_language = MagicMock(return_value = ln_c) - e = CodeGenEnvironment(lctx=lctx, - loader=DictLoader({'test': template}), - extensions=[UseQuery]) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) \ + .set_extensions(UseQuery)\ + .create() try: result = e.get_template('test').render() @@ -158,9 +157,6 @@ class UseQuery(Extension): tags = set(["ifuses", "ifnuses"]) - def __init__(self, environment: Environment): - super().__init__(environment) - def parse(self, parser: Parser) -> nodes.Node: """ See http://jinja.pocoo.org/docs/2.10/extensions/ for help writing @@ -195,12 +191,12 @@ def parse(self, parser: Parser) -> nodes.Node: node = nodes.If(lineno=parser.stream.current.lineno) result.elif_.append(node) continue - elif token.test("name:elifnuses"): + if token.test("name:elifnuses"): negate = True node = nodes.If(lineno=parser.stream.current.lineno) result.elif_.append(node) continue - elif token.test("name:else"): + if token.test("name:else"): result.else_ = parser.parse_statements( ( "name:endifuses", @@ -221,11 +217,11 @@ def _use_query_common(self, uses_query_name: str, lineno: int, name: str, filena uses_query = typing.cast( typing.Callable[..., bool], getattr(self.environment.target_language_uses_queries, uses_query_name) ) - except AttributeError: + except AttributeError as e: raise UndefinedError( - 'use query "{}" for language "{}" is not defined ' - "(line={}, name={}, filename={})".format(uses_query_name, target_language.name, lineno, name, filename) - ) + f'use query "{uses_query_name}" for language "{target_language.name}" is not defined ' + "(line={lineno}, name={name}, filename={filename})" + ) from e return uses_query() diff --git a/src/nunavut/jinja/loaders.py b/src/nunavut/jinja/loaders.py index a6736e0a..2d82d617 100644 --- a/src/nunavut/jinja/loaders.py +++ b/src/nunavut/jinja/loaders.py @@ -1,8 +1,12 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # +""" +Contains template loaders for Nunavut's Jinja2 environment. +""" + import collections import importlib import logging @@ -55,7 +59,7 @@ def __init__( package_name_for_templates: typing.Optional[str] = None, builtin_template_path: str = DEFAULT_TEMPLATE_PATH, search_policy: ResourceSearchPolicy = ResourceSearchPolicy.FIND_ALL, - **kwargs: typing.Any + **kwargs: typing.Any, ): super().__init__(**kwargs) self._type_to_template_lookup_cache: typing.Dict[pydsdl.Any, pathlib.Path] = dict() @@ -63,8 +67,8 @@ def __init__( if templates_dirs is not None: for templates_dir_item in templates_dirs: if not templates_dir_item.exists(): - raise ValueError("Templates directory {} did not exist?".format(str(templates_dir_item))) - logger.info("Loading templates from file system at {}".format(templates_dirs)) + raise ValueError(f"Templates directory {str(templates_dir_item)} did not exist?") + logger.info("Loading templates from file system at %s", templates_dirs) self._fsloader = FileSystemLoader((str(d) for d in templates_dirs), followlinks=followlinks) else: self._fsloader = None @@ -221,7 +225,7 @@ def type_to_template(self, value_type: typing.Type) -> typing.Optional[pathlib.P # +----------------------------------------------------------------------------------------------------------------+ @staticmethod def _filter_template_list_by_suffix(files: typing.List[str]) -> typing.List[str]: - return [f for f in files if (pathlib.Path(f).suffix == TEMPLATE_SUFFIX)] + return [f for f in files if pathlib.Path(f).suffix == TEMPLATE_SUFFIX] def _type_to_template_internal( self, value_type: typing.Type, templates: typing.Mapping[str, pathlib.Path] @@ -241,9 +245,9 @@ def _type_to_template_internal( try: logging.debug( - "NunavutTemplateLoader.type_to_template for {}: considering {}...".format( - value_type.__name__, current_search_type.__name__ - ) + "NunavutTemplateLoader.type_to_template for %s: considering %s...", + value_type.__name__, + current_search_type.__name__, ) template_path = templates[current_search_type.__name__] self._type_to_template_lookup_cache[current_search_type] = template_path diff --git a/src/nunavut/lang/__init__.py b/src/nunavut/lang/__init__.py index c4d51c98..22a7946e 100644 --- a/src/nunavut/lang/__init__.py +++ b/src/nunavut/lang/__init__.py @@ -1,9 +1,10 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # -"""Language-specific support in nunavut. +""" +Language-specific support in nunavut. This package contains modules that provide specific support for generating source for various languages using templates. @@ -13,9 +14,8 @@ import pathlib import typing -from ._config import LanguageConfig as LanguageConfig -from ._language import Language as Language -from ._language import LanguageClassLoader as LanguageClassLoader +from ._config import LanguageConfig +from ._language import Language, LanguageClassLoader logger = logging.getLogger(__name__) @@ -25,12 +25,10 @@ class UnsupportedLanguageError(ValueError): Error type raised if an unsupported language type is used. """ - pass - class LanguageContextBuilder: """ - Used to instatiate new :class:`LanguageContext` objects. + Used to instantiate new :class:`LanguageContext` objects. The simplest invocation will always work by using the :data:`LanguageContextBuilder.DEFAULT_TARGET_LANGUAGE` constant: @@ -83,6 +81,9 @@ def get_supported_language_names(self) -> typing.Iterable[str]: @property def config(self) -> LanguageConfig: + """ + The configuration object that will be used to create the language context. + """ return self._ln_loader.config # +-----------------------------------------------------------------------+ @@ -92,7 +93,7 @@ def config(self) -> LanguageConfig: def set_target_language_configuration_override(self, key: str, value: typing.Any) -> "LanguageContextBuilder": """ Stores a key and value to override in the configuration for a language target when a LanguageContext is crated. - These overrides are always set under the language section of the target langauge. + These overrides are always set under the language section of the target language. .. invisible-code-block: python @@ -114,7 +115,7 @@ def set_target_language_configuration_override(self, key: str, value: typing.Any builder.set_target_language_configuration_override(Language.WKCV_DEFINITION_FILE_EXTENSION, ".foo") - ...but that value will not be overriden until you create the target language: + ...but that value will not be overridden until you create the target language: .. code-block:: python @@ -132,7 +133,7 @@ def set_target_language_configuration_override(self, key: str, value: typing.Any assert overridden_c_file_extension == ".foo" - Note that the config is scoped by the builder but is then inherited by the langauge objects created by the + Note that the config is scoped by the builder but is then inherited by the language objects created by the builder: .. code-block:: python @@ -220,15 +221,16 @@ def set_target_language(self, target_language: typing.Optional[str]) -> "Languag self._target_language_name = LanguageClassLoader.to_language_name(target_language) return self - def load_default_config(self, language_standard: str) -> None: - self._ln_loader.config.apply_defaults(language_standard) - - def validate_langauge_options(self) -> None: - self._ln_loader.config.validate_language_options() - def set_additional_config_files( self, additional_config_files: typing.List[pathlib.Path] ) -> "LanguageContextBuilder": + """ + Deprecated. Use :func:`add_config_files` instead. + """ + logger.warning("set_additional_config_files is deprecated. Use add_config_files instead.") + return self.add_config_files(*additional_config_files) + + def add_config_files(self, *additional_config_files: pathlib.Path) -> "LanguageContextBuilder": """ A list of paths to additional yaml files to load as configuration. These will override any values found in the :file:`nunavut.lang.properties.yaml` file and files @@ -241,13 +243,13 @@ def set_additional_config_files( import textwrap from nunavut.lang import LanguageContextBuilder, Language, LanguageClassLoader - overrides_file = gen_paths.out_dir / pathlib.Path("overrides1.yaml") + overrides_file = gen_paths_for_module.out_dir / pathlib.Path("overrides1.yaml") overrides_data = {LanguageClassLoader.to_language_module_name("c"): {Language.WKCV_DEFINITION_FILE_EXTENSION: ".foo"} } - with open(overrides_file, "w") as overrides_handle: + with open(overrides_file, "w", encoding="utf-8") as overrides_handle: yaml.dump(overrides_data, overrides_handle) .. code-block:: python @@ -255,7 +257,7 @@ def set_additional_config_files( target_language_w_overrides = ( LanguageContextBuilder() .set_target_language("c") - .set_additional_config_files([overrides_file]) + .add_config_files(overrides_file) .create() .get_target_language() ) @@ -270,7 +272,7 @@ def set_additional_config_files( assert target_language_w_overrides.extension == ".foo" assert target_language_no_overrides.extension == ".h" - Overrides are applies as unions. For example, given this override data: + Overrides are applied as unions. For example, given this override data: .. code-block:: python @@ -284,8 +286,8 @@ def set_additional_config_files( .. invisible-code-block: python - second_overrides_file = gen_paths.out_dir / pathlib.Path("overrides2.yaml") - with open(second_overrides_file, "w") as overrides_handle: + second_overrides_file = gen_paths_for_module.out_dir / pathlib.Path("overrides2.yaml") + with open(second_overrides_file, "w", encoding="utf-8") as overrides_handle: overrides_handle.write(textwrap.dedent(overrides_data)) .. code-block:: python @@ -293,7 +295,7 @@ def set_additional_config_files( target_language_w_overrides = ( LanguageContextBuilder() .set_target_language("c") - .set_additional_config_files([second_overrides_file]) + .add_config_files(second_overrides_file) .create() .get_target_language() ) @@ -301,16 +303,69 @@ def set_additional_config_files( assert ".foo" == target_language_w_overrides.extension assert "bar" == target_language_w_overrides.get_config_value("non-standard") + .. invisible-code-block: python + + from nunavut import DefaultValue + + # verification of issue #329 fix + with_default = {"enable_serialization_asserts" : DefaultValue(False) } + without_default = {"enable_serialization_asserts" : False } + + # verification of issue #329 fix + overrides_data = ''' + nunavut.lang.c: + options: + enable_serialization_asserts: true + ''' + + issue_329_overrides = gen_paths_for_module.out_dir / pathlib.Path("overrides329.yaml") + with open(issue_329_overrides, "w", encoding="utf-8") as overrides_handle: + overrides_handle.write(textwrap.dedent(overrides_data)) + + target_language_329_no_file_overrides = ( + LanguageContextBuilder() + .set_target_language("c") + .set_target_language_configuration_override(Language.WKCV_LANGUAGE_OPTIONS, with_default) + .create() + .get_target_language() + ) + + target_language_329_file_override = ( + LanguageContextBuilder() + .set_target_language("c") + .add_config_files(issue_329_overrides) + .set_target_language_configuration_override(Language.WKCV_LANGUAGE_OPTIONS, with_default) + .create() + .get_target_language() + ) + + target_language_329_file_override_overridden = ( + LanguageContextBuilder() + .set_target_language("c") + .add_config_files(issue_329_overrides) + .set_target_language_configuration_override(Language.WKCV_LANGUAGE_OPTIONS, without_default) + .create() + .get_target_language() + ) + + # default from command line + assert not target_language_329_no_file_overrides.get_option("enable_serialization_asserts") + + # default from command line overridden by file + assert target_language_329_file_override.get_option("enable_serialization_asserts") + + # command-line overrides file + assert not target_language_329_file_override_overridden.get_option("enable_serialization_asserts") """ for additional_path in additional_config_files: - with open(str(additional_path), "r") as additional_file: - self._ln_loader.config.update_from_file(additional_file) + with open(str(additional_path), "r", encoding="utf-8") as additional_file: + self.config.update_from_yaml_file(additional_file) return self def create(self) -> "LanguageContext": """ - Applies all pending configuration overrides to the internal :class:`LanguageConfig` object and instatiates + Applies all pending configuration overrides to the internal :class:`LanguageConfig` object and instantiates a :class:`LanguageContext` object. """ # First find the target language to use... @@ -337,13 +392,11 @@ def _new_language_w_experimental_handling(self, language_name: str) -> Language: try: language = self._ln_loader.new_language(language_name) except ImportError as e: - logger.debug("Import Error {} when trying to load language {}".format(str(e), language_name)) - raise KeyError("language {} is not a supported language".format(language_name)) + logger.debug("Import Error %s when trying to load language %s", e, language_name) + raise KeyError(f"language {language_name} is not a supported language") from e if not (language.stable_support or self._include_experimental_languages): raise UnsupportedLanguageError( - "{} support is only experimental, but experimental language support is not enabled".format( - language_name - ) + f"{language_name} support is only experimental, but experimental language support is not enabled" ) return language @@ -377,9 +430,8 @@ def _resolve_target_language(self, explicit_value: typing.Optional[str]) -> str: if inferred_target_language_name is None: inferred_target_language_name = self.DEFAULT_TARGET_LANGUAGE logger.info( - "No target language specified and none could be inferred. Using default language, {}".format( - self.DEFAULT_TARGET_LANGUAGE - ) + "No target language specified and none could be inferred. Using default language, %s", + self.DEFAULT_TARGET_LANGUAGE, ) else: logging.info( @@ -472,4 +524,8 @@ def get_supported_languages(self) -> typing.Dict[str, Language]: @property def config(self) -> LanguageConfig: + """ + Returns the :class:`nunavut.lang.LanguageConfig` object that contains the configuration for all + supported languages. This is the same object that is used to instantiate the :class:`nunavut.lang.Language` + """ return self._config diff --git a/src/nunavut/lang/_common.py b/src/nunavut/lang/_common.py index cc0bb289..55733854 100644 --- a/src/nunavut/lang/_common.py +++ b/src/nunavut/lang/_common.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """Language-specific support in nunavut. @@ -19,13 +19,28 @@ from ._language import Language +# +-------------------------------------------------------------------------------------------------------------------+ +# | GENERATORS +# +-------------------------------------------------------------------------------------------------------------------+ + + class IncludeGenerator: + """ + Generates include file paths for a given language and datatype. + """ + def __init__(self, language: Language, t: pydsdl.CompositeType, omit_serialization_support: bool): self._type = t self._language = language self._omit_serialization_support = omit_serialization_support def generate_include_filepart_list(self, output_extension: str, sort: bool) -> typing.List[str]: + """ + Generates a list of include file paths for a given datatype and language. + :param output_extension: The file extension to use for the include file paths. + :param sort: If True the list of include file paths will be sorted. + :return: A list of include file paths. + """ dep_types = self._language.get_dependency_builder(self._type).direct() path_list = [ @@ -43,14 +58,14 @@ def generate_include_filepart_list(self, output_extension: str, sort: bool) -> t prefer_system_includes = self._language.get_config_value_as_bool("prefer_system_includes", False) if prefer_system_includes: - path_list_with_punctuation = ["<{}>".format(p) for p in path_list] + path_list_with_punctuation = [f"<{p}>" for p in path_list] else: - path_list_with_punctuation = ['"{}"'.format(p) for p in path_list] + path_list_with_punctuation = [f'"{p}"' for p in path_list] if sort: return sorted(path_list_with_punctuation + self._language.get_includes(dep_types)) - else: - return path_list_with_punctuation + self._language.get_includes(dep_types) + + return path_list_with_punctuation + self._language.get_includes(dep_types) @classmethod def make_path( @@ -60,7 +75,7 @@ def make_path( output_extension: typing.Optional[str] = None, ) -> pathlib.Path: """ - Common method for createing a relative path to a datatype source file. + Common method for creating a relative path to a datatype source file. .. invisible-code-block: python @@ -94,9 +109,7 @@ def make_path( """ if language is None: - short_name = "{short}_{major}_{minor}".format( - short=dt.short_name, major=dt.version.major, minor=dt.version.minor - ) + short_name = f"{dt.short_name}_{dt.version.major}_{dt.version.minor}" else: short_name = language.filter_short_reference_name(dt, id_type="path") @@ -116,8 +129,10 @@ def make_path( def _make_ns_list(cls, language: typing.Optional[Language], dt: pydsdl.SerializableType) -> typing.List[str]: if language is not None and language.enable_stropping: return [language.filter_id(x, id_type="path") for x in dt.full_namespace.split(".")] - else: - return typing.cast(typing.List[str], dt.full_namespace.split(".")) + return typing.cast(typing.List[str], dt.full_namespace.split(".")) + + +# +-------------------------------------------------------------------------------------------------------------------+ class UniqueNameGenerator: @@ -126,17 +141,23 @@ class UniqueNameGenerator: This should be made available as a private global within each template. """ - _singleton = None # type: typing.Optional['UniqueNameGenerator'] + _singleton: typing.Optional["UniqueNameGenerator"] = None def __init__(self) -> None: - self._index_map = {} # type: typing.Dict[str, typing.Dict[str, int]] + self._index_map: typing.Dict[str, typing.Dict[str, int]] = {} @classmethod def reset(cls) -> None: + """ + Resets the singleton instance of the UniqueNameGenerator. + """ cls._singleton = cls() @classmethod def get_instance(cls) -> "UniqueNameGenerator": + """ + Returns the singleton instance of the UniqueNameGenerator. + """ if cls._singleton is None: raise RuntimeError("No UniqueNameGenerator has been created. Please use reset to create.") return cls._singleton @@ -159,9 +180,12 @@ def __call__(self, key: str, base_token: str, prefix: str, suffix: str) -> str: next_index = 0 keymap[base_token] = 1 - return "{prefix}{base_token}{index}{suffix}".format( - prefix=prefix, base_token=base_token, index=next_index, suffix=suffix - ) + return f"{prefix}{base_token}{next_index}{suffix}" + + +# +-------------------------------------------------------------------------------------------------------------------+ +# | ENCODERS +# +-------------------------------------------------------------------------------------------------------------------+ class TokenEncoder: @@ -329,9 +353,7 @@ def __init__( self._stropping_suffix = language.get_config_value("stropping_suffix", "") self._encoding_prefix = language.get_config_value("encoding_prefix", "") try: - self._whitespace_encoding_char = language.get_config_value( - "whitespace_encoding_char" - ) # type: typing.Optional[str] + self._whitespace_encoding_char: typing.Optional[str] = language.get_config_value("whitespace_encoding_char") except KeyError: self._whitespace_encoding_char = None self._collapse_whitespace_when_encoding = language.get_config_value_as_bool("collapse_whitespace_when_encoding") @@ -370,9 +392,8 @@ def _encode(self, token: str, token_type: str, dry_run: bool) -> str: encoded = token_pattern.sub(self._encoding_filter, encoded) elif token_pattern.match(encoded): raise RuntimeError( - 'Unstable encoding: using prefix "{}" partially encoded token: "{}"'.format( - self._encoding_prefix, encoded - ) + f'Unstable encoding: using prefix "{self._encoding_prefix}" partially encoded token: ' + '"{encoded}"' ) except KeyError: pass @@ -386,8 +407,8 @@ def _strop_by_keyword(self, token: str, token_type: str, dry_run: bool) -> str: stropped = self._stropping_prefix + stropped + self._stropping_suffix else: raise RuntimeError( - 'input token "{}" of type "{}" yielded an illegal token after ' - "stropping: {}".format(stropped, token_type, stropped) + f'input token "{stropped}" of type "{token_type}" yielded an illegal token after ' + "stropping: {stropped}" ) return stropped @@ -402,8 +423,8 @@ def _strop_by_pattern(self, token: str, token_type: str, dry_run: bool) -> str: stropped = self._stropping_prefix + stropped + self._stropping_suffix else: raise RuntimeError( - 'input token "{}" of type "{}" yielded an illegal token after ' - "stropping: {}".format(stropped, token_type, stropped) + f'input token "{stropped}" of type "{token_type}" yielded an illegal token after ' + "stropping: {stropped}" ) return stropped @@ -431,13 +452,21 @@ def _do_for_type_and_all( # +------------------------------------------------------------------------------------------------------------+ def encode_character(self, c: str) -> str: + """ + Encode a character into a string representation. + + :param c: The character to encode. + :return: The string representation of the encoded character. + """ if self._whitespace_encoding_char is not None and c.isspace(): return self._whitespace_encoding_char - else: - return "{}{:04X}".format(self._encoding_prefix, ord(c)) + return f"{self._encoding_prefix}{ord(c):04X}" @functools.lru_cache(maxsize=1024) - def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 + def strop(self, token: str, token_type: str = "any") -> str: + """ + Strops a token such that it is a valid identifier for the given language. + """ token_type_lower = token_type.lower() if token_type_lower == "all": raise ValueError( @@ -461,8 +490,7 @@ def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 except RuntimeError as pending_error: if self._stropping_failure_handler is None: raise pending_error - else: - stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) + stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) # and check that the stropping didn't result in a keyword try: @@ -470,8 +498,7 @@ def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 except RuntimeError as pending_error: if self._stropping_failure_handler is None: raise pending_error - else: - stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) + stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) # finally, we make sure stropping didn't result in encoding violations try: @@ -479,8 +506,7 @@ def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 except RuntimeError as pending_error: if self._encoding_failure_handler is None: raise pending_error - else: - stropped = self._encoding_failure_handler(self, stropped, token_type, pending_error) + stropped = self._encoding_failure_handler(self, stropped, token_type, pending_error) return stropped diff --git a/src/nunavut/lang/_config.py b/src/nunavut/lang/_config.py index 12bab95d..6ce76d17 100644 --- a/src/nunavut/lang/_config.py +++ b/src/nunavut/lang/_config.py @@ -1,53 +1,23 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # -"""Logic for parsing language configuration. - +""" +Logic for parsing language configuration. """ import re import types import typing -from enum import auto, Enum - -from yaml import Loader as YamlLoader +from yaml import SafeLoader as YamlLoader from yaml import load as yaml_loader -from nunavut._utilities import deep_update - -NUNAVUT_LANG_CPP = "nunavut.lang.cpp" - - -class ConstructorConvention(Enum): - Default = "default" - UsesLeadingAllocator = "uses-leading-allocator" - UsesTrailingAllocator = "uses-trailing-allocator" - - @staticmethod - def parse_string(s: str) -> typing.Optional[typing.Any]: # annoying mypy cheat due to returning type being defined - for e in ConstructorConvention: - if s == e.value: - return e - return None - - -class SpecialMethod(Enum): - """ - Enum used in the Jinja templates to differentiate different kinds of constructrors - """ - - AllocatorConstructor = auto() - """ Constructor that takes an allocator as its single, required argument """ - - InitializingConstructorWithAllocator = auto() - """ Constructor that takes an initializing value for each field followed by the allocator argument """ - CopyConstructorWithAllocator = auto() - """ Copy constructor that also takes an allocator argument """ +from nunavut._utilities import deep_update, no_default_value - MoveConstructorWithAllocator = auto() - """ Move constructor that also takes an allocator argument """ +# +-------------------------------------------------------------------------------------------------------------------+ +# | LANGUAGE CONFIGURATION +# +-------------------------------------------------------------------------------------------------------------------+ class LanguageConfig: @@ -76,7 +46,7 @@ class LanguageConfig: from nunavut.lang import LanguageConfig config = LanguageConfig() - config.update_from_string(example_yaml) + config.update_from_yaml_string(example_yaml) data = config.sections() assert len(data) == 3 @@ -104,17 +74,17 @@ class LanguageConfig: .. invisible-code-block: python - config.update_from_string(example_yaml) + config.update_from_yaml_string(example_yaml) assert 'a_dictionary' == config.sections()['nunavut.lang.d']['key_one'][2]['list']['is'] """ SECTION_NAME_PATTERN = re.compile( r"^nunavut\.lang\.([a-zA-Z]{1}\w*)$" - ) #: Required pattern for section name identifers. + ) #: Required pattern for section name identifiers. def __init__(self): # type: ignore - self._sections = dict() # type: typing.Dict[str, typing.Dict[str, typing.Any]] + self._sections: typing.Dict[str, typing.Dict[str, typing.Any]] = {} def update(self, configuration: typing.Any) -> None: """ @@ -246,25 +216,46 @@ def update(self, configuration: typing.Any) -> None: raise TypeError("section names must be strings") if not self.SECTION_NAME_PATTERN.match(section_name): raise ValueError( - 'Section name "{}" is invalid. See LanguageConfig documentation for rules.'.format(section_name) + f'Section name "{section_name}" is invalid. See LanguageConfig documentation for rules.' ) self.update_section(section_name, section_data) def update_section(self, section_name: str, configuration: typing.Any) -> None: + """ + Update a section of the configuration. + """ self._sections[section_name] = deep_update(self._sections.get(section_name, {}), configuration) def sections(self) -> typing.Dict[str, typing.Dict[str, typing.Any]]: + """ + Get all sections of the configuration. + """ return self._sections - def update_from_string(self, string: str, context: typing.Optional[str] = None) -> None: + def update_from_yaml_string(self, string: str) -> None: + """ + Update the configuration from a yaml string. + Calls :meth:`update` with the parsed yaml data and will raise the same exceptions. + """ configuration = yaml_loader(string, Loader=YamlLoader) self.update(configuration) - def update_from_file(self, f: typing.TextIO, context: typing.Optional[str] = None) -> None: + def update_from_yaml_file(self, f: typing.TextIO) -> None: + """ + Update the configuration from a yaml file. + Calls :meth:`update` with the parsed yaml data and will raise the same exceptions. + """ configuration = yaml_loader(f, Loader=YamlLoader) self.update(configuration) def set(self, section: str, option: str, value: typing.Any) -> None: + """ + Set a configuration value. + + :param section: The section to set the value in. + :param option: The option to set. + :param value: The value to set. + """ self._sections[section][option] = value def add_section(self, section_name: str) -> None: @@ -306,15 +297,14 @@ def add_section(self, section_name: str) -> None: if not isinstance(section_name, str): raise TypeError("section names must be strings") if not self.SECTION_NAME_PATTERN.match(section_name): - raise ValueError( - 'Section name "{}" is invalid. See LanguageConfig documentation for rules.'.format(section_name) - ) + raise ValueError(f'Section name "{section_name}" is invalid. See LanguageConfig documentation for rules.') if section_name in self._sections: - raise ValueError("Section {} is already defined.".format(section_name)) - self._sections[section_name] = dict() + raise ValueError(f"Section {section_name} is already defined.") + self._sections[section_name] = {} _UNSET = object() # Used internally to allow "None" as a default value. + @no_default_value def _get_config_value_raw(self, section_name: str, key: str, default_value: typing.Any) -> typing.Any: """ .. invisible-code-block: python @@ -361,15 +351,13 @@ def _get_config_value_raw(self, section_name: str, key: str, default_value: typi except KeyError: if default_value is not self._UNSET: return default_value - else: - raise + raise try: return section_data[key] except KeyError: if default_value is not self._UNSET: return default_value - else: - raise + raise def get_config_value(self, section_name: str, key: str, default_value: typing.Optional[str] = None) -> str: """ @@ -486,8 +474,7 @@ def get_config_value_as_bool(self, section_name: str, key: str, default_value: b result = self.get_config_value(section_name, key, default_value="false" if not default_value else "true") if result.lower() == "false" or result == "0": return False - else: - return bool(result) + return bool(result) def get_config_value_as_dict( self, section_name: str, key: str, default_value: typing.Optional[typing.Dict] = None @@ -556,7 +543,7 @@ def get_config_value_as_dict( return raw_value if default_value is None: - raise TypeError("{}.{} exists but is not a dict. (is type {})".format(section_name, key, type(raw_value))) + raise TypeError(f"{section_name}.{key} exists but is not a dict. (is type {type(raw_value)})") return default_value @@ -626,30 +613,10 @@ def get_config_value_as_list( return raw_value if default_value is None: - raise TypeError("{}.{} exists but is not a list. (is type {})".format(section_name, key, type(raw_value))) + raise TypeError(f"{section_name}.{key} exists but is not a list. (is type {type(raw_value)})") return default_value - def apply_defaults(self, language_standard: str) -> None: - defaults_key = f"{language_standard}_options" - if defaults_key in self.sections()[NUNAVUT_LANG_CPP]: - defaults_data = self.get_config_value_as_dict(NUNAVUT_LANG_CPP, defaults_key) - self.update_section(NUNAVUT_LANG_CPP, {"options": defaults_data}) - - def validate_language_options(self) -> None: - options = self.get_config_value_as_dict(NUNAVUT_LANG_CPP, "options") - ctor_convention_str: str = options["ctor_convention"] - ctor_convention = ConstructorConvention.parse_string(ctor_convention_str) - if not ctor_convention: - raise RuntimeError( - f"ctor_convention property '{ctor_convention_str}' is invalid and must be one of " - + (",".join([f"'{e.value}'" for e in ConstructorConvention])) - ) - if ctor_convention != ConstructorConvention.Default and not options["allocator_type"]: - raise RuntimeError( - f"allocator_type property must be specified when ctor_convention is '{ctor_convention_str}'" - ) - # +-------------------------------------------------------------------------------------------------------------------+ # | VersionReader @@ -658,45 +625,59 @@ def validate_language_options(self) -> None: class VersionReader: """ - Helper to read an "x.y.z" semantic version from python modules as a module variable - "__version__" + Helper to read an "x.y.z" semantic version from python modules as a module variable `MODULE_VERSION_ATTRIBUTE_NAME`. + :param module_name: The name of the module to read the version from. """ MODULE_VERSION_ATTRIBUTE_NAME = "__version__" @classmethod def parse_version(cls, version_string: str) -> typing.Optional[typing.Tuple[int, int, int]]: + """ + Parse a version string into a tuple of (major, minor, patch). + :param version_string: The version string to parse. + :return: The version as a tuple of (major, minor, patch) or None if the version string is not in the expected + format. + """ version_array = [int(x) for x in version_string.split(".")] if len(version_array) != 3: return None - else: - return (version_array[0], version_array[1], version_array[2]) + return (version_array[0], version_array[1], version_array[2]) @classmethod def read_version(cls, module: "types.ModuleType") -> typing.Tuple[int, int, int]: - version = getattr(module, cls.MODULE_VERSION_ATTRIBUTE_NAME, "0.0.0") # type: str + """ + Read the version from a module. + + :param module: The module to read the version from. + :return: The version as a tuple of (major, minor, patch). + :raises: ValueError if the version is not in the expected format. + """ + version: str = getattr(module, cls.MODULE_VERSION_ATTRIBUTE_NAME, "0.0.0") version_tuple = cls.parse_version(version) if version_tuple is None: - raise RuntimeError( - 'Invalid {} "{}" for module {} (expected "x.y.z")'.format( - cls.MODULE_VERSION_ATTRIBUTE_NAME, version, module.__name__ - ) + raise ValueError( + f'Invalid {cls.MODULE_VERSION_ATTRIBUTE_NAME} "{version}" for module {module.__name__}' + '(expected "x.y.z")' ) return version_tuple def __init__(self, module_name: str): self._module_name = module_name - self._cached = None # type: typing.Optional[typing.Tuple[int, int, int]] + self._cached: typing.Optional[typing.Tuple[int, int, int]] = None @property def version(self) -> typing.Tuple[int, int, int]: + """ + The version of the module as a tuple of (major, minor, patch). + """ if self._cached is None: self._cached = self._get_version() return self._cached def _get_version(self) -> typing.Tuple[int, int, int]: - import importlib + import importlib # pylint: disable=import-outside-toplevel try: return self.read_version(importlib.import_module(self._module_name)) diff --git a/src/nunavut/lang/_language.py b/src/nunavut/lang/_language.py index 0734e96f..280b0ab2 100644 --- a/src/nunavut/lang/_language.py +++ b/src/nunavut/lang/_language.py @@ -1,7 +1,7 @@ # -# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2022 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Language-specific support in nunavut. @@ -56,21 +56,21 @@ class Language(metaclass=abc.ABCMeta): my_lang = _GenericLanguage("foo", mock_config) # module must be within 'nunavut' assert False - except RuntimeError: + except ValueError: pass try: my_lang = _GenericLanguage("nunavut.foo", mock_config) # module must be within 'nunavut.lang' assert False - except RuntimeError: + except ValueError: pass try: my_lang = _GenericLanguage("not.nunavut.foo", mock_config) # module must be within 'nunavut.lang' assert False - except RuntimeError: + except ValueError: pass my_lang = _GenericLanguage("nunavut.lang.foo", mock_config) @@ -88,6 +88,7 @@ class Language(metaclass=abc.ABCMeta): WKCV_NAMED_TYPES = "named_types" WKCV_NAMED_VALUES = "named_values" WKCV_LANGUAGE_OPTIONS = "options" + WKCV_LANGUAGE_OPTION_DEFAULTS = "defaults" @classmethod def default_filter_id_for_target(cls, instance: typing.Any) -> str: @@ -106,17 +107,23 @@ def default_filter_id_for_target(cls, instance: typing.Any) -> str: # | LIFECYCLE AND DATA MODEL # +-----------------------------------------------------------------------+ - def __init__(self, language_module_name: str, config: LanguageConfig, **kwargs: typing.Any): - self._globals = None # type: typing.Optional[typing.Mapping[str, typing.Any]] + def __init__( + self, language_module_name: str, config: LanguageConfig, **kwargs: typing.Any + ): # pylint: disable=unused-argument + self._globals: typing.Optional[typing.Mapping[str, typing.Any]] = None self._section = language_module_name if not self._section.startswith(LanguageClassLoader.MODULE_PREFIX): - raise RuntimeError("Unknown module name for language: {}".format(self._section)) + raise ValueError(f"Unknown module name for language: {self._section}") self._language_name = LanguageClassLoader.to_language_name(self._section) self._config = config - self._language_options = config.get_config_value_as_dict(self._section, self.WKCV_LANGUAGE_OPTIONS, dict()) - self._filters = dict() # type: typing.Dict[str, typing.Callable] - self._tests = dict() # type: typing.Dict[str, typing.Callable] - self._uses = dict() # type: typing.Dict[str, typing.Callable] + self._filters: typing.Dict[str, typing.Callable] = {} + self._tests: typing.Dict[str, typing.Callable] = {} + self._uses: typing.Dict[str, typing.Callable] = {} + + self._language_options = self._validate_language_options( + config.get_config_value_as_dict(self._section, self.WKCV_LANGUAGE_OPTION_DEFAULTS, {}), + config.get_config_value_as_dict(self._section, self.WKCV_LANGUAGE_OPTIONS, {}), + ) def __getattr__(self, name: str) -> typing.Any: """ @@ -128,7 +135,34 @@ def __getattr__(self, name: str) -> typing.Any: try: return self.get_globals()[name] except KeyError as e: - raise AttributeError(e) + raise AttributeError(e) from e + + def __str__(self) -> str: + return self._language_name + + def _validate_language_options( + self, defaults: typing.Dict[str, typing.Any], options: typing.Dict[str, typing.Any] + ) -> typing.Dict[str, typing.Any]: + # pylint: disable=unused-argument + """ + Subclasses may override this method to validate language options. It will be invoked once + by the base class constructor before setting the language options property. + :param defaults: The a section of the language configuration that contains default values for options. The + format of this section is language-specific + :param options: The options to validate. + :return: The validated or modified options. + :throws: ValueError if the options are invalid. + """ + return options + + def _validate_globals(self, globals_map: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: + """ + Subclasses may override this method to populate additional language-specific globals + :param globals_map: The globals map to validate. + :return: The validated or modified globals map. + :throws: ValueError if the globals are invalid. + """ + return globals_map # +-----------------------------------------------------------------------+ # | PROPERTIES @@ -222,10 +256,6 @@ def named_values(self) -> typing.Mapping[str, str]: # | METHODS # +-----------------------------------------------------------------------+ - def _add_additional_globals(self, globals_map: typing.Dict[str, typing.Any]) -> None: - """Subclasses may override this method to populate additional language-specific globals""" - pass - def get_support_module(self) -> typing.Tuple[str, typing.Tuple[int, int, int], typing.Optional["types.ModuleType"]]: """ Returns the module object for the language support files. @@ -247,7 +277,7 @@ def get_support_module(self) -> typing.Tuple[str, typing.Tuple[int, int, int], t assert support_version[0] == 1 """ - module_name = "{}.support".format(self._section) + module_name = f"{self._section}.support" try: module = importlib.import_module(module_name) @@ -261,6 +291,9 @@ def get_support_module(self) -> typing.Tuple[str, typing.Tuple[int, int, int], t @functools.lru_cache() def get_dependency_builder(self, for_type: pydsdl.Any) -> DependencyBuilder: + """ + Get a dependency builder for the given type. + """ return DependencyBuilder(for_type) @abc.abstractmethod @@ -270,9 +303,8 @@ def get_includes(self, dep_types: Dependencies) -> typing.List[str]: :param Dependencies dep_types: A description of the dependencies includes are needed for. :return: A list of include file paths. The list may be empty if no includes were needed. """ - pass - def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: + def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: # pylint: disable=unused-argument """ Produces a valid identifier in the language for a given object. The encoding may not be reversible. @@ -299,11 +331,10 @@ def filter_short_reference_name( value can be 'typedef', 'macro', 'function', or 'enum'. Use 'any' to apply stropping rules for all identifier types to the instance. """ - short_name = "{short}_{major}_{minor}".format(short=t.short_name, major=t.version.major, minor=t.version.minor) + short_name = f"{t.short_name}_{t.version.major}_{t.version.minor}" if YesNoDefault.test_truth(stropping, self.enable_stropping): return self.filter_id(short_name, id_type) - else: - return short_name + return short_name def get_config_value(self, key: str, default_value: typing.Optional[str] = None) -> str: """ @@ -352,7 +383,7 @@ def get_config_value_as_dict( :param str key: The config value to retrieve. :param default_value: The value to return if the key was not in the configuration. If provided this method - will not raise a KeyError nor a TypeError. + will not raise a KeyError nor a TypeError. :type default_value: typing.Optional[typing.Mapping[str, typing.Any]] :return: Either the value from the config or the default_value if provided. :rtype: typing.Mapping[str, typing.Any] @@ -370,7 +401,7 @@ def get_config_value_as_list( :param str key: The config value to retrieve. :param default_value: The value to return if the key was not in the configuration. If provided this method - will not raise a KeyError nor a TypeError. + will not raise a KeyError nor a TypeError. :type default_value: typing.Optional[typing.List[typing.Any]] :return: Either the value from the config or the default_value if provided. :rtype: typing.List[typing.Any] @@ -408,9 +439,9 @@ def get_support_files( if module is not None: # All language support modules must provide a list_support_files method # to allow the copy generator access to the packaged support files. - list_support_files = getattr( + list_support_files: typing.Callable[[ResourceType], typing.Generator[pathlib.Path, None, None]] = getattr( module, "list_support_files" - ) # type: typing.Callable[[ResourceType], typing.Generator[pathlib.Path, None, None]] + ) return list_support_files(resource_type) else: return empty_list_support_files() @@ -466,16 +497,14 @@ def get_globals(self) -> typing.Mapping[str, typing.Any]: :return: A mapping of global names to global values. """ if self._globals is None: - globals_map = dict() # type: typing.Dict[str, typing.Any] + globals_map: typing.Dict[str, typing.Any] = {} for key, value in self.named_types.items(): - globals_map["typename_{}".format(key)] = value + globals_map[f"typename_{key}"] = value for key, value in self.named_values.items(): - globals_map["valuetoken_{}".format(key)] = value - - self._add_additional_globals(globals_map) + globals_map[f"valuetoken_{key}"] = value - self._globals = globals_map + self._globals = self._validate_globals(globals_map) return self._globals def get_options(self) -> typing.Mapping[str, typing.Any]: @@ -537,7 +566,7 @@ class LanguageClassLoader: def to_language_name(cls, unknown_string: str) -> str: """ Helper method to take a string that is either a language name or a language module name - and always return a langauge name. + and always return a language name. .. invisible-code-block: python from nunavut.lang import LanguageClassLoader @@ -556,7 +585,7 @@ def to_language_name(cls, unknown_string: str) -> str: def to_language_module_name(cls, unknown_string: str) -> str: """ Helper method to take a string that is either a language name or a language module name - and always return a langauge module name. + and always return a language module name. .. invisible-code-block: python from nunavut.lang import LanguageClassLoader @@ -578,14 +607,17 @@ def _load_config(cls) -> LanguageConfig: parser = LanguageConfig() for resource in iter_package_resources(cls.MODULE_NAME, ".yaml"): ini_string = resource.read_text() - parser.update_from_string(ini_string) + parser.update_from_yaml_string(ini_string) return parser def __init__(self) -> None: - self._config = None # type: typing.Optional[LanguageConfig] + self._config: typing.Optional[LanguageConfig] = None @classmethod - def load_language_module(cls, language_name: str) -> "types.ModuleType": + def load_language_module(cls, language_name: str) -> types.ModuleType: + """ + Load a language module by name. + """ module_name = cls.to_language_module_name(language_name) return importlib.import_module(module_name) @@ -621,9 +653,7 @@ def load_language_class(self, language_name: str) -> typing.Tuple[types.ModuleTy language_type = typing.cast(typing.Type["Language"], getattr(ln_module, "Language")) except AttributeError: logging.debug( - "Unable to find a Language object in nunavut.lang.{}. Using a Generic language object".format( - language_name - ) + "Unable to find a Language object in nunavut.lang.%s. Using a Generic language object", language_name ) language_type = _GenericLanguage diff --git a/src/nunavut/lang/c/__init__.py b/src/nunavut/lang/c/__init__.py index 838d83cc..f1df4d44 100644 --- a/src/nunavut/lang/c/__init__.py +++ b/src/nunavut/lang/c/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating C. All filters in this @@ -10,7 +10,6 @@ import enum import fractions -import functools import re import typing @@ -24,7 +23,7 @@ template_language_test, template_volatile_filter, ) -from nunavut._utilities import YesNoDefault +from nunavut._utilities import YesNoDefault, cached_property from nunavut.jinja.environment import Environment from nunavut.lang._common import IncludeGenerator, TokenEncoder, UniqueNameGenerator from nunavut.lang._language import Language as BaseLanguage @@ -52,8 +51,8 @@ def _handle_stropping_failure( # we couldn't help after all. raise the pending error. raise pending_error - @functools.lru_cache(maxsize=None) - def _get_token_encoder(self) -> TokenEncoder: + @cached_property + def _token_encoder(self) -> TokenEncoder: """ Caching getter to ensure we don't have to recompile TokenEncoders for each filter invocation. """ @@ -72,12 +71,12 @@ def get_includes(self, dep_types: Dependencies) -> typing.List[str]: if dep_types.uses_primitive_static_array: # We include this for memset. std_includes.append("string.h") - return ["<{}>".format(include) for include in sorted(std_includes)] + return [f"<{include}>" for include in sorted(std_includes)] def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: raw_name = self.default_filter_id_for_target(instance) - vne = self._get_token_encoder() + vne = self._token_encoder return vne.strop(raw_name, id_type) @@ -286,7 +285,7 @@ def to_c_int(self, is_signed: bool) -> str: return intname def to_c_float(self) -> str: - if self.value == 8 or self.value == 16 or self.value == 32: + if self.value in (8, 16, 32): return "float" else: return "double" diff --git a/src/nunavut/lang/c/support/__init__.py b/src/nunavut/lang/c/support/__init__.py index 41fd4185..1b14f7c3 100644 --- a/src/nunavut/lang/c/support/__init__.py +++ b/src/nunavut/lang/c/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains supporting C headers to distribute with generated types. diff --git a/src/nunavut/lang/c/templates/__init__.py b/src/nunavut/lang/c/templates/__init__.py index 8c8df7c9..b692c232 100644 --- a/src/nunavut/lang/c/templates/__init__.py +++ b/src/nunavut/lang/c/templates/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains the Jinja templates to generate C headers. diff --git a/src/nunavut/lang/c/templates/base.j2 b/src/nunavut/lang/c/templates/base.j2 index 0b76dac1..b985675c 100644 --- a/src/nunavut/lang/c/templates/base.j2 +++ b/src/nunavut/lang/c/templates/base.j2 @@ -9,15 +9,21 @@ // This is an AUTO-GENERATED Cyphal DSDL data type implementation. Curious? See https://opencyphal.org. // You shouldn't attempt to edit this file. // -// Checking this file under version control is not recommended unless it is used as part of a high-SIL -// safety-critical codebase. The typical usage scenario is to generate it as part of the build process. +{%- if nunavut.embed_auditing_info %} +// Checking this file under version control is not recommended since metadata in this header will change for each +// build invocation (do not use --embed-auditing-info option to remove this comment). +{%- endif %} // // To avoid conflicts with definitions given in the source DSDL file, all entities created by the code generator // are named with an underscore at the end, like foo_bar_(). // // Generator: nunavut-{{ nunavut.version }} (serialization was {{ 'not ' * nunavut.support.omit }}enabled) +{%- if nunavut.embed_auditing_info %} // Source file: {{ T.source_file_path.as_posix() }} // Generated at: {{ now_utc }} UTC +{%- else %} +// Source file: {{ T.source_file_path.name }} +{%- endif %} // Is deprecated: {{ T.deprecated and 'yes' or 'no' }} // Fixed port-ID: {{ T.fixed_port_id }} // Full name: {{ T.full_name }} diff --git a/src/nunavut/lang/cpp/__init__.py b/src/nunavut/lang/cpp/__init__.py index 547e6d0d..25133244 100644 --- a/src/nunavut/lang/cpp/__init__.py +++ b/src/nunavut/lang/cpp/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating C++. All filters in this @@ -15,6 +15,7 @@ import re import textwrap import typing +from enum import Enum, auto import pydsdl @@ -25,14 +26,93 @@ template_language_list_filter, template_language_test, ) -from nunavut._utilities import YesNoDefault +from nunavut._utilities import YesNoDefault, cached_property from nunavut.jinja.environment import Environment from nunavut.lang._common import IncludeGenerator, TokenEncoder, UniqueNameGenerator -from nunavut.lang._config import ConstructorConvention, SpecialMethod from nunavut.lang._language import Language as BaseLanguage from nunavut.lang.c import _CFit from nunavut.lang.c import filter_literal as c_filter_literal +# +-------------------------------------------------------------------------------------------------------------------+ +# | ENUMERATIONS +# +-------------------------------------------------------------------------------------------------------------------+ + + +class ConstructorConvention(Enum): + """ + Indicates the convention used for constructors in the target language. + + .. invisible-code-block: python + + from nunavut.lang.cpp import ConstructorConvention + + assert "default" == ConstructorConvention.DEFAULT + assert ConstructorConvention.DEFAULT == ConstructorConvention.from_string("default") + assert ConstructorConvention.USES_LEADING_ALLOCATOR == \ + ConstructorConvention.from_string("uses-leading-allocator") + assert "uses-trailing-allocator" == str(ConstructorConvention.USES_TRAILING_ALLOCATOR) + + from pytest import raises as assert_raises + assert_raises(ValueError, ConstructorConvention.from_string, "not-a-convention") + + """ + + DEFAULT = "default" + USES_LEADING_ALLOCATOR = "uses-leading-allocator" + USES_TRAILING_ALLOCATOR = "uses-trailing-allocator" + + def __str__(self) -> str: + """ + Return a string representation of the ConstructorConvention enum value. + + :return: The string representation of the enum value. + :rtype: str + """ + return self.value + + def __eq__(self, value: object) -> bool: + if isinstance(value, str): + return super().__eq__(ConstructorConvention.from_string(value)) + return super().__eq__(value) + + @staticmethod + def from_string(s: str) -> "ConstructorConvention": + """ + Parse a string into a ConstructorConvention enum value. + + :param s: The string to parse. + :return: The enum value corresponding to the string. + :rtype: ConstructorConvention + :raises: ValueError if the string does not correspond to a valid enum value. + """ + for e in ConstructorConvention: + if s.lower().replace("_", "-") == e.value: + return e + raise ValueError(f"Invalid ConstructorConvention string '{s}'") + + +class SpecialMethod(Enum): + """ + Enum used in the Jinja templates to differentiate different kinds of constructors + """ + + ALLOCATOR_CONSTRUCTOR = auto() + """ Constructor that takes an allocator as its single, required argument """ + + INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR = auto() + """ Constructor that takes an initializing value for each field followed by the allocator argument """ + + COPY_CONSTRUCTOR_WITH_ALLOCATOR = auto() + """ Copy constructor that also takes an allocator argument """ + + MOVE_CONSTRUCTOR_WITH_ALLOCATOR = auto() + """ Move constructor that also takes an allocator argument """ + + +# +-------------------------------------------------------------------------------------------------------------------+ +# | LANGUAGE SUPPORT +# +-------------------------------------------------------------------------------------------------------------------+ + class Language(BaseLanguage): """ @@ -41,10 +121,77 @@ class Language(BaseLanguage): CPP_STD_EXTRACT_NUMBER_PATTERN = re.compile(r"(?:gnu|c)\+\+(\d(?:\w))") + def _validate_language_options( + self, defaults: typing.Dict[str, typing.Any], options: typing.Dict[str, typing.Any] + ) -> typing.Dict[str, typing.Any]: + """ + apply defaults based on language standard + + .. invisible-code-block: python + + from nunavut.lang import LanguageContextBuilder + from pytest import raises as assert_raises + + # test std + language = LanguageContextBuilder(include_experimental_languages=True)\ + .set_target_language("cpp")\ + .set_target_language_configuration_override("options", { "std_flavor":"std"})\ + .create()\ + .get_target_language() + + assert language._validate_language_options( + {}, + { + "std":"c++17", + "ctor_convention": "default" + } + ) == { + "std":"c++17", + "ctor_convention": "default" + } + + assert_raises(ValueError, \ + language._validate_language_options, {}, {"ctor_convention": "default"}) + + assert_raises(ValueError, \ + language._validate_language_options, {}, {"std":"c++17", "ctor_convention": "uses-leading-allocator"}) + + assert_raises(ValueError, \ + language._validate_language_options, {}, \ + {"std":"c++17", "ctor_convention": "uses-leading-allocator", "allocator_type": ""}) + + """ + try: + language_standard = options["std"] + except KeyError as e: + raise ValueError("The 'std' option must be in the language options for the C++ language.") from e + + if language_standard in defaults: + options.update(defaults[language_standard]) + + try: + ctor_convention = ConstructorConvention.from_string(options["ctor_convention"]) + except KeyError as e: + raise ValueError("No constructor convention option in C++ language options. This is required.") from e + + if ctor_convention != ConstructorConvention.DEFAULT and ( + "allocator_type" not in options or not options["allocator_type"] + ): + raise ValueError( + f"allocator_type property must be specified when ctor_convention is '{str(ctor_convention)}'" + ) + return options + + def _validate_globals(self, globals_map: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: + globals_map["ConstructorConvention"] = ConstructorConvention + globals_map["SpecialMethod"] = SpecialMethod + return globals_map + @staticmethod def _handle_stropping_or_encoding_failure( encoder: TokenEncoder, stropped: str, token_type: str, pending_error: RuntimeError ) -> str: + # pylint: disable=unused-argument """ If the generic stropping fails we take one last look to see if there is something c++-specific we can do. """ @@ -57,15 +204,15 @@ def _handle_stropping_or_encoding_failure( if m: # Resolve the conflict between C's global identifier rules and our desire to use # '_' as a stropping prefix: - return "_{}{}".format(m.group(1).lower(), stropped[m.end() :]) + return f"_{m.group(1).lower()}{stropped[m.end() :]}" # we couldn't help after all. raise the pending error. raise pending_error - @functools.lru_cache(maxsize=None) - def _get_token_encoder(self) -> TokenEncoder: + @cached_property + def _token_encoder(self) -> TokenEncoder: """ - Caching getter to ensure we don't have to recompile TokenEncoders for each filter invocation. + Cached property to ensure we don't have to recompile TokenEncoders for each filter invocation. """ return TokenEncoder( self, @@ -73,8 +220,39 @@ def _get_token_encoder(self) -> TokenEncoder: encoding_failure_handler=self._handle_stropping_or_encoding_failure, ) - def _standard_version(self) -> int: + @property + def standard_flavor(self) -> str: + """ + A flavor of the C++ language standard being targeted. + + .. invisible-code-block: python + + from nunavut.lang import LanguageContextBuilder + + # test std + language = LanguageContextBuilder(include_experimental_languages=True)\ + .set_target_language("cpp")\ + .set_target_language_configuration_override("options", { "std_flavor":"std"})\ + .create()\ + .get_target_language() + + assert language.standard_flavor == 'std' + + # test cetl + language = LanguageContextBuilder(include_experimental_languages=True)\ + .set_target_language("cpp")\ + .set_target_language_configuration_override("options", { "std_flavor":"cetl"})\ + .create()\ + .get_target_language() + assert language.standard_flavor == 'cetl' + """ + return str(self.get_option("std_flavor")) + + @property + def standard_version(self) -> int: """ + The numeric version of the C++ language standard being targeted. + .. invisible-code-block: python from nunavut.lang import LanguageContextBuilder @@ -86,7 +264,7 @@ def _standard_version(self) -> int: .create()\ .get_target_language() - assert language._standard_version() == 17 + assert language.standard_version == 17 # test c++14 language = LanguageContextBuilder(include_experimental_languages=True)\ @@ -94,7 +272,7 @@ def _standard_version(self) -> int: .set_target_language_configuration_override("options", { "std":"c++14"})\ .create()\ .get_target_language() - assert language._standard_version() == 14 + assert language.standard_version == 14 # test gnu++20 language = LanguageContextBuilder(include_experimental_languages=True)\ @@ -103,7 +281,7 @@ def _standard_version(self) -> int: .create()\ .get_target_language() - assert language._standard_version() == 20 + assert language.standard_version == 20 """ std = str(self.get_option("std", "")) @@ -111,50 +289,48 @@ def _standard_version(self) -> int: if match is not None and len(match.groups()) >= 1: return int(match.group(1)) - else: - return 0 + return 0 - def _has_variant(self) -> bool: + @property + def has_variant(self) -> bool: """ .. invisible-code-block: python from nunavut.lang import LanguageClassLoader # test c++17 - language = LanguageContextBuilder(include_experimental_languages=True)\ - .set_target_language("cpp")\ - .set_target_language_configuration_override("options", { "std":"c++17"})\ - .create()\ - .get_target_language() + language = ( + LanguageContextBuilder(include_experimental_languages=True) + .set_target_language("cpp") + .set_target_language_configuration_override("options", { "std":"c++17"}) + .create() + .get_target_language() + ) - assert language._has_variant() + assert language.has_variant # test c++14 - language = LanguageContextBuilder(include_experimental_languages=True)\ - .set_target_language("cpp")\ - .set_target_language_configuration_override("options", { "std":"c++14"})\ - .create()\ + language = ( + LanguageContextBuilder(include_experimental_languages=True) + .set_target_language("cpp") + .set_target_language_configuration_override("options", { "std":"c++14"}) + .create() .get_target_language() - - assert not language._has_variant() + ) + assert not language.has_variant # test gnu++20 - language = LanguageContextBuilder(include_experimental_languages=True)\ - .set_target_language("cpp")\ - .set_target_language_configuration_override("options", { "std":"gnu++20"})\ - .create()\ + language = ( + LanguageContextBuilder(include_experimental_languages=True) + .set_target_language("cpp") + .set_target_language_configuration_override("options", { "std":"gnu++20"}) + .create() .get_target_language() + ) - assert language._has_variant() + assert language.has_variant """ - return self._standard_version() >= 17 - - def _add_additional_globals(self, globals_map: typing.Dict[str, typing.Any]) -> None: - """ - Make additional globals available in the cpp jinja templates - """ - globals_map["ConstructorConvention"] = ConstructorConvention - globals_map["SpecialMethod"] = SpecialMethod + return self.standard_version >= 17 def get_includes(self, dep_types: Dependencies) -> typing.List[str]: """ @@ -211,7 +387,7 @@ def do_includes_test(override_vla_include, override_allocator_include): do_includes_test(False, False) do_includes_test(False, True) """ - std_includes = [] # type: typing.List[str] + std_includes: typing.List[str] = [] std_includes.append("limits") # we always include limits to support static assertions if self.get_config_value_as_bool("use_standard_types"): if dep_types.uses_integer: @@ -220,9 +396,9 @@ def do_includes_test(override_vla_include, override_allocator_include): std_includes.append("array") if dep_types.uses_boolean_static_array: std_includes.append("bitset") - if dep_types.uses_union and self._has_variant(): + if dep_types.uses_union and self.has_variant: std_includes.append("variant") - includes_formatted = ["<{}>".format(include) for include in sorted(std_includes)] + includes_formatted = [f"<{include}>" for include in sorted(std_includes)] allocator_include = str(self.get_option("allocator_include", "")) if len(allocator_include) > 0: @@ -238,20 +414,20 @@ def do_includes_test(override_vla_include, override_allocator_include): def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: raw_name = self.default_filter_id_for_target(instance) - return self._get_token_encoder().strop(raw_name, id_type) + return self._token_encoder.strop(raw_name, id_type) - def create_bitset_decl(self, type: str, max_size: int) -> str: - return "std::bitset<{MAX_SIZE}>".format(MAX_SIZE=max_size) + def create_bitset_decl(self, max_size: int) -> str: + return f"std::bitset<{max_size}>" - def create_array_decl(self, type: str, max_size: int) -> str: - return "std::array<{TYPE},{MAX_SIZE}>".format(TYPE=type, MAX_SIZE=max_size) + def create_array_decl(self, data_type: str, max_size: int) -> str: + return f"std::array<{data_type},{max_size}>" - def create_vla_decl(self, type: str, max_size: int) -> str: + def create_vla_decl(self, data_type: str, max_size: int) -> str: variable_array_type_template = self.get_option("variable_array_type_template") if not isinstance(variable_array_type_template, str) or len(variable_array_type_template) == 0: raise RuntimeError("You must specify a value for the 'variable_array_type_template' option.") - rebind_allocator = "std::allocator_traits::rebind_alloc<{TYPE}>".format(TYPE=type) - return variable_array_type_template.format(TYPE=type, MAX_SIZE=max_size, REBIND_ALLOCATOR=rebind_allocator) + rebind_allocator = f"std::allocator_traits::rebind_alloc<{data_type}>" + return variable_array_type_template.format(TYPE=data_type, MAX_SIZE=max_size, REBIND_ALLOCATOR=rebind_allocator) @template_language_test(__name__) @@ -301,7 +477,27 @@ def uses_std_variant(language: Language) -> bool: jinja_filter_tester(None, template, '#include "user_variant.h"', lctx) """ - return language._has_variant() + return language.has_variant + + +@template_language_test(__name__) +def uses_cetl(language: Language) -> bool: + """ + Uses query for Cyphal Embedded Template Library. + + If this is true then CETL is used to ensure compatibility back to C++14. + """ + return language.standard_flavor == "cetl" + + +@template_language_test(__name__) +def uses_pmr(language: Language) -> bool: + """ + Uses query for C++17 Polymorphic Memory Resources. + + If this is true then additional C++ code is generated to support the use of polymorphic memory resources. + """ + return language.standard_flavor == "pmr" @template_language_filter(__name__) @@ -815,8 +1011,7 @@ def filter_short_reference_name(language: Language, t: pydsdl.CompositeType) -> if isinstance(t, pydsdl.ServiceType): if YesNoDefault.test_truth(YesNoDefault.DEFAULT, language.enable_stropping): return language.filter_id(t.short_name) - else: - return str(t.short_name) + return str(t.short_name) return language.filter_short_reference_name(t) @@ -932,14 +1127,14 @@ def filter_explicit_decorator(language: Language, instance: pydsdl.Any, special_ arg_count: int = len(instance.fields_except_padding) + ( 0 if language.get_option("allocator_is_default_constructible") else 1 ) - if special_method == SpecialMethod.InitializingConstructorWithAllocator and arg_count == 1: + if special_method == SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR and arg_count == 1: return f"explicit {name}" - else: - return f"{name}" + return f"{name}" @template_language_filter(__name__) def filter_default_value_initializer(language: Language, instance: pydsdl.Any) -> str: + # pylint: disable=unused-argument """ Emit a default initialization expression for the given instance if primitive, array, or composite. @@ -955,14 +1150,14 @@ def filter_default_value_initializer(language: Language, instance: pydsdl.Any) - def needs_initializing_value(special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" - return special_method == SpecialMethod.InitializingConstructorWithAllocator or needs_rhs(special_method) + return special_method == SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR or needs_rhs(special_method) def needs_rhs(special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" return special_method in ( - SpecialMethod.CopyConstructorWithAllocator, - SpecialMethod.MoveConstructorWithAllocator, + SpecialMethod.COPY_CONSTRUCTOR_WITH_ALLOCATOR, + SpecialMethod.MOVE_CONSTRUCTOR_WITH_ALLOCATOR, ) @@ -975,14 +1170,14 @@ def needs_allocator(instance: pydsdl.Any) -> bool: def needs_vla_init_args(instance: pydsdl.Any, special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" - return special_method == SpecialMethod.AllocatorConstructor and isinstance( + return special_method == SpecialMethod.ALLOCATOR_CONSTRUCTOR and isinstance( instance.data_type, pydsdl.VariableLengthArrayType ) def needs_move(special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" - return special_method == SpecialMethod.MoveConstructorWithAllocator + return special_method == SpecialMethod.MOVE_CONSTRUCTOR_WITH_ALLOCATOR def requires_initialization(instance: pydsdl.Any) -> bool: @@ -999,7 +1194,7 @@ def assemble_initializer_expression( ) -> str: """Helper method used by filter_value_initializer()""" if wrap: - rhs = "{}({})".format(wrap, rhs) + rhs = f"{wrap}({rhs})" args = [] if rhs: args.append(rhs) @@ -1031,7 +1226,7 @@ def filter_value_initializer(language: Language, instance: pydsdl.Any, special_m trailing_args.append(constructor_args.format(MAX_SIZE=instance.data_type.capacity)) if needs_allocator(instance): - if language.get_option("ctor_convention") == ConstructorConvention.UsesLeadingAllocator.value: + if language.get_option("ctor_convention") == ConstructorConvention.USES_LEADING_ALLOCATOR.value: leading_args.extend(["std::allocator_arg", "allocator"]) else: trailing_args.append("allocator") @@ -1046,13 +1241,15 @@ def filter_value_initializer(language: Language, instance: pydsdl.Any, special_m @template_language_filter(__name__) def filter_default_construction(language: Language, instance: pydsdl.Any, reference: str) -> str: + """ + Emit a default construction expression for the given instance if it is a composite type. + """ if ( isinstance(instance, pydsdl.CompositeType) - and language.get_option("ctor_convention") != ConstructorConvention.Default.value + and language.get_option("ctor_convention") != ConstructorConvention.DEFAULT.value ): return f"{reference}.get_allocator()" - else: - return "" + return "" @template_language_filter(__name__) @@ -1060,17 +1257,16 @@ def filter_declaration(language: Language, instance: pydsdl.Any) -> str: """ Emit a declaration statement for the given instance. """ - if isinstance(instance, pydsdl.PrimitiveType) or isinstance(instance, pydsdl.VoidType): + if isinstance(instance, (pydsdl.PrimitiveType, pydsdl.VoidType)): return filter_type_from_primitive(language, instance) - elif isinstance(instance, pydsdl.VariableLengthArrayType): + if isinstance(instance, pydsdl.VariableLengthArrayType): return language.create_vla_decl(filter_declaration(language, instance.element_type), instance.capacity) - elif isinstance(instance, pydsdl.ArrayType): + if isinstance(instance, pydsdl.ArrayType): if isinstance(instance.element_type, pydsdl.BooleanType): - return language.create_bitset_decl(filter_declaration(language, instance.element_type), instance.capacity) - else: - return language.create_array_decl(filter_declaration(language, instance.element_type), instance.capacity) - else: - return filter_full_reference_name(language, instance) + return language.create_bitset_decl(instance.capacity) + return language.create_array_decl(filter_declaration(language, instance.element_type), instance.capacity) + + return filter_full_reference_name(language, instance) @template_language_filter(__name__) @@ -1167,8 +1363,7 @@ def filter_to_namespace_qualifier(namespace_list: typing.List[str]) -> str: """ if namespace_list is None or len(namespace_list) == 0: return "" - else: - return "::".join(namespace_list) + "::" + return "::".join(namespace_list) + "::" def filter_to_template_unique_name(base_token: str) -> str: @@ -1232,7 +1427,7 @@ def filter_to_template_unique_name(base_token: str) -> str: else: adj_base_token = base_token - return UniqueNameGenerator.get_instance()("cpp", adj_base_token, "_", "_") + return UniqueNameGenerator.get_instance()("cpp", adj_base_token, "_", "_") # pylint: disable=not-callable def filter_as_boolean_value(value: bool) -> str: @@ -1337,7 +1532,7 @@ def filter_indent_if_not(language: Language, text: str, depth: int = 1) -> str: configured_indent = int(language.get_config_value("indent")) lines = text.splitlines(keepends=True) result = "" - for i in range(0, len(lines)): + for i, line in enumerate(lines): line = lines[i].lstrip() if len(line) == 0: # don't indent blank lines @@ -1417,11 +1612,11 @@ def filter_minimum_required_capacity_bits(t: pydsdl.SerializableType) -> int: @functools.lru_cache(3) -def _make_textwrap(width: int, initial_indent: str, subseqent_indent: str) -> textwrap.TextWrapper: +def _make_textwrap(width: int, initial_indent: str, subsequent_indent: str) -> textwrap.TextWrapper: return textwrap.TextWrapper( width=width, initial_indent=initial_indent, - subsequent_indent=subseqent_indent, + subsequent_indent=subsequent_indent, break_on_hyphens=True, break_long_words=False, replace_whitespace=False, @@ -1429,22 +1624,22 @@ def _make_textwrap(width: int, initial_indent: str, subseqent_indent: str) -> te def _make_block_comment(text: str, prefix: str, comment: str, suffix: str, indent: int, line_length: int) -> str: - doc_lines = text.splitlines() # type: typing.List[str] - indented_comment = "{}{}".format(" " * indent, comment) + doc_lines: typing.List[str] = text.splitlines() + indented_comment = f"{' ' * indent}{comment}" - commented_doc_lines = [] # type: typing.List[str] + commented_doc_lines: typing.List[str] = [] if len(doc_lines) > 0: if len(prefix) > 0: commented_doc_lines.append(prefix) else: commented_doc_lines.extend( - _make_textwrap(width=line_length, initial_indent=comment, subseqent_indent=indented_comment).wrap( + _make_textwrap(width=line_length, initial_indent=comment, subsequent_indent=indented_comment).wrap( doc_lines.pop(0) ) ) - tw = _make_textwrap(width=line_length, initial_indent=indented_comment, subseqent_indent=indented_comment) + tw = _make_textwrap(width=line_length, initial_indent=indented_comment, subsequent_indent=indented_comment) for docline in doc_lines: # The docs for textwrap.TextWrapper.wrap say: @@ -1453,7 +1648,7 @@ def _make_block_comment(text: str, prefix: str, comment: str, suffix: str, inden commented_doc_lines.extend(tw.wrap(docline) if docline.strip() else [indented_comment]) if len(suffix) > 0 and len(commented_doc_lines) > 0: - commented_doc_lines.append("{}{}".format(" " * indent, suffix)) + commented_doc_lines.append(f"{' ' * indent}{suffix}") return "\n".join(commented_doc_lines) @@ -1656,16 +1851,14 @@ def filter_block_comment(language: Language, text: str, style: str, indent: int """ - config_styles = language.get_config_value_as_dict( - "comment_styles" - ) # type: typing.Mapping[str, typing.Mapping[str, str]] + config_styles: typing.Mapping[str, typing.Mapping[str, str]] = language.get_config_value_as_dict("comment_styles") try: config_style = config_styles[style.lower()] - except KeyError: + except KeyError as ke: raise ValueError( - "{} is not a supported comment style. Supported is c, cpp, cpp-doxygen, and javadoc".format(style) - ) + f"{style} is not a supported comment style. Supported is c, cpp, cpp-doxygen, and javadoc" + ) from ke return _make_block_comment( text=text, diff --git a/src/nunavut/lang/cpp/support/__init__.py b/src/nunavut/lang/cpp/support/__init__.py index 9c7e4583..5900178f 100644 --- a/src/nunavut/lang/cpp/support/__init__.py +++ b/src/nunavut/lang/cpp/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains supporting C++ headers to distribute with generated types. diff --git a/src/nunavut/lang/cpp/templates/__init__.py b/src/nunavut/lang/cpp/templates/__init__.py index cbca9c40..f679535b 100644 --- a/src/nunavut/lang/cpp/templates/__init__.py +++ b/src/nunavut/lang/cpp/templates/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains the Jinja templates to generate C++ headers. diff --git a/src/nunavut/lang/cpp/templates/_composite_type.j2 b/src/nunavut/lang/cpp/templates/_composite_type.j2 index 8f0bd60c..63fbbe71 100644 --- a/src/nunavut/lang/cpp/templates/_composite_type.j2 +++ b/src/nunavut/lang/cpp/templates/_composite_type.j2 @@ -1,13 +1,13 @@ {#- - # Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. - # Copyright (C) 2021 OpenCyphal Development Team - # This software is distributed under the terms of the MIT License. + # Copyright (C) OpenCyphal Development Team + # Copyright Amazon.com Inc. or its affiliates. + # SPDX-License-Identifier: MIT -#} {%- from '_definitions.j2' import assert -%} {%- ifuses "std_variant" %} // +-------------------------------------------------------------------------------------------------------------------+ // | This implementation uses the C++17 standard library variant type with wrappers for the emplace and -// | get_if methods to support forward-compatibility with the C++14 version of this object. The union_value type +// | get_if methods to support backwards-compatibility with the C++14 version of this object. The union_value type // | extends std::variant and can be used with the entire set of variant methods. Using std::variant directly does mean // | your code will not be backwards compatible with the C++14 version of this object. // +-------------------------------------------------------------------------------------------------------------------+ @@ -33,7 +33,7 @@ struct {% if composite_type.deprecated -%} {%- endif -%} {{composite_type|short_reference_name}} final { -{%- if options.ctor_convention != ConstructorConvention.Default.value %} +{%- if options.ctor_convention != ConstructorConvention.DEFAULT %} using allocator_type = {{ options.allocator_type }}; {%- endif %} @@ -84,7 +84,7 @@ struct {% if composite_type.deprecated -%} {%- endif %} {%- endfor %} }; -{% if options.ctor_convention != ConstructorConvention.Default.value %} +{% if options.ctor_convention != ConstructorConvention.DEFAULT %} {%- if options.allocator_is_default_constructible %} // Default constructor {%- if composite_type.inner_type is UnionType %} @@ -107,7 +107,7 @@ struct {% if composite_type.deprecated -%} union_value{} // can't make use of the allocator with a union {%- else %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.AllocatorConstructor) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.ALLOCATOR_CONSTRUCTOR) }}{%if not loop.last %},{%endif %} {%- endfor %} {%- endif %} { @@ -117,7 +117,7 @@ struct {% if composite_type.deprecated -%} {%- if composite_type.inner_type is not UnionType %} {% if composite_type.fields_except_padding %} // Initializing constructor - {{ composite_type | explicit_decorator(SpecialMethod.InitializingConstructorWithAllocator)}}( + {{ composite_type | explicit_decorator(SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR)}}( {%- for field in composite_type.fields_except_padding %} const _traits_::TypeOf::{{ field | id }}& {{ field | id }}, {%- endfor %} @@ -125,7 +125,7 @@ struct {% if composite_type.deprecated -%} {%- if options.allocator_is_default_constructible %} = allocator_type(){% endif %}) {%- if composite_type.fields_except_padding %} :{% endif %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.InitializingConstructorWithAllocator) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR) }}{%if not loop.last %},{%endif %} {%- endfor %} { (void)allocator; // avoid unused param warning @@ -143,7 +143,7 @@ struct {% if composite_type.deprecated -%} union_value{rhs.union_value} // can't make use of the allocator with a union {%- else %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.CopyConstructorWithAllocator) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.COPY_CONSTRUCTOR_WITH_ALLOCATOR) }}{%if not loop.last %},{%endif %} {%- endfor %} {% endif %} { @@ -161,7 +161,7 @@ struct {% if composite_type.deprecated -%} union_value{} // can't make use of the allocator with a union {%- else %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.MoveConstructorWithAllocator) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.MOVE_CONSTRUCTOR_WITH_ALLOCATOR) }}{%if not loop.last %},{%endif %} {%- endfor %} {%- endif %} { diff --git a/src/nunavut/lang/cpp/templates/_fields.j2 b/src/nunavut/lang/cpp/templates/_fields.j2 index d9c02496..1d28b91f 100644 --- a/src/nunavut/lang/cpp/templates/_fields.j2 +++ b/src/nunavut/lang/cpp/templates/_fields.j2 @@ -10,7 +10,7 @@ // +----------------------------------------------------------------------+ {% endif -%} {{ field.doc | block_comment('cpp-doxygen', 4, 120) }} - {% if options.ctor_convention != ConstructorConvention.Default.value -%} + {% if options.ctor_convention != ConstructorConvention.DEFAULT -%} _traits_::TypeOf::{{field.name|id}} {{ field | id }}; {%- else -%} _traits_::TypeOf::{{field.name|id}} {{ field | id }}{{ field.data_type | default_value_initializer }}; diff --git a/src/nunavut/lang/cpp/templates/base.j2 b/src/nunavut/lang/cpp/templates/base.j2 index abfbddfb..66988bba 100644 --- a/src/nunavut/lang/cpp/templates/base.j2 +++ b/src/nunavut/lang/cpp/templates/base.j2 @@ -1,45 +1,53 @@ {#- - # Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. - # Copyright (C) 2021 OpenCyphal Development Team - # This software is distributed under the terms of the MIT License. + # Copyright (C) OpenCyphal Development Team + # Copyright Amazon.com Inc. or its affiliates. + # SPDX-License-Identifier: MIT -#} // // This is an AUTO-GENERATED Cyphal DSDL data type implementation. Curious? See https://opencyphal.org. // You shouldn't attempt to edit this file. +{%- if nunavut.embed_auditing_info %} // // Checking this file under version control is not recommended since metadata in this header will change for each -// build invocation. TODO: add --reproducible option to prevent any volatile metadata from being generated. +// build invocation (do not use --embed-auditing-info option to remove this comment). +{%- endif %} +// +// Generator : nunavut-{{ nunavut.version }} (serialization was {{ 'not ' * nunavut.support.omit }}enabled) +{%- if nunavut.embed_auditing_info %} +// Source file : {{ T.source_file_path.as_posix() }} +// Generated at : {{ now_utc }} UTC +{%- else %} +// Source file : {{ T.source_file_path.name }} +{%- endif %} +// Is deprecated : {{ T.deprecated and 'yes' or 'no' }} +// Fixed port-ID : {{ T.fixed_port_id }} +// Full name : {{ T.full_name }} +// Type Version : {{ T.version.major }}.{{ T.version.minor }} // -// Generator: nunavut-{{ nunavut.version }} (serialization was {{ 'not ' * nunavut.support.omit }}enabled) -// Source file: {{ T.source_file_path.as_posix() }} -// Generated at: {{ now_utc }} UTC -// Is deprecated: {{ T.deprecated and 'yes' or 'no' }} -// Fixed port-ID: {{ T.fixed_port_id }} -// Full name: {{ T.full_name }} -// Type Version: {{ T.version.major }}.{{ T.version.minor }} // Support {%- if nunavut.support.omit %} // (support file generation disabled) {%- else %} -// Support Namespace: {{ nunavut.support.namespace }} -// Support Version: {{ nunavut.support.version }} +// Support Namespace : {{ nunavut.support.namespace }} +// Support Version : {{ nunavut.support.version }} {%- endif %} {%- for template_set in nunavut.template_sets %} +// // Template Set ({{ template_set[0] }}) -// priority: {{ loop.index0 }} -// package: {{ template_set[1] }} -// version: {{ template_set[2] }} +// priority : {{ loop.index0 }} +// package : {{ template_set[1] }} +// version : {{ template_set[2] }} {%- endfor %} +// // Platform -{%- for key, value in nunavut.platform_version.items() %} -// {{ key }}: {{ value }} -{%- endfor %} +{{ nunavut.platform_version | text_table("// ") }} +// // Language Options -{%- for key, value in options.items() %} -// {{ key }}: {{ value }} -{%- endfor %} +{{ options | text_table("// ") }} +// // Uses Language Features // Uses std_variant: {%- ifuses "std_variant" -%}yes{%- else -%}no{%- endifuses -%} +// {%- if T.deprecated and options.std | int < 14 %} {#- Courtesy http://patorjk.com/software/taag/#p=display&f=Big&t=DEPRECATED #} {#- [[deprecated]] becomes available in c++14 #} diff --git a/src/nunavut/lang/html/__init__.py b/src/nunavut/lang/html/__init__.py index 2bb4894d..982093ae 100644 --- a/src/nunavut/lang/html/__init__.py +++ b/src/nunavut/lang/html/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating docs. All filters in this @@ -24,20 +24,29 @@ def filter_extent(instance: pydsdl.Any) -> int: + """ + Filter that returns the dsdl extend property of a given type. + """ try: return instance.extent or 0 except TypeError as e: - raise TemplateAssertionError(e) + raise TemplateAssertionError(e) from None def filter_max_bit_length(instance: pydsdl.Any) -> int: + """ + Filter that returns the dsdl max bit length property of a given type. + """ try: return instance.bit_length_set.max or 0 except TypeError as e: - raise TemplateAssertionError(e) + raise TemplateAssertionError(e) from None def filter_tag_id(instance: pydsdl.Any) -> str: + """ + Emit a tag id for a given type. + """ if isinstance(instance, pydsdl.ArrayType): return "{}_array".format(str(instance.element_type).replace(".", "_").replace(" ", "_")) else: @@ -49,6 +58,9 @@ def filter_tag_id(instance: pydsdl.Any) -> str: def filter_url_from_type(instance: pydsdl.Any) -> str: + """ + Emit a path to the documentation for a given type. + """ root_ns = instance.root_namespace tag_id = "{}_{}_{}".format(instance.full_name.replace(".", "_"), instance.version[0], instance.version[1]) return "../{}/#{}".format(root_ns, tag_id) @@ -121,6 +133,9 @@ def filter_make_unique(_: typing.Any, base_token: str) -> str: def filter_namespace_doc(ns: nunavut.Namespace) -> str: + """ + Generate HTML documentation for a namespace. + """ result = "" for t, _ in ns.get_nested_types(): if t.short_name == "_": @@ -130,6 +145,9 @@ def filter_namespace_doc(ns: nunavut.Namespace) -> str: def filter_display_type(instance: pydsdl.Any) -> str: + """ + Deprecated. Don't use this filter. Needs refactoring. + """ # TODO: this whole thing needs to be in the template. if isinstance(instance, pydsdl.FixedLengthArrayType): capacity = '[{}]'.format(instance.capacity) diff --git a/src/nunavut/lang/html/support/__init__.py b/src/nunavut/lang/html/support/__init__.py index f5c35ff9..e4c95a6b 100644 --- a/src/nunavut/lang/html/support/__init__.py +++ b/src/nunavut/lang/html/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Empty python package to ensure the support generator doesn't explode. @@ -16,4 +16,9 @@ def list_support_files(resource_type: ResourceType = ResourceType.ANY) -> typing.Generator[pathlib.Path, None, None]: + """ + Get a list of HTML support files embedded in this package. + :param resource_type: A type of support file to list. + """ + # pylint: disable=unused-argument return empty_list_support_files() diff --git a/src/nunavut/lang/js/__init__.py b/src/nunavut/lang/js/__init__.py index 10ddf3b8..482de62a 100644 --- a/src/nunavut/lang/js/__init__.py +++ b/src/nunavut/lang/js/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating javascript. All filters in this diff --git a/src/nunavut/lang/js/support/__init__.py b/src/nunavut/lang/js/support/__init__.py index 9f0dfc3c..8dae41c0 100644 --- a/src/nunavut/lang/js/support/__init__.py +++ b/src/nunavut/lang/js/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Empty python package to ensure the support generator doesn't explode. @@ -16,4 +16,9 @@ def list_support_files(resource_type: ResourceType = ResourceType.ANY) -> typing.Generator[pathlib.Path, None, None]: + """ + Get a list of javascript support files embedded in this package. + :param resource_type: A type of support file to list. + """ + # pylint: disable=unused-argument return empty_list_support_files() diff --git a/src/nunavut/lang/properties.yaml b/src/nunavut/lang/properties.yaml index 07243fce..f42a02a0 100644 --- a/src/nunavut/lang/properties.yaml +++ b/src/nunavut/lang/properties.yaml @@ -1,8 +1,8 @@ %YAML 1.2 # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # --- nunavut.lang.c: @@ -194,6 +194,7 @@ nunavut.lang.c: - defined - __has_include - __has_cpp_attribute + - pascal token_encoding_rules_by_identifier_type: all: - '\s+' @@ -311,6 +312,7 @@ nunavut.lang.cpp: enable_serialization_asserts: false enable_override_variable_array_capacity: false std: c++14 + std_flavor: std cast_format: "static_cast<{type}>({value})" # Provide non-empty values to override the type used for variable-length arrays in C++ types. variable_array_type_include: "" @@ -320,22 +322,27 @@ nunavut.lang.cpp: allocator_type: "" allocator_is_default_constructible: true ctor_convention: "default" - cetl++14-17_options: - variable_array_type_include: '"cetl/variable_length_array.hpp"' - variable_array_type_template: "cetl::VariableLengthArray<{TYPE}, {REBIND_ALLOCATOR}>" - variable_array_type_constructor_args: "{MAX_SIZE}" - allocator_include: '"cetl/pf17/sys/memory_resource.hpp"' - allocator_type: "cetl::pf17::pmr::polymorphic_allocator" - allocator_is_default_constructible: false - ctor_convention: "uses-trailing-allocator" - c++17-pmr_options: - variable_array_type_include: "" - variable_array_type_template: "std::vector<{TYPE}, {REBIND_ALLOCATOR}>" - variable_array_type_constructor_args: "" - allocator_include: "" - allocator_type: "std::pmr::polymorphic_allocator" - allocator_is_default_constructible: true - ctor_convention: "uses-trailing-allocator" + defaults: + cetl++14-17: + std: c++14 + std_flavor: cetl + variable_array_type_include: '"cetl/variable_length_array.hpp"' + variable_array_type_template: "cetl::VariableLengthArray<{TYPE}, {REBIND_ALLOCATOR}>" + variable_array_type_constructor_args: "{MAX_SIZE}" + allocator_include: '"cetl/pf17/sys/memory_resource.hpp"' + allocator_type: "cetl::pf17::pmr::polymorphic_allocator" + allocator_is_default_constructible: false + ctor_convention: "uses-trailing-allocator" + c++17-pmr: + std: c++17 + std_flavor: pmr + variable_array_type_include: "" + variable_array_type_template: "std::vector<{TYPE}, {REBIND_ALLOCATOR}>" + variable_array_type_constructor_args: "" + allocator_include: "" + allocator_type: "std::pmr::polymorphic_allocator" + allocator_is_default_constructible: true + ctor_convention: "uses-trailing-allocator" nunavut.lang.py: diff --git a/src/nunavut/lang/py/__init__.py b/src/nunavut/lang/py/__init__.py index 5b7b9e69..4624fbf6 100644 --- a/src/nunavut/lang/py/__init__.py +++ b/src/nunavut/lang/py/__init__.py @@ -1,21 +1,22 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating python. All filters in this module will be available in the template's global namespace as ``py``. """ from __future__ import annotations + +import base64 import builtins import functools -import keyword -import base64 import gzip -import pickle import itertools -from typing import Any, Iterable +import keyword +import pickle +from typing import Any, Dict, Iterable import pydsdl @@ -27,6 +28,7 @@ template_language_int_filter, template_language_list_filter, ) +from nunavut._utilities import cached_property from nunavut.lang import Language as BaseLanguage from nunavut.lang._common import TokenEncoder, UniqueNameGenerator @@ -38,12 +40,13 @@ class Language(BaseLanguage): PYTHON_RESERVED_IDENTIFIERS: list[str] = sorted(list(map(str, list(keyword.kwlist) + dir(builtins)))) - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self._language_options["enable_serialization_asserts"] = True + def _validate_language_options(self, defaults: Dict[str, Any], options: Dict[str, Any]) -> Dict[str, Any]: + # pylint: disable=unused-argument + options["enable_serialization_asserts"] = True # always enable serialization asserts for python + return options - @functools.lru_cache(maxsize=None) - def _get_token_encoder(self) -> TokenEncoder: + @cached_property + def _token_encoder(self) -> TokenEncoder: """ Caching getter to ensure we don't have to recompile TokenEncoders for each filter invocation. """ @@ -56,11 +59,12 @@ def get_includes(self, dep_types: Dependencies) -> list[str]: def filter_id(self, instance: Any, id_type: str = "any") -> str: raw_name = self.default_filter_id_for_target(instance) - return self._get_token_encoder().strop(raw_name, id_type) + return self._token_encoder.strop(raw_name, id_type) @template_context_filter def filter_to_template_unique_name(context: SupportsTemplateContext, base_token: str) -> str: + # pylint: disable=unused-argument """ Filter that takes a base token and forms a name that is very likely to be unique within the template the filter is invoked. This diff --git a/src/nunavut/lang/py/templates/base.j2 b/src/nunavut/lang/py/templates/base.j2 index d0592c22..7e8d2dea 100644 --- a/src/nunavut/lang/py/templates/base.j2 +++ b/src/nunavut/lang/py/templates/base.j2 @@ -5,10 +5,15 @@ -#} # AUTOGENERATED, DO NOT EDIT. # +{%- if nunavut.embed_auditing_info %} # Source file: # {{ T.source_file_path }} # # Generated at: {{ now_utc }} UTC +{%- else %} +# Source file: +# {{ T.source_file_path.name }} +{%- endif %} # Is deprecated: {{ T.deprecated and 'yes' or 'no' }} # Fixed port ID: {{ T.fixed_port_id }} # Full name: {{ T.full_name }} diff --git a/test/gentest_dsdl/test_dsdl.py b/test/gentest_dsdl/test_dsdl.py index 2d7033d3..49c34b4d 100644 --- a/test/gentest_dsdl/test_dsdl.py +++ b/test/gentest_dsdl/test_dsdl.py @@ -18,7 +18,7 @@ ("cpp", True, {"std": "c++17"}), ("c", False, {}), ("c", True, {}), - ("py", False, {}), + ("py", True, {}), ("html", False, {}), ], ) diff --git a/test/gentest_namespaces/test_namespaces.py b/test/gentest_namespaces/test_namespaces.py index 5bc120f4..b4900b90 100644 --- a/test/gentest_namespaces/test_namespaces.py +++ b/test/gentest_namespaces/test_namespaces.py @@ -9,24 +9,30 @@ from pathlib import Path import pytest +from pydsdl import Any, CompositeType, read_namespace + from nunavut import Namespace, build_namespace_tree from nunavut._utilities import YesNoDefault from nunavut.jinja import DSDLCodeGenerator from nunavut.lang import Language, LanguageContext, LanguageContextBuilder -from pydsdl import Any, CompositeType, read_namespace class DummyType(Any): """Fake dsdl 'any' type for testing.""" def __init__(self, namespace: str = "uavcan", name: str = "Dummy"): - self._full_name = "{}.{}".format(namespace, name) + self._full_name = f"{namespace}.{name}" # +-----------------------------------------------------------------------+ - # | DUCK TYPEING: CompositeType + # | DUCK TYPING: CompositeType # +-----------------------------------------------------------------------+ @property def full_name(self) -> str: + """ + Returns the full name of the object. + + :return: The full name as a string. + """ return self._full_name # +-----------------------------------------------------------------------+ @@ -36,8 +42,7 @@ def full_name(self) -> str: def __eq__(self, other: object) -> bool: if isinstance(other, DummyType): return self._full_name == other._full_name - else: - return False + return False def __str__(self) -> str: return self.full_name @@ -49,6 +54,17 @@ def __hash__(self) -> int: def gen_test_namespace( gen_paths: typing.Any, language_context: LanguageContext ) -> typing.Tuple[Namespace, str, typing.List[CompositeType]]: + """ + Generate a test namespace. + + :param gen_paths (typing.Any): The paths for generating the namespace. + :param language_context (LanguageContext): The language context for generating the namespace. + + :return (typing.Tuple[Namespace, str, typing.List[CompositeType]]): + A tuple containing the generated namespace, + the root namespace path, and a list of composite types. + + """ root_namespace_path = str(gen_paths.dsdl_dir / Path("scotec")) includes = [str(gen_paths.dsdl_dir / Path("uavcan"))] compound_types = read_namespace(root_namespace_path, includes, allow_unregulated_fixed_port_id=True) @@ -85,7 +101,7 @@ def test_get_all_types(gen_paths): # type: ignore """Verify the get_all_namespaces method in Namespace""" language_context = LanguageContextBuilder(include_experimental_languages=True).set_target_language("js").create() namespace, _, _ = gen_test_namespace(gen_paths, language_context) - index = dict() + index = {} for ns, path in namespace.get_all_types(): index[path] = ns @@ -140,7 +156,7 @@ def test_namespace_namespace_template(gen_paths): # type: ignore def test_namespace_generation(gen_paths): # type: ignore - """Test actually generating a namepace file.""" + """Test actually generating a namespace file.""" language_context = ( LanguageContextBuilder(include_experimental_languages=True) .set_target_language("js") @@ -162,7 +178,7 @@ def test_namespace_generation(gen_paths): # type: ignore assert outfile is not None - with open(str(outfile), "r") as json_file: + with open(str(outfile), "r", encoding="utf-8") as json_file: json_blob = json.load(json_file) assert json_blob is not None @@ -180,34 +196,38 @@ def test_build_namespace_tree_from_nothing(gen_paths): # type: ignore @pytest.mark.parametrize( - "language_key,expected_file_ext,expected_stropp_part_0,expected_stropp_part_1", + "language_key,expected_file_ext,expected_strop_part_0,expected_strop_part_1", [("c", ".h", "_typedef", "str"), ("py", ".py", "typedef", "str_")], ) # type: ignore def test_namespace_stropping( - gen_paths, language_key, expected_file_ext, expected_stropp_part_0, expected_stropp_part_1 + gen_paths, + language_key: str, + expected_file_ext: str, + expected_strop_part_0: str, + expected_strop_part_1: str, ): """Test generating a namespace that uses a reserved keyword for a given language.""" language_context = ( LanguageContextBuilder(include_experimental_languages=True).set_target_language(language_key).create() ) - namespace, root_namespace_path, compound_types = gen_test_namespace(gen_paths, language_context) + namespace, _, compound_types = gen_test_namespace(gen_paths, language_context) assert len(compound_types) == 2 generator = DSDLCodeGenerator( namespace, generate_namespace_types=YesNoDefault.YES, templates_dir=gen_paths.templates_dir / Path("default") ) generator.generate_all() - expected_stropped_ns = "scotec.{}.{}".format(expected_stropp_part_0, expected_stropp_part_1) + expected_stropped_ns = f"scotec.{expected_strop_part_0}.{expected_strop_part_1}" outfile = gen_paths.find_outfile_in_namespace(expected_stropped_ns, namespace) assert outfile is not None - with open(str(outfile), "r") as json_file: + with open(str(outfile), "r", encoding="utf-8") as json_file: json_blob = json.load(json_file) assert json_blob is not None output_path_for_stropped = namespace.find_output_path_for_type(compound_types[1]) expected_stable_path = gen_paths.out_dir / "scotec" - expected_path_and_file = expected_stable_path / expected_stropp_part_0 / expected_stropp_part_1 / "ATOMIC_TYPE_0_1" + expected_path_and_file = expected_stable_path / expected_strop_part_0 / expected_strop_part_1 / "ATOMIC_TYPE_0_1" assert expected_path_and_file.with_suffix(expected_file_ext) == output_path_for_stropped diff --git a/test/gentest_nnvg/test_nnvg.py b/test/gentest_nnvg/test_nnvg.py index 0cfeeeb5..8197781f 100644 --- a/test/gentest_nnvg/test_nnvg.py +++ b/test/gentest_nnvg/test_nnvg.py @@ -1,8 +1,9 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # +# cSpell:ignore scotec, herringtec import json import os import pathlib @@ -19,7 +20,7 @@ @pytest.mark.parametrize("env_var_name", ["DSDL_INCLUDE_PATH"]) def test_DSDL_INCLUDE_PATH(gen_paths: typing.Any, run_nnvg: typing.Callable, env_var_name: str) -> None: """ - Verify that supported environment variables are used by nnvg. + Verify that the DSDL_INCLUDE_PATH environment variable and any aliases are used by nnvg. """ nnvg_args0 = [ @@ -38,7 +39,7 @@ def test_DSDL_INCLUDE_PATH(gen_paths: typing.Any, run_nnvg: typing.Callable, env scotec_path = (gen_paths.dsdl_dir / pathlib.Path("scotec")).as_posix() herringtec_path = (gen_paths.dsdl_dir / pathlib.Path("herringtec")).as_posix() - env = {env_var_name: "{}{}{}".format(herringtec_path, os.pathsep, scotec_path)} + env = {env_var_name: f"{herringtec_path}{os.pathsep}{scotec_path}"} run_nnvg(gen_paths, nnvg_args0, env=env) @@ -94,7 +95,7 @@ def test_list_inputs(gen_paths: typing.Any, run_nnvg: typing.Callable, generate_ (gen_paths.dsdl_dir / pathlib.Path("scotec")).as_posix(), "--list-inputs", (gen_paths.dsdl_dir / pathlib.Path("uavcan")).as_posix(), - "--generate-support={}".format(generate_support), + f"--generate-support={generate_support}", ] if generate_support == "only": diff --git a/test/gettest_properties/test_properties.py b/test/gettest_properties/test_properties.py index 9551ef2f..2d9c36c3 100644 --- a/test/gettest_properties/test_properties.py +++ b/test/gettest_properties/test_properties.py @@ -1,16 +1,15 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright (C) 2018-2021 OpenCyphal Development Team # This software is distributed under the terms of the MIT License. # import pathlib +import re from pathlib import Path import pydsdl -import pytest import yaml -import re from nunavut import build_namespace_tree from nunavut.jinja import DSDLCodeGenerator @@ -30,7 +29,7 @@ def test_issue_277(gen_paths): # type: ignore "allocator_include": '"MyCrazyAllocator.hpp"', "allocator_type": "MyCrazyAllocator", "allocator_is_default_constructible": True, - "ctor_convention": "uses-leading-allocator" + "ctor_convention": "uses-leading-allocator", } vla_decl_pattern = re.compile(r"\b|^MyCrazyArray\B") @@ -44,7 +43,7 @@ def test_issue_277(gen_paths): # type: ignore LanguageClassLoader.to_language_module_name("cpp"): {Language.WKCV_LANGUAGE_OPTIONS: override_language_options} } - with open(overrides_file, "w") as overrides_handle: + with open(overrides_file, "w", encoding="utf-8") as overrides_handle: yaml.dump(overrides_data, overrides_handle) root_namespace = str(gen_paths.dsdl_dir / Path("proptest")) @@ -52,7 +51,7 @@ def test_issue_277(gen_paths): # type: ignore language_context = ( LanguageContextBuilder(include_experimental_languages=True) .set_target_language("cpp") - .set_additional_config_files([overrides_file]) + .add_config_files(overrides_file) .create() ) namespace = build_namespace_tree(compound_types, root_namespace, gen_paths.out_dir, language_context) @@ -64,12 +63,11 @@ def test_issue_277(gen_paths): # type: ignore assert outfile is not None - found_vla_decl = False found_vla_include = False found_alloc_include = False found_vla_constructor_args = False - with open(str(outfile), "r") as header_file: + with open(str(outfile), "r", encoding="utf-8") as header_file: for line in header_file: if not found_vla_decl and vla_decl_pattern.search(line): found_vla_decl = True diff --git a/tox.ini b/tox.ini index 4686c7c3..4820700f 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,13 @@ # The standard version to develop against is 3.10. # [tox] -envlist = {py37,py38,py39,py310,py311}-{test,nnvg,doctest,rstdoctest},lint,report,docs +envlist = {py37,py38,py39,py310,py311,py312}-{test,nnvg,doctest,rstdoctest},lint,report,docs [base] deps = - Sybil >= 3.0.0, < 4.0.0 - pytest >= 5.4.3, < 7.0.0 + Sybil + pytest pytest-timeout coverage types-PyYAML @@ -19,19 +19,21 @@ deps = autopep8 rope isort + nox # +---------------------------------------------------------------------------+ # | CONFIGURATION # +---------------------------------------------------------------------------+ -[flake8] -max-complexity = 10 +[pylint] max-line-length = 120 -doctests = True -statistics = True -show-source = True -ignore = E203, W503 +max-args = 8 +max-attributes = 12 +ignore-paths = .*/(jinja2|markupsafe)/.* +min-public-methods = 0 +source-roots = src +disable = no-else-return,invalid-name [pytest] @@ -39,15 +41,11 @@ log_file = pytest.log log_level = DEBUG log_cli = true log_cli_level = WARNING -addopts: --keep-generated norecursedirs = submodules .* build* verification .tox -# The fill fixtures deprecation warning comes from Sybil, which we don't have any control over. Remove when updated. -filterwarnings = - error - ignore:A private pytest class or function was used.:DeprecationWarning - +addopts = -p no:doctest [coverage:run] +data_file = build/coverage-py/.coverage branch=True parallel=True include = @@ -70,7 +68,7 @@ source = [coverage:report] -exclude_lines = +exclude_also = pragma: no cover def __repr__ raise AssertionError @@ -78,7 +76,7 @@ exclude_lines = assert False if False: if __name__ == .__main__.: - +omit = *.j2 [doc8] max-line-length = 120 @@ -96,6 +94,7 @@ warn_redundant_casts = True warn_unused_ignores = True show_error_context = True mypy_path = src +exclude = (jinja2|markupsafe) [mypy-pydsdl] ignore_missing_imports = True @@ -140,8 +139,8 @@ commands = nnvg: -O {envtmpdir} \ nnvg: --target-language cpp \ nnvg: --experimental-languages \ + nnvg: --language-standard c++17-pmr \ nnvg: -v \ - nnvg: --dry-run \ nnvg: {toxinidir}/submodules/public_regulated_data_types/uavcan test: coverage run \ @@ -163,34 +162,12 @@ commands = [testenv:docs] deps = -rrequirements.txt - sphinx ~= 6.2.1 - sphinx-rtd-theme + sphinx readthedocs-sphinx-ext commands = sphinx-build -W -b html {toxinidir} {envtmpdir} -[testenv:gen-apidoc] -allowlist_externals = rm -deps = - sphinx-autoapi - -commands = - rm -rf {toxinidir}/docs/api - sphinx-apidoc \ - --doc-project library \ - --output-dir {toxinidir}/docs/api \ - --ext-autodoc \ - --ext-intersphinx \ - --templatedir={toxinidir}/docs/sphinx_templates/apidoc \ - --tocfile=library \ - --module-first \ - src \ - "**/conftest.py" \ - "src/nunavut/jinja/jinja2/**" \ - "src/nunavut/jinja/markupsafe/**" - - [testenv:report] deps = coverage skip_install = true @@ -205,7 +182,7 @@ basepython = python3.10 deps = {[dev]deps} black - flake8 + pylint doc8 Pygments mypy @@ -214,15 +191,19 @@ deps = types-PyYAML commands = - flake8 --benchmark --tee --output-file={envtmpdir}/flake8.txt --filename=*.py --exclude=**/jinja2/*,**/markupsafe/* src + pylint --reports=y \ + --rcfile={toxinidir}/tox.ini \ + --output={envtmpdir}/pylint.txt \ + --output-format=json2 \ + --clear-cache-post-run=y \ + --confidence=HIGH \ + {toxinidir}/src/nunavut black --check --line-length 120 --force-exclude '(/jinja2/|/markupsafe\/)' src doc8 {toxinidir}/docs - mypy -m nunavut \ - -m nunavut.jinja \ - -p nunavut.lang \ - --cache-dir {envtmpdir} \ - --txt-report {envtmpdir}/mypy-report-lib \ - --config-file {toxinidir}/tox.ini + mypy -p nunavut \ + --cache-dir {envtmpdir} \ + --txt-report {envtmpdir}/mypy-report-lib \ + --config-file {toxinidir}/tox.ini [testenv:package] @@ -247,9 +228,4 @@ deps = {[testenv:docs]deps} {[testenv:lint]deps} commands = - mypy -m nunavut \ - -m nunavut.jinja \ - -p nunavut.lang \ - --config-file {toxinidir}/tox.ini \ - --install-types \ - --non-interactive + python --version diff --git a/verification/.devcontainer/devcontainer.json b/verification/.devcontainer/devcontainer.json index b2de9e38..533a16a2 100644 --- a/verification/.devcontainer/devcontainer.json +++ b/verification/.devcontainer/devcontainer.json @@ -20,8 +20,7 @@ "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", - "ms-python.flake8", - "ms-python.autopep8" + "ms-python.pylint" ] } }, diff --git a/verification/CMakeLists.txt b/verification/CMakeLists.txt index a970e48e..63fbfcd6 100644 --- a/verification/CMakeLists.txt +++ b/verification/CMakeLists.txt @@ -285,7 +285,7 @@ endif() if(NUNAVUT_VERIFICATION_LANG_STANDARD STREQUAL "cetl++14-17" OR NUNAVUT_VERIFICATION_LANG_STANDARD STREQUAL "c++17-pmr") # - # Generate serialization support headers with config matching the langauge standard + # Generate serialization support headers with config matching the language standard # create_dsdl_target(nunavut-support-array-with-allocator ${NUNAVUT_VERIFICATION_LANG} @@ -301,7 +301,7 @@ if(NUNAVUT_VERIFICATION_LANG_STANDARD STREQUAL "cetl++14-17" OR NUNAVUT_VERIFICA "only") # - # Generate additional types with config matching the langauge standard + # Generate additional types with config matching the language standard # create_dsdl_target(dsdl-test-array-with-allocator ${NUNAVUT_VERIFICATION_LANG}