From 8790ff93871a4fc8c554d6a3fb0c88da5a6a5bc0 Mon Sep 17 00:00:00 2001 From: Scott Dixon Date: Thu, 7 Mar 2024 08:14:40 -0800 Subject: [PATCH] Issue #324 and #318 fixes Things in this PR 1. fix issue #324 to add `--embed-auditing-info` and to remove, by default, all ephemeral meta data from the built-in templates. 2. fix issue #318 by adding `pascal` to the reserved word list for C 3. fixup copyrights to be dateless (in library) 4. general cleanup based on linter feedback as I encountered it. 5. modernized pytest dependencies 6. updated pydsdl dependency to use the latest 7. updated development environment detils (e.g. newer toxic container, improvements to coverage configuration and cSpell, switch to pylint, etc.) 8.various documentation improvements and cleanup --- .devcontainer/devcontainer.json | 3 +- .github/verify.py | 6 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 12 +- .gitignore | 2 + .readthedocs.yaml | 2 + .vscode/extensions.json | 3 +- .vscode/launch.json | 21 +- .vscode/nunavut-words.txt | 52 +++ .vscode/settings.json | 74 +--- CONTRIBUTING.rst | 31 +- LICENSE.rst | 2 +- README.rst | 2 +- conf.py | 6 +- conftest.py | 212 +++++---- cyphal-words.txt | 10 + docs/templates.rst | 12 + setup.cfg | 5 +- src/nunavut/__init__.py | 50 ++- src/nunavut/__main__.py | 6 +- src/nunavut/_dependencies.py | 6 +- src/nunavut/_exceptions.py | 6 +- src/nunavut/_generators.py | 29 +- src/nunavut/_namespace.py | 7 +- src/nunavut/_postprocessors.py | 6 +- src/nunavut/_templates.py | 6 +- src/nunavut/_utilities.py | 278 +++++++++++- src/nunavut/_version.py | 8 +- src/nunavut/cli/__init__.py | 66 ++- src/nunavut/cli/runners.py | 52 ++- src/nunavut/jinja/__init__.py | 164 +++++-- src/nunavut/jinja/environment.py | 415 ++++++++++++++---- src/nunavut/jinja/extensions.py | 54 ++- src/nunavut/jinja/loaders.py | 24 +- src/nunavut/lang/__init__.py | 130 ++++-- src/nunavut/lang/_common.py | 100 +++-- src/nunavut/lang/_config.py | 163 +++---- src/nunavut/lang/_language.py | 114 +++-- src/nunavut/lang/c/__init__.py | 19 +- src/nunavut/lang/c/support/__init__.py | 6 +- src/nunavut/lang/c/templates/__init__.py | 6 +- src/nunavut/lang/c/templates/base.j2 | 10 +- src/nunavut/lang/cpp/__init__.py | 381 ++++++++++++---- src/nunavut/lang/cpp/support/__init__.py | 6 +- src/nunavut/lang/cpp/templates/__init__.py | 6 +- .../lang/cpp/templates/_composite_type.j2 | 22 +- src/nunavut/lang/cpp/templates/_fields.j2 | 2 +- src/nunavut/lang/cpp/templates/base.j2 | 52 ++- src/nunavut/lang/html/__init__.py | 28 +- src/nunavut/lang/html/support/__init__.py | 11 +- src/nunavut/lang/js/__init__.py | 6 +- src/nunavut/lang/js/support/__init__.py | 11 +- src/nunavut/lang/properties.yaml | 45 +- src/nunavut/lang/py/__init__.py | 30 +- src/nunavut/lang/py/templates/base.j2 | 5 + test/gentest_dsdl/test_dsdl.py | 2 +- test/gentest_namespaces/test_namespaces.py | 48 +- test/gentest_nnvg/test_nnvg.py | 13 +- test/gettest_properties/test_properties.py | 16 +- tox.ini | 85 ++-- verification/.devcontainer/devcontainer.json | 3 +- verification/CMakeLists.txt | 4 +- 62 files changed, 2017 insertions(+), 941 deletions(-) create mode 100644 .vscode/nunavut-words.txt create mode 100644 cyphal-words.txt diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5fd082b4..ba9c8008 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,8 +16,7 @@ "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", - "ms-python.flake8", - "ms-python.autopep8" + "ms-python.pylint" ] } }, diff --git a/.github/verify.py b/.github/verify.py index db9ab0bd..04b9bcc6 100755 --- a/.github/verify.py +++ b/.github/verify.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Command-line helper for running verification builds. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a8d0a62..163e5ec7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ jobs: release: if: ${{ github.event_name == 'release' && !github.event.release.prerelease }} runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36b99958..a6d398ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: jobs: test: runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 steps: - uses: actions/checkout@v3 with: @@ -95,7 +95,7 @@ jobs: compat-test-python3-windows-and-mac: strategy: matrix: - python3-version: ['10','11'] + python3-version: ['11','12'] python3-platform: ['windows-latest', 'macos-latest'] runs-on: ${{ matrix.python3-platform }} needs: test @@ -115,9 +115,9 @@ jobs: compat-test-python3-ubuntu: strategy: matrix: - python3-version: ['7', '8', '9', '10', '11'] + python3-version: ['7', '8', '9', '10', '11', '12'] runs-on: ubuntu-latest - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 needs: test steps: - uses: actions/checkout@v3 @@ -129,7 +129,7 @@ jobs: language-verification-c: runs-on: ubuntu-latest needs: test - container: ghcr.io/opencyphal/toolshed:ts22.4.1 + container: ghcr.io/opencyphal/toolshed:ts22.4.2 strategy: matrix: build_type: [Debug, Release, MinSizeRel] @@ -223,7 +223,7 @@ jobs: language-verification-python: runs-on: ubuntu-latest needs: test - container: ghcr.io/opencyphal/toxic:tx22.4.1 + container: ghcr.io/opencyphal/toxic:tx22.4.2 steps: - uses: actions/checkout@v3 with: diff --git a/.gitignore b/.gitignore index 298af685..2ebdd67f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__ *.egg-info *.patch .tox +.nox .coverage* .venv coverage.xml @@ -26,6 +27,7 @@ dist .DS_Store prof out +venv # Eclipse .metadata diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c70118b6..b47700b5 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,6 +3,8 @@ build: os: ubuntu-22.04 tools: python: "3.11" +sphinx: + configuration: conf.py python: install: - requirements: requirements.txt diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 8152e0c8..e45ff5b8 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,7 +8,6 @@ "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", - "ms-python.flake8", - "ms-python.autopep8" + "ms-python.pylint" ] } diff --git a/.vscode/launch.json b/.vscode/launch.json index 6f2412d0..b7419f4c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,7 +6,7 @@ "configurations": [ { "name": "Python: nnvg c++", - "type": "python", + "type": "debugpy", "request": "launch", "module": "nunavut", "cwd": "${workspaceFolder}/src", @@ -18,22 +18,33 @@ }, { "name": "Pytest: current test", - "type": "python", + "type": "debugpy", "request": "launch", "module": "pytest", "args": [ "--keep-generated", + "--rootdir=${workspaceFolder}", "${file}" ], - "console": "internalConsole", + "cwd": "${workspaceFolder}" + }, + { + "name": "Pytest: all doc tests", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": [ + "--keep-generated", + "--rootdir=${workspaceFolder}", + "${workspaceFolder}/src" + ], "cwd": "${workspaceFolder}" }, { "name": "Pytest: all tests", - "type": "python", + "type": "debugpy", "request": "launch", "module": "pytest", - "console": "internalConsole", "cwd": "${workspaceFolder}" }, { diff --git a/.vscode/nunavut-words.txt b/.vscode/nunavut-words.txt new file mode 100644 index 00000000..6c8090df --- /dev/null +++ b/.vscode/nunavut-words.txt @@ -0,0 +1,52 @@ +addoption +allclose +astype +autouse +behaviour +bitorder +bools +builtins +Bxxx +caplog +CDEF +codegen +doctests +dryrun +dtype +EDCB +elementwise +emptylines +endianness +errstate +fillvalue +finalizer +fpid +frombuffer +functor +functors +htmlcov +itemsize +lctx +markupsafe +maxsplit +nbytes +ndarray +ndim +nnvg +noxfile +outdir +packbits +postprocessor +postprocessors +roadmap +rtype +Sriram +tobytes +transcompilation +typecheck +Unionant +unpackbits +unseparate +unstropped +WKCV +Xlang diff --git a/.vscode/settings.json b/.vscode/settings.json index c8527680..2132ab87 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,12 +3,18 @@ "python.testing.cwd": "${workspaceFolder}", "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, + "pylint.args": [ + "--rcfile=${workspaceFolder}/tox.ini" + ], "black-formatter.args": [ "--line-length=120" ], "mypy-type-checker.args": [ "--config-file=${workspaceFolder}/tox.ini" ], + "flake8.args": [ + "--line-length=120" + ], "files.exclude": { "**/.git": true, "**/.svn": true, @@ -168,56 +174,20 @@ "environment": "", "sourceFileMap": "${sourceFileMapObj}" }, - "cSpell.words": [ - "allclose", - "astype", - "autouse", - "bitorder", - "bools", - "builtins", - "Bxxx", - "caplog", - "CDEF", - "codegen", - "Cyphal", - "doctests", - "DSDL", - "dtype", - "EDCB", - "elementwise", - "emptylines", - "endianness", - "errstate", - "fillvalue", - "fpid", - "frombuffer", - "htmlcov", - "itemsize", - "Kirienko", - "maxsplit", - "nbytes", - "ndarray", - "ndim", - "nnvg", - "noxfile", - "opencyphal", - "outdir", - "packbits", - "Pavel", - "postprocessor", - "postprocessors", - "pycyphal", - "pydsdl", - "roadmap", - "Sriram", - "tobytes", - "transcompilation", - "typecheck", - "uavcan", - "unpackbits", - "unseparate", - "unstropped", - "WKCV", - "Unionant" - ], + "cSpell.allowCompoundWords": true, + "cSpell.caseSensitive": false, + "cSpell.customDictionaries": { + "cyphal" : { + "name": "Cyphal-words", + "path": "${workspaceRoot}/cyphal-words.txt", + "description": "Words used in Cyphal and UAVCAN to add to spell checker dictionaries.", + "addWords": true + }, + "nunavut" : { + "name": "Nunavut-words", + "path": "${workspaceRoot}/.vscode/nunavut-words.txt", + "description": "Words used in in the Nunavut codebase to add to spell checker dictionaries.", + "addWords": true + } + } } diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c22bf796..cb715996 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -24,7 +24,7 @@ your dev environment setup. Tools ************************************************ -tox -e local +tox devenv -e local ================================================ I highly recommend using the local tox environment when doing python development. It'll save you hours @@ -33,8 +33,8 @@ global python environment. You can install tox from brew on osx or apt-get on GN recommend the following environment for vscode:: git submodule update --init --recursive - tox -e local - source .tox/local/bin/activate + tox devenv -e local + source venv/bin/activate cmake @@ -58,8 +58,8 @@ Do:: cd path/to/nunavut git submodule update --init --recursive - tox -e local - source .tox/local/bin/activate + tox devenv -e local + source venv/bin/activate code . Then install recommended extensions. @@ -72,16 +72,16 @@ To run the full suite of `tox`_ tests locally you'll need docker. Once you have and running do:: git submodule update --init --recursive - docker pull ghcr.io/opencyphal/toxic:tx22.4.1 - docker run --rm -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.1 tox + docker pull ghcr.io/opencyphal/toxic:tx22.4.2 + docker run --rm -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.2 tox To run a limited suite using only locally available interpreters directly on your host machine, -skip the docker invocations and use ``tox -s``. +skip the docker invocations and use ``tox run -s``. To run the language verification build you'll need to use a different docker container:: - docker pull ghcr.io/opencyphal/toolshed:ts22.4.3 - docker run --rm -it -v $PWD:/workspace ghcr.io/opencyphal/toolshed:ts22.4.3 + docker pull ghcr.io/opencyphal/toolshed:ts22.4.5 + docker run --rm -it -v $PWD:/workspace ghcr.io/opencyphal/toolshed:ts22.4.5 cd /workspace ./.github/verify.py -l c ./.github/verify.py -l cpp @@ -187,7 +187,7 @@ Building The Docs We rely on `read the docs`_ to build our documentation from github but we also verify this build as part of our tox build. This means you can view a local copy after completing a full, successful test run (See `Running The Tests`_) or do -:code:`docker run --rm -t -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.1 /bin/sh -c "tox -e docs"` to build +:code:`docker run --rm -t -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.2 /bin/sh -c "tox run -e docs"` to build the docs target. You can open the index.html under ``.tox/docs/tmp/index.html`` or run a local web-server:: @@ -198,15 +198,6 @@ Of course, you can just use `Visual Studio Code`_ to build and preview the docs :code:`> reStructuredText: Open Preview`. -apidoc -================================================ - -We manually generate the api doc using ``sphinx-apidoc``. To regenerate use ``tox -e gen-apidoc``. - -.. warning:: - - ``tox -e gen-apidoc`` will start by deleting the docs/api directory. - ************************************************ Coverage and Linting Reports ************************************************ diff --git a/LICENSE.rst b/LICENSE.rst index 16cfd48d..d5ee39ee 100644 --- a/LICENSE.rst +++ b/LICENSE.rst @@ -1,5 +1,5 @@ ################################################ -Licence +License ################################################ ************************************* diff --git a/README.rst b/README.rst index d2cfb399..9cbe3845 100644 --- a/README.rst +++ b/README.rst @@ -68,7 +68,7 @@ Nunavut is invoked to generate code for the former. .. code-block:: shell - nnvg --target-language c --target-endianness=little --enable-serialization-asserts public_regulated_data_types/reg --lookup-dir public_regulated_data_types/uavcan + nnvg --target-language c --enable-serialization-asserts public_regulated_data_types/reg --lookup-dir public_regulated_data_types/uavcan Generate HTML documentation pages using the command-line tool ------------------------------------------------------------- diff --git a/conf.py b/conf.py index 5439a007..f402a5d4 100644 --- a/conf.py +++ b/conf.py @@ -24,13 +24,13 @@ _version_tuple = nunavut_version.split(".") # The short X.Y version -version = "{}.{}".format(_version_tuple[0], _version_tuple[1]) +version = f"{_version_tuple[0]}.{_version_tuple[1]}" # The full version, including alpha/beta/rc tags release = nunavut_version -exclude_patterns = ["**/test", "**/.nox"] +exclude_patterns = ["**/test"] -with open(".gitignore", "r") as gif: +with open(".gitignore", "r", encoding="utf-8") as gif: for line in gif: stripped = line.strip() if len(stripped) > 0 and not stripped.startswith("#"): diff --git a/conftest.py b/conftest.py index c27056ac..99453557 100644 --- a/conftest.py +++ b/conftest.py @@ -1,9 +1,10 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ -Fixtures for our tests. +Configuration for pytest tests including fixtures and hooks. """ import logging @@ -20,69 +21,24 @@ import pydsdl import pytest from sybil import Sybil - -try: - from sybil.parsers.codeblock import PythonCodeBlockParser -except ImportError: - from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser - -from sybil.parsers.doctest import DocTestParser +from sybil.parsers.rest import DocTestParser, PythonCodeBlockParser from nunavut import Namespace -# +-------------------------------------------------------------------------------------------------------------------+ -# | PYTEST HOOKS -# +-------------------------------------------------------------------------------------------------------------------+ - - -def pytest_configure(config: typing.Any) -> None: - """ - See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks - """ - # pydsdl._dsdl_definition is reeeeeeealy verbose at the INFO level and below. Turn this down to reduce - # scroll-blindness. - logging.getLogger("pydsdl._dsdl_definition").setLevel(logging.WARNING) - # A lot of DEBUG noise in the other loggers so we'll tune this down to INFO and higher. - logging.getLogger("pydsdl._namespace").setLevel(logging.INFO) - logging.getLogger("pydsdl._data_type_builder").setLevel(logging.INFO) - - -def pytest_addoption(parser): # type: ignore - """ - See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks - """ - parser.addoption( - "--keep-generated", - action="store_true", - help=textwrap.dedent( - """ - If set then the temporary directory used to generate files for each test will be left after - the test has completed. Normally this directory is temporary and therefore cleaned up automatically. - - :: WARNING :: - This will leave orphaned files on disk. They won't be big but there will be a lot of them. - - :: WARNING :: - Do not run tests in parallel when using this option. - """ - ), - ) - - # +-------------------------------------------------------------------------------------------------------------------+ # | TEST FIXTURES # +-------------------------------------------------------------------------------------------------------------------+ @pytest.fixture -def run_nnvg(request): # type: ignore +def run_nnvg(request: pytest.FixtureRequest) -> typing.Callable: # pylint: disable=unused-argument """ - Test helper for invoking the nnvg commandline script as part of a unit test. + Test helper for invoking the nnvg command-line script as part of a unit test. """ def _run_nnvg( - gen_paths: typing.Any, + _: typing.Any, args: typing.List[str], check_result: bool = True, env: typing.Optional[typing.Dict[str, str]] = None, @@ -102,8 +58,7 @@ def _run_nnvg( except subprocess.CalledProcessError as e: if raise_called_process_error: raise e - else: - raise AssertionError(e.stderr.decode("utf-8")) + raise AssertionError(e.stderr.decode("utf-8")) from e return _run_nnvg @@ -113,7 +68,7 @@ class GenTestPaths: def __init__(self, test_file: str, keep_temporaries: bool, node_name: str): test_file_path = pathlib.Path(test_file) - self.test_name = "{}_{}".format(test_file_path.parent.stem, node_name) + self.test_name = f"{test_file_path.parent.stem}_{node_name}" self.test_dir = test_file_path.parent search_dir = self.test_dir.resolve() while search_dir.is_dir() and not (search_dir / pathlib.Path("src")).is_dir(): @@ -125,19 +80,25 @@ def __init__(self, test_file: str, keep_temporaries: bool, node_name: str): self.lang_src_dir = self.root_dir / pathlib.Path("src") / pathlib.Path("nunavut") / pathlib.Path("lang") self._keep_temp = keep_temporaries - self._out_dir = None # type: typing.Optional[pathlib.Path] - self._build_dir = None # type: typing.Optional[pathlib.Path] - self._dsdl_dir = None # type: typing.Optional[pathlib.Path] - self._temp_dirs = [] # type: typing.List[tempfile.TemporaryDirectory] - print('Paths for test "{}" under dir {}'.format(self.test_name, self.test_dir)) - print("(root directory: {})".format(self.root_dir)) + self._out_dir: typing.Optional[pathlib.Path] = None + self._build_dir: typing.Optional[pathlib.Path] = None + self._dsdl_dir: typing.Optional[pathlib.Path] = None + self._temp_dirs: typing.List[tempfile.TemporaryDirectory] = [] + print(f'Paths for test "{self.test_name}" under dir {self.test_dir}') + print(f"(root directory: {self.root_dir})") def test_path_finalizer(self) -> None: + """ + Finalizer to clean up any temporary directories created during the test. + """ for temporary_dir in self._temp_dirs: temporary_dir.cleanup() self._temp_dirs.clear() def create_new_temp_dir(self, dir_key: str) -> pathlib.Path: + """ + Create a new temporary directory for the test case. + """ if self._keep_temp: result = self._ensure_dir(self.build_dir / pathlib.Path(dir_key)) else: @@ -157,6 +118,9 @@ def out_dir(self) -> pathlib.Path: @property def build_dir(self) -> pathlib.Path: + """ + The directory to place build artifacts under for this test case. + """ if self._build_dir is None: self._build_dir = self._ensure_dir(self.root_dir / pathlib.Path("build")) return self._build_dir @@ -165,7 +129,10 @@ def build_dir(self) -> pathlib.Path: def find_outfile_in_namespace( typename: str, namespace: Namespace, type_version: pydsdl.Version = None ) -> typing.Optional[str]: - found_outfile = None # type: typing.Optional[str] + """ + Find the output file for a given type in a namespace. + """ + found_outfile: typing.Optional[str] = None for dsdl_type, outfile in namespace.get_all_types(): if dsdl_type.full_name == typename: if type_version is not None: @@ -176,8 +143,8 @@ def find_outfile_in_namespace( # of the type we're looking for. elif found_outfile is not None: raise RuntimeError( - "Type {} had more than one version for this test but no type version argument" - " was provided.".format(typename) + f"Type {typename} had more than one version for this test but no type version argument" + " was provided." ) else: found_outfile = str(outfile) @@ -191,14 +158,14 @@ def _ensure_dir(path_dir: pathlib.Path) -> pathlib.Path: except FileExistsError: pass if not path_dir.exists() or not path_dir.is_dir(): - raise RuntimeWarning('Test directory "{}" was not setup properly. Tests may fail.'.format(path_dir)) + raise RuntimeWarning(f'Test directory "{path_dir}" was not setup properly. Tests may fail.') return path_dir @pytest.fixture(scope="function") -def gen_paths(request): # type: ignore +def gen_paths(request: pytest.FixtureRequest) -> GenTestPaths: """ - Used by the "gentest" unittests in Nunavut to standardize output paths for generated code created as part of + Used by the "gentest" unit tests in Nunavut to standardize output paths for generated code created as part of the tests. Use the --keep-generated argument to disable the auto-clean behaviour this fixture provides by default. """ g = GenTestPaths(str(request.fspath), request.config.option.keep_generated, request.node.name) @@ -206,9 +173,23 @@ def gen_paths(request): # type: ignore return g +@pytest.fixture(scope="module") +def gen_paths_for_module(request: pytest.FixtureRequest) -> GenTestPaths: # pylint: disable=unused-argument + """ + Used by our Sybil doctests in Nunavut to standardize output paths for generated code created as part of + the tests. Use the --keep-generated argument to disable the auto-clean behaviour this fixture provides by default. + + Note: this fixture is different than gen_paths because it is scoped to the module level. This is useful for + Sybil tests that share temporary files across different test blocks within the same document. + """ + g = GenTestPaths(str(request.fspath), request.config.option.keep_generated, request.node.name) + request.addfinalizer(g.test_path_finalizer) + return g + + class _UniqueNameEvaluator: def __init__(self) -> None: - self._found_names = set() # type: typing.Set[str] + self._found_names: typing.Set[str] = set() def __call__(self, expected_pattern: str, actual_value: str) -> None: assert re.match(expected_pattern, actual_value) is not None @@ -217,7 +198,7 @@ def __call__(self, expected_pattern: str, actual_value: str) -> None: @pytest.fixture(scope="function") -def unique_name_evaluator(request): # type: ignore +def unique_name_evaluator(request: pytest.FixtureRequest) -> _UniqueNameEvaluator: # pylint: disable=unused-argument """ Class that defined ``assert_is_expected_and_unique`` allowing assertion that a set of values in a single test adhere to a provided pattern and are unique values (compared to other values @@ -240,11 +221,11 @@ def test_is_unique(unique_name_evaluator) -> None: @pytest.fixture -def assert_language_config_value(request): # type: ignore +def assert_language_config_value(request: pytest.FixtureRequest) -> typing.Callable: # pylint: disable=unused-argument """ Assert that a given configuration value is set for the target language. """ - from nunavut.lang import LanguageContext, LanguageContextBuilder + from nunavut.lang import LanguageContext, LanguageContextBuilder # pylint: disable=import-outside-toplevel def _assert_language_config_value( target_language: typing.Union[str, LanguageContext], @@ -271,7 +252,7 @@ def _assert_language_config_value( @pytest.fixture -def jinja_filter_tester(request): # type: ignore +def jinja_filter_tester(request: pytest.FixtureRequest): # pylint: disable=unused-argument """ Use to create fluent but testable documentation for Jinja filters and tests @@ -311,17 +292,17 @@ def filter_dummy(env, input): jinja_filter_tester(filter_dummy, template, rendered, lctx, I=I) """ - from nunavut.jinja.jinja2 import DictLoader - from nunavut.lang import LanguageContext, LanguageContextBuilder + from nunavut.jinja.jinja2 import DictLoader # pylint: disable=import-outside-toplevel + from nunavut.lang import LanguageContext, LanguageContextBuilder # pylint: disable=import-outside-toplevel def _make_filter_test_template( filter_or_list_of_filters: typing.Union[None, typing.Callable, typing.List[typing.Callable]], body: str, expected: str, target_language_or_language_context: typing.Union[str, LanguageContext], - **globals: typing.Optional[typing.Dict[str, typing.Any]] + **additional_globals: typing.Optional[typing.Dict[str, typing.Any]], ) -> str: - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder # pylint: disable=import-outside-toplevel if isinstance(target_language_or_language_context, LanguageContext): lctx = target_language_or_language_context @@ -333,23 +314,23 @@ def _make_filter_test_template( ) if filter_or_list_of_filters is None: - additional_filters = dict() # type: typing.Optional[typing.Dict[str, typing.Callable]] + additional_filters: typing.Optional[typing.Dict[str, typing.Callable]] = {} elif isinstance(filter_or_list_of_filters, list): - additional_filters = dict() - for filter in filter_or_list_of_filters: - additional_filters[filter.__name__] = filter + additional_filters = {} + for filter_method in filter_or_list_of_filters: + additional_filters[filter_method.__name__] = filter_method else: additional_filters = {filter_or_list_of_filters.__name__: filter_or_list_of_filters} - e = CodeGenEnvironment( - lctx=lctx, - loader=DictLoader({"test": body}), - allow_filter_test_or_use_query_overwrite=True, - additional_filters=additional_filters, - additional_globals=globals, + e = ( + CodeGenEnvironmentBuilder(DictLoader({"test": body}), lctx) + .set_allow_filter_test_or_use_query_overwrite(True) + .add_filters(**additional_filters) + .add_globals(**additional_globals) + .create() ) e.update_nunavut_globals( - *lctx.get_target_language().get_support_module(), is_dryrun=True, omit_serialization_support=True + *lctx.get_target_language().get_support_module(), omit_serialization_support=True, embed_auditing_info=True ) rendered = str(e.get_template("test").render()) @@ -364,18 +345,57 @@ def _make_filter_test_template( @pytest.fixture -def mock_environment(request): # type: ignore +def mock_environment(request: pytest.FixtureRequest) -> typing.Any: # pylint: disable=unused-argument """ A MagicMock that can be used where a jinja environment is needed. """ - from unittest.mock import MagicMock + from unittest.mock import MagicMock # pylint: disable=import-outside-toplevel - mock_environment = MagicMock() + magic_mock_environment = MagicMock() support_mock = MagicMock() - mock_environment.globals = {"nunavut": support_mock} + magic_mock_environment.globals = {"nunavut": support_mock} support_mock.support = {"omit": True} - return mock_environment + return magic_mock_environment + + +# +-------------------------------------------------------------------------------------------------------------------+ +# | PYTEST HOOKS +# +-------------------------------------------------------------------------------------------------------------------+ + + +def pytest_configure(config: typing.Any) -> None: # pylint: disable=unused-argument + """ + See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks + """ + # pydsdl._dsdl_definition is reeeeeeealy verbose at the INFO level and below. Turn this down to reduce + # scroll-blindness. + logging.getLogger("pydsdl._dsdl_definition").setLevel(logging.WARNING) + # A lot of DEBUG noise in the other loggers so we'll tune this down to INFO and higher. + logging.getLogger("pydsdl._namespace").setLevel(logging.INFO) + logging.getLogger("pydsdl._data_type_builder").setLevel(logging.INFO) + + +def pytest_addoption(parser: pytest.Parser) -> None: + """ + See https://docs.pytest.org/en/6.2.x/reference.html#initialization-hooks + """ + parser.addoption( + "--keep-generated", + action="store_true", + help=textwrap.dedent( + """ + If set then the temporary directory used to generate files for each test will be left after + the test has completed. Normally this directory is temporary and therefore cleaned up automatically. + + :: WARNING :: + This will leave orphaned files on disk. They won't be big but there will be a lot of them. + + :: WARNING :: + Do not run tests in parallel when using this option. + """ + ), + ) # +-------------------------------------------------------------------------------------------------------------------+ @@ -383,7 +403,7 @@ def mock_environment(request): # type: ignore # +-------------------------------------------------------------------------------------------------------------------+ -_sy = Sybil( +pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS), PythonCodeBlockParser(), @@ -404,9 +424,7 @@ def mock_environment(request): # type: ignore fixtures=[ "jinja_filter_tester", "gen_paths", + "gen_paths_for_module", "assert_language_config_value", ], -) - - -pytest_collect_file = _sy.pytest() +).pytest() diff --git a/cyphal-words.txt b/cyphal-words.txt new file mode 100644 index 00000000..83117e7a --- /dev/null +++ b/cyphal-words.txt @@ -0,0 +1,10 @@ +cetl +Cyphal +DSDL +Kirienko +opencyphal +Pavel +pycyphal +pydsdl +roadmap +uavcan diff --git a/docs/templates.rst b/docs/templates.rst index 1f538ae2..b37e9f48 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -237,6 +237,8 @@ Common Filters :noindex: .. autofunction:: nunavut.jinja.DSDLCodeGenerator.filter_bits2bytes_ceil :noindex: +.. autofunction:: nunavut.jinja.DSDLCodeGenerator.filter_text_table + :noindex: Common Tests ------------------------------------------------- @@ -325,6 +327,10 @@ C++ Use Queries ------------------------------------------------- .. autofunction:: nunavut.lang.cpp.uses_std_variant :noindex: +.. autofunction:: nunavut.lang.cpp.uses_cetl + :noindex: +.. autofunction:: nunavut.lang.cpp.uses_pmr + :noindex: Python Filters @@ -342,6 +348,12 @@ Python Filters :noindex: .. autofunction:: nunavut.lang.py.filter_longest_id_length :noindex: +.. autofunction:: nunavut.lang.py.filter_pickle + :noindex: +.. autofunction:: nunavut.lang.py.filter_numpy_scalar_type + :noindex: +.. autofunction:: nunavut.lang.py.filter_newest_minor_version_aliases + :noindex: HTML Filters diff --git a/setup.cfg b/setup.cfg index bd714ac0..206f014c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ long_description = file: README.rst long_description_content_type = text/x-rst license = MIT license_files = LICENSE.rst -keywords = uavcan, dsdl, can, can-bus, codegen, cyphal, opencyphal +keywords = uavcan, dsdl, can, can-bus, ethernet, udp, codegen, cyphal, opencyphal classifiers = Development Status :: 3 - Alpha Environment :: Console @@ -21,6 +21,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: 3 :: Only Topic :: Scientific/Engineering Topic :: Software Development :: Embedded Systems @@ -37,7 +38,7 @@ package_dir= packages=find: package_data={"nunavut": ["py.typed"]} install_requires= - pydsdl ~= 1.18 + pydsdl pyyaml importlib-resources diff --git a/src/nunavut/__init__.py b/src/nunavut/__init__.py index 7c6a5a07..b0a7d511 100644 --- a/src/nunavut/__init__.py +++ b/src/nunavut/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2022 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Code generator built on top of pydsdl. @@ -68,28 +68,29 @@ """ import sys as _sys -from ._generators import AbstractGenerator as AbstractGenerator -from ._generators import generate_types as generate_types -from ._namespace import Namespace as Namespace -from ._namespace import build_namespace_tree as build_namespace_tree -from ._utilities import TEMPLATE_SUFFIX as TEMPLATE_SUFFIX +from ._generators import AbstractGenerator +from ._generators import generate_types +from ._namespace import Namespace +from ._namespace import build_namespace_tree +from ._utilities import TEMPLATE_SUFFIX from ._utilities import YesNoDefault -from ._version import __author__ as __author__ -from ._version import __copyright__ as __copyright__ -from ._version import __email__ as __email__ -from ._version import __license__ as __license__ -from ._version import __version__ as __version__ -from .jinja import CodeGenerator as CodeGenerator -from .jinja import DSDLCodeGenerator as DSDLCodeGenerator -from .jinja import SupportGenerator as SupportGenerator -from .lang import Language as Language -from .lang import LanguageContext as LanguageContext -from .lang import LanguageContextBuilder as LanguageContextBuilder -from .lang import UnsupportedLanguageError as UnsupportedLanguageError -from .lang._config import LanguageConfig as LanguageConfig -from ._exceptions import InternalError as InternalError - -if _sys.version_info[:2] < (3, 5): # pragma: no cover +from ._utilities import DefaultValue +from ._version import __author__ +from ._version import __copyright__ +from ._version import __email__ +from ._version import __license__ +from ._version import __version__ +from .jinja import CodeGenerator +from .jinja import DSDLCodeGenerator +from .jinja import SupportGenerator +from .lang import Language +from .lang import LanguageContext +from .lang import LanguageContextBuilder +from .lang import UnsupportedLanguageError +from .lang._config import LanguageConfig +from ._exceptions import InternalError + +if _sys.version_info[:2] < (3, 7): # pragma: no cover print("A newer version of Python is required", file=_sys.stderr) _sys.exit(1) @@ -102,6 +103,7 @@ "DSDLCodeGenerator", "generate_types", "LanguageConfig", + "DefaultValue", "Language", "LanguageContext", "LanguageContextBuilder", diff --git a/src/nunavut/__main__.py b/src/nunavut/__main__.py index 7adcd9f9..c6d0f6ee 100644 --- a/src/nunavut/__main__.py +++ b/src/nunavut/__main__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Command-line entrypoint. diff --git a/src/nunavut/_dependencies.py b/src/nunavut/_dependencies.py index 45fd400d..995ecc98 100644 --- a/src/nunavut/_dependencies.py +++ b/src/nunavut/_dependencies.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Objects and utilities for handling DSDL dependencies when generating code for a given type. diff --git a/src/nunavut/_exceptions.py b/src/nunavut/_exceptions.py index 5a9e517c..8d9552bd 100644 --- a/src/nunavut/_exceptions.py +++ b/src/nunavut/_exceptions.py @@ -1,7 +1,7 @@ # -# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2023 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Exception types thrown by the core library. diff --git a/src/nunavut/_generators.py b/src/nunavut/_generators.py index 9cd9b512..40d804bd 100644 --- a/src/nunavut/_generators.py +++ b/src/nunavut/_generators.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Module containing types and utilities for building generator objects. @@ -78,7 +78,11 @@ def get_templates(self, omit_serialization_support: bool = False) -> typing.Iter @abc.abstractmethod def generate_all( - self, is_dryrun: bool = False, allow_overwrite: bool = True, omit_serialization_support: bool = False + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> typing.Iterable[pathlib.Path]: """ Generates all output for a given :class:`nunavut.Namespace` and using @@ -91,6 +95,9 @@ def generate_all( output file exists and the generation is not a dry-run. :param bool omit_serialization_support: If True then the generator will emit only types without additional serialization and deserialization support and logic. + :param embed_auditing_info: If True then additional information about the inputs and environment used to + generate source will be embedded in the generated files at the cost of build + reproducibility. :return: 0 for success. Non-zero for errors. :raises: PermissionError if :attr:`allow_overwrite` is False and the file exists. """ @@ -108,7 +115,7 @@ def create_default_generators( :return: Tuple with the first item being the code-generator and the second the support-library generator. """ - from nunavut.jinja import DSDLCodeGenerator, SupportGenerator + from nunavut.jinja import DSDLCodeGenerator, SupportGenerator # pylint: disable=import-outside-toplevel return (DSDLCodeGenerator(namespace, **kwargs), SupportGenerator(namespace, **kwargs)) @@ -127,8 +134,9 @@ def generate_types( allow_overwrite: bool = True, lookup_directories: typing.Optional[typing.Iterable[str]] = None, allow_unregulated_fixed_port_id: bool = False, - language_options: typing.Mapping[str, typing.Any] = {}, + language_options: typing.Optional[typing.Mapping[str, typing.Any]] = None, include_experimental_languages: bool = False, + embed_auditing_info: bool = False, ) -> None: """ Helper method that uses default settings and built-in templates to generate types for a given @@ -153,7 +161,12 @@ def generate_types( language objects. The supported arguments and valid values are different depending on the language specified by the `language_key` parameter. :param bool include_experimental_languages: If true then experimental languages will also be available. + :param embed_auditing_info: If True then additional information about the inputs and environment used to + generate source will be embedded in the generated files at the cost of build + reproducibility. """ + if language_options is None: + language_options = {} language_context = ( LanguageContextBuilder(include_experimental_languages=include_experimental_languages) @@ -172,5 +185,5 @@ def generate_types( namespace = build_namespace_tree(type_map, str(root_namespace_dir), str(out_dir), language_context) generator, support_generator = create_default_generators(namespace) - support_generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support) - generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support) + support_generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support, embed_auditing_info) + generator.generate_all(is_dryrun, allow_overwrite, omit_serialization_support, embed_auditing_info) diff --git a/src/nunavut/_namespace.py b/src/nunavut/_namespace.py index c4ed79f9..80da5413 100644 --- a/src/nunavut/_namespace.py +++ b/src/nunavut/_namespace.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Namespace object and associated utilities. Nunavut namespaces provide an internal representation of dsdl namespaces @@ -315,6 +315,7 @@ def build_namespace_tree( :param nunavut.LanguageContext language_context: The language context to use when building :class:`nunavut.Namespace` objects. :return: The root :class:`nunavut.Namespace`. + :rtype: nunavut.Namespace """ namespace_index = set() # type: typing.Set[str] diff --git a/src/nunavut/_postprocessors.py b/src/nunavut/_postprocessors.py index 9919b0a7..356e12a5 100644 --- a/src/nunavut/_postprocessors.py +++ b/src/nunavut/_postprocessors.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Module containing post processing logic to run on generated files. diff --git a/src/nunavut/_templates.py b/src/nunavut/_templates.py index 8c89eded..5510e03e 100644 --- a/src/nunavut/_templates.py +++ b/src/nunavut/_templates.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Abstractions around template engine internals. diff --git a/src/nunavut/_utilities.py b/src/nunavut/_utilities.py index 7f4e18a8..ecfdff45 100644 --- a/src/nunavut/_utilities.py +++ b/src/nunavut/_utilities.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ A small collection of common utilities. @@ -17,7 +17,7 @@ import enum import logging import pathlib -from typing import Generator, MutableMapping, cast, TypeVar +from typing import Any, Callable, Generator, MutableMapping, Optional, TypeVar, cast, Generic import importlib_resources @@ -113,7 +113,7 @@ class ResourceType(enum.Enum): @enum.unique class ResourceSearchPolicy(enum.Enum): """ - Generic policy type for controlling the behaviour of things that seach for resources. + Generic policy type for controlling the behaviour of things that search for resources. """ FIND_ALL = 0 @@ -122,12 +122,24 @@ class ResourceSearchPolicy(enum.Enum): def iter_package_resources(pkg_name: str, *suffix_filters: str) -> Generator[pathlib.Path, None, None]: """ - >>> from nunavut._utilities import iter_package_resources - >>> rs = [x for x in iter_package_resources("nunavut.lang", ".py") if x.name == "__init__.py"] - >>> len(rs) - 1 - >>> rs[0].name - '__init__.py' + A generator that yields all the resources in a package that match a given suffix filter. + + Example usage: + + .. invisible-code-block: python + + from nunavut._utilities import iter_package_resources + + .. code-block:: python + + for x in iter_package_resources("nunavut.lang", ".py"): + print(x) + + .. invisible-code-block: python + + rs = [x for x in iter_package_resources("nunavut.lang", ".py") if x.name == "__init__.py"] + assert 1 == len(rs) + assert rs[0].name == '__init__.py' """ for resource in importlib_resources.files(pkg_name).iterdir(): @@ -152,17 +164,158 @@ def empty_list_support_files() -> Generator[pathlib.Path, None, None]: yield from () -DeepUpdateType = TypeVar("DeepUpdateType", bound=MutableMapping) +class DefaultValue: + """ + Represents a default value in the language configuration. Use this to differentiate between explicit values and + default values when merging configuration. For example, given the following configuration: + + .. invisible-code-block: python + + from nunavut import DefaultValue + + .. code-block:: python + + collection = { + 'a': DefaultValue(1), + 'b': 2 + } + + overrides = [ + { + 'a': 3, + 'b': DefaultValue(4) + }, + { + 'a': DefaultValue(5), + 'b': 6 + } + ] + + Then the merged configuration should be: + + .. code-block:: python + + merged = { + 'a': 3, + 'b': 6 + } + + .. invisible-code-block: python + + # let's try it + for override in overrides: + collection = deep_update(collection, override) + + assert collection['a'] == merged['a'] + assert collection['b'] == merged['b'] + + Other properties of DefaultValue: + + .. code-block:: python + + assert DefaultValue(1) == 1 + assert DefaultValue(1) != 2 + assert DefaultValue(1) == DefaultValue(1) + assert DefaultValue(1) != DefaultValue(2) + assert eval(repr(DefaultValue(1))) == DefaultValue(1) + assert hash(DefaultValue(1)) == hash(1) + assert bool(DefaultValue(1)) + assert not bool(DefaultValue(None)) + repred = eval(repr(DefaultValue(8))) + assert repred.value == 8 + + """ + + @classmethod + def assign_to_if_not_default(cls, target: MutableMapping[str, Any], key: str, value: Any) -> Any: + """ + Assigns a value to a key in a dictionary unless the key already has a value and the value is not a + `DefaultValue`. The one exception to this is if the value is a `DefaultValue` and the value for the key is + already a `DefaultValue`. In this case the new `DefaultValue` value will be assigned to the key. + + :param target: The dictionary to assign to. + :param key: The key to assign to. + :param value: The value to test and assign. + :return: The value assigned to the key. This is the value of the `value` parameter if it was assigned or the + value of the key in the target dictionary if it was not assigned. + """ + try: + if isinstance(value, DefaultValue) and not isinstance(target[key], DefaultValue): + return target[key] + except KeyError: + pass + target[key] = value + return value + + def __init__(self, value: Any) -> None: + self._value = value + + @property + def value(self) -> Any: + """ + The default value. + """ + return self._value + + def __eq__(self, other: Any) -> bool: + if isinstance(other, DefaultValue): + return bool(self._value == other.value) + return bool(self._value == other) + + def __ne__(self, other: Any) -> bool: + return not self.__eq__(other) + + def __repr__(self) -> str: + return f"DefaultValue({self.value})" + + def __str__(self) -> str: + return f"DefaultValue({self.value})" + + def __hash__(self) -> int: + return hash(self._value) + + def __bool__(self) -> bool: + return bool(self._value) + +def no_default_value(func: Callable[..., Any]) -> Callable[..., Any]: + """ + Decorator to convert a function that may return `DefaultValue`s to a function that returns the value of the any + `DefaultValue`s found. For example: + + .. invisible-code-block: python + + from nunavut._utilities import DefaultValue, no_default_value + + .. code-block:: python -def deep_update(target: DeepUpdateType, source: DeepUpdateType) -> DeepUpdateType: + @no_default_value + def some_function() -> DefaultValue: + return DefaultValue(1) + + assert some_function() == 1 + assert not isinstance(some_function(), DefaultValue) + """ + + def wrapper(*args: Any, **kwargs: Any) -> Any: + result = func(*args, **kwargs) + if isinstance(result, DefaultValue): + return result.value + return result + + return wrapper + + +DeepUpdateT = TypeVar("DeepUpdateT", bound=MutableMapping) + + +def deep_update(target: DeepUpdateT, source: DeepUpdateT) -> DeepUpdateT: """ Helper method to do a recursive update of a map that may contain maps as values. .. invisible-code-block: python from nunavut._utilities import deep_update - import collections.abc .. code-block:: python @@ -186,13 +339,106 @@ def deep_update(target: DeepUpdateType, source: DeepUpdateType) -> DeepUpdateTyp assert "c" in target_map assert target_map["c"] == "see" + Note that this method is `DefaultValue` aware. If a value in the target map is a `DefaultValue` then it will not + overwrite the value in the target map. If the value in the source map is a `DefaultValue` then it will not be + used to update existing values of any type in the target map but will be used to update the target map if the + target map does not have a value for the given key. In such cases the `DefaultValue` will be inserted into the + target map. + + .. code-block:: python + + from nunavut import DefaultValue + target_map = { + "a": { "one": 1, "two": DefaultValue(2) }, + "b": "not a default", + "c": DefaultValue("one default...") + } + update_from = { + "a": { "two": { "i": "this value" }, "three": DefaultValue("that value")}, + "b": DefaultValue("see"), + "c": DefaultValue("...deserves another."), + "d": DefaultValue("This happened.") + } + + target_map = deep_update(target_map, update_from) + + assert target_map["a"]["one"] == 1 + assert target_map["a"]["two"]["i"] == "this value" + assert target_map["a"]["three"] == "that value" + assert target_map["b"] == "not a default" + assert target_map["c"] == "...deserves another." + assert target_map["d"] == "This happened." + """ if isinstance(target, collections.abc.Mapping): for key, value in source.items(): if isinstance(value, collections.abc.Mapping): - target[key] = deep_update(target.get(key, {}), cast(DeepUpdateType, value)) + target[key] = deep_update(target.get(key, {}), cast(DeepUpdateT, value)) else: - target[key] = value + DefaultValue.assign_to_if_not_default(target, key, value) else: target = copy.copy(source) return target + + +PropertyT = TypeVar("PropertyT") + + +class cached_property(Generic[PropertyT]): + """ + Based on `functools.cached_property` (Python Foundation License 2.0, SPDX: PSF-2.0) implementation in Python 3.11, + this is both a backport for older Python versions and a version that omits the problematic lock as documented for + Python 3.12. As such, this version is not thread safe. + + :param func: The function to be wrapped by this decorator. + + .. invisible-code-block: python + + from nunavut._utilities import cached_property + + class Test: + + @classmethod + @cached_property + def cls_test(cls) -> int: + return 1 + + def __init__(self) -> None: + self.calls = 0 + + @cached_property + def test(self) -> int: + self.calls += 1 + return self.calls + + t = Test() + assert t.test == 1 + assert t.test == 1 + assert t.test == 1 + try: + _ = t.cls_test + assert False + except TypeError: + pass + + """ + + _NOT_FOUND = object() + + def __init__(self, func: Callable[..., PropertyT]): + self._func = func + self._attr_name: Optional[str] = None + self.__doc__ = func.__doc__ + + def __set_name__(self, owner: Any, name: str) -> None: + self._attr_name = name + + def __get__(self, instance: Any, owner: Optional[Any] = None) -> PropertyT: + if self._attr_name is None: + raise TypeError("Cannot use cached_property instance without calling __set_name__ on it.") + cache = instance.__dict__ + val = cast(PropertyT, cache.get(self._attr_name, self._NOT_FOUND)) + if val is self._NOT_FOUND: + val = self._func(instance) + cache[self._attr_name] = val + return val diff --git a/src/nunavut/_version.py b/src/nunavut/_version.py index 29d43911..b5c8fde9 100644 --- a/src/nunavut/_version.py +++ b/src/nunavut/_version.py @@ -1,14 +1,14 @@ # -# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2022 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ .. autodata:: __version__ """ -__version__ = "2.3.2.dev0" +__version__ = "2.3.3.dev0" __license__ = "MIT" __author__ = "OpenCyphal" __copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. Copyright (c) 2023 OpenCyphal." diff --git a/src/nunavut/cli/__init__.py b/src/nunavut/cli/__init__.py index 63807a14..ca2c8907 100644 --- a/src/nunavut/cli/__init__.py +++ b/src/nunavut/cli/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Command-line for using nunavut and jinja to generate code @@ -23,6 +23,8 @@ class _LazyVersionAction(argparse._VersionAction): if the --version action is requested. """ + # pylint: disable=protected-access + def __call__( self, parser: argparse.ArgumentParser, @@ -30,6 +32,7 @@ def __call__( values: typing.Any, option_string: typing.Optional[str] = None, ) -> None: + # pylint: disable=import-outside-toplevel from nunavut._version import __version__ parser._print_message(__version__, sys.stdout) @@ -41,9 +44,7 @@ class _NunavutArgumentParser(argparse.ArgumentParser): Specialization of argparse.ArgumentParser to encapsulate inter-argument rules. """ - def parse_known_args( - self, args: typing.Optional[typing.Sequence[str]] = None, namespace: typing.Optional[argparse.Namespace] = None - ) -> typing.Tuple[argparse.Namespace, typing.List[str]]: + def parse_known_args(self, args=None, namespace=None): # type: ignore parsed_args, argv = super().parse_known_args(args, namespace) self._post_process_args(parsed_args) return (parsed_args, argv) @@ -96,7 +97,6 @@ def _make_parser() -> argparse.ArgumentParser: parser.add_argument( "--lookup-dir", "-I", - default=[], action="append", help=textwrap.dedent( """ @@ -280,7 +280,6 @@ def extension_type(raw_arg: str) -> str: parser.add_argument( "--namespace-output-stem", - default=None, help="The name of the file generated when --generate-namespace-types is provided.", ) @@ -355,6 +354,39 @@ def extension_type(raw_arg: str) -> str: ) parser.add_argument( + "--embed-auditing-info", + action="store_true", + help=textwrap.dedent( + """ + + If set, generators are instructed to add additional information in the form of + language-specific comments or meta-data to use when auditing source code generated by + Nunavut. This data may change based on the environment in use which may interfere with + the reproducibility of your builds. For example, paths to input files used to generate + a type may be included with this option where these paths will be different depending + on the server used to run nnvg. + + """ + ).lstrip(), + ) + + # +-----------------------------------------------------------------------+ + # | Post-Processing Options + # +-----------------------------------------------------------------------+ + + ln_pp_group = parser.add_argument_group( + "post-processing options", + description=textwrap.dedent( + """ + + Options that enable various post-generation steps because Pavel Kirienko doesn't + like writing jinja templates. + + """ + ).lstrip(), + ) + + ln_pp_group.add_argument( "--pp-max-emptylines", type=int, help=textwrap.dedent( @@ -371,7 +403,7 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) - parser.add_argument( + ln_pp_group.add_argument( "--pp-trim-trailing-whitespace", action="store_true", help=textwrap.dedent( @@ -388,7 +420,7 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) - parser.add_argument( + ln_pp_group.add_argument( "-pp-rp", "--pp-run-program", help=textwrap.dedent( @@ -408,7 +440,7 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) - parser.add_argument( + ln_pp_group.add_argument( "-pp-rpa", "--pp-run-program-arg", action="append", @@ -422,6 +454,9 @@ def extension_type(raw_arg: str) -> str: ).lstrip(), ) + # +-----------------------------------------------------------------------+ + # | Language Options + # +-----------------------------------------------------------------------+ ln_opt_group = parser.add_argument_group( "language options", description=textwrap.dedent( @@ -520,7 +555,7 @@ def extension_type(raw_arg: str) -> str: help=textwrap.dedent( """ - There is a set of built-in configuration for Nunvut that provides default falues for known + There is a set of built-in configuration for Nunavut that provides default values for known languages as documented `in the template guide `_. This argument lets you specify override configuration yamls. @@ -547,7 +582,7 @@ def extension_type(raw_arg: str) -> str: def _extra_includes_from_env(env_var_name: str) -> typing.List[str]: try: extra_includes_from_env = os.environ[env_var_name].split(os.pathsep) - logging.info("Additional include directories from {}: {}".format(env_var_name, str(extra_includes_from_env))) + logging.info("Additional include directories from %s: %s", env_var_name, str(extra_includes_from_env)) return extra_includes_from_env except KeyError: return [] @@ -575,13 +610,14 @@ def main() -> int: # # Parse DSDL_INCLUDE_PATH # - extra_includes = args.lookup_dir + extra_includes: typing.List[str] = args.lookup_dir if args.lookup_dir is not None else [] extra_includes_from_env = _extra_includes_from_env("DSDL_INCLUDE_PATH") extra_includes += sorted(extra_includes_from_env) + # pylint: disable=import-outside-toplevel from nunavut.cli.runners import ArgparseRunner - runner = ArgparseRunner(args, extra_includes) + runner = ArgparseRunner(args.root_namespace, args, extra_includes) runner.run() return 0 diff --git a/src/nunavut/cli/runners.py b/src/nunavut/cli/runners.py index f6f8ee58..178034e2 100644 --- a/src/nunavut/cli/runners.py +++ b/src/nunavut/cli/runners.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Objects that utilize command-line inputs to run a program using Nunavut. @@ -23,7 +23,7 @@ SetFileMode, TrimTrailingWhitespace, ) -from nunavut._utilities import YesNoDefault +from nunavut._utilities import DefaultValue, YesNoDefault from nunavut.lang import Language, LanguageContext, LanguageContextBuilder @@ -31,12 +31,18 @@ class ArgparseRunner: """ Runner that uses Python argparse arguments to define a run. - :param argparse.Namespace args: The commandline arguments. + :param root_namespace: The root namespace to generate code for. + :param argparse.Namespace args: The command line arguments. :param typing.Optional[typing.Union[str, typing.List[str]]] extra_includes: A list of paths to additional DSDL root folders. """ - def __init__(self, args: argparse.Namespace, extra_includes: typing.Optional[typing.Union[str, typing.List[str]]]): + def __init__( + self, + root_namespace: pathlib.Path, + args: argparse.Namespace, + extra_includes: typing.Optional[typing.Union[str, typing.List[str]]], + ): self._args = args if extra_includes is None: @@ -53,7 +59,7 @@ def __init__(self, args: argparse.Namespace, extra_includes: typing.Optional[typ if self._args.generate_support != "only" and not self._args.list_configuration: type_map = read_dsdl_namespace( - self._args.root_namespace, + root_namespace, self._extra_includes, allow_unregulated_fixed_port_id=self._args.allow_unregulated_fixed_port_id, ) @@ -61,13 +67,12 @@ def __init__(self, args: argparse.Namespace, extra_includes: typing.Optional[typ type_map = [] self._root_namespace = build_namespace_tree( - type_map, self._args.root_namespace, self._args.outdir, self._language_context + type_map, str(root_namespace), self._args.outdir, self._language_context ) # # nunavut : create generators # - generator_args = { "generate_namespace_types": ( YesNoDefault.YES if self._args.generate_namespace_types else YesNoDefault.DEFAULT @@ -112,8 +117,7 @@ def run(self) -> None: def _should_generate_support(self) -> bool: if self._args.generate_support == "as-needed": return self._args.omit_serialization_support is None or not self._args.omit_serialization_support - else: - return bool(self._args.generate_support == "always" or self._args.generate_support == "only") + return bool(self._args.generate_support in ("always", "only")) def _build_ext_program_postprocessor(self, program: str) -> FilePostProcessor: subprocess_args = [program] @@ -140,12 +144,18 @@ def _build_post_processor_list_from_args(self) -> typing.List[PostProcessor]: return post_processors def _create_language_context(self) -> LanguageContext: - language_options = dict() + language_options = {} if self._args.target_endianness is not None: language_options["target_endianness"] = self._args.target_endianness - language_options["omit_float_serialization_support"] = self._args.omit_float_serialization_support - language_options["enable_serialization_asserts"] = self._args.enable_serialization_asserts - language_options["enable_override_variable_array_capacity"] = self._args.enable_override_variable_array_capacity + language_options["omit_float_serialization_support"] = ( + True if self._args.omit_float_serialization_support else DefaultValue(False) + ) + language_options["enable_serialization_asserts"] = ( + True if self._args.enable_serialization_asserts else DefaultValue(False) + ) + language_options["enable_override_variable_array_capacity"] = ( + True if self._args.enable_override_variable_array_capacity else DefaultValue(False) + ) if self._args.language_standard is not None: language_options["std"] = self._args.language_standard @@ -162,9 +172,7 @@ def _create_language_context(self) -> LanguageContext: include_experimental_languages=self._args.experimental_languages ) builder.set_target_language(target_language_name) - builder.load_default_config(self._args.language_standard) - builder.set_additional_config_files(additional_config_files) - builder.validate_langauge_options() + builder.add_config_files(*additional_config_files) builder.set_target_language_extension(self._args.output_extension) builder.set_target_language_configuration_override( Language.WKCV_NAMESPACE_FILE_STEM, self._args.namespace_output_stem @@ -184,10 +192,10 @@ def _stdout_lister( def _list_outputs_only(self) -> None: if self._args.generate_support != "only": - self._stdout_lister(self._generator.generate_all(is_dryrun=True), lambda p: str(p)) + self._stdout_lister(self._generator.generate_all(is_dryrun=True), str) if self._should_generate_support(): - self._stdout_lister(self._support_generator.generate_all(is_dryrun=True), lambda p: str(p)) + self._stdout_lister(self._support_generator.generate_all(is_dryrun=True), str) def _list_inputs_only(self) -> None: if self._args.generate_support != "only": @@ -216,7 +224,7 @@ def _list_inputs_only(self) -> None: def _list_configuration_only(self) -> None: lctx = self._language_context - import yaml + import yaml # pylint: disable=import-outside-toplevel sys.stdout.write("target_language: '") sys.stdout.write(lctx.get_target_language().name) @@ -230,6 +238,7 @@ def _generate(self) -> None: is_dryrun=self._args.dry_run, allow_overwrite=not self._args.no_overwrite, omit_serialization_support=self._args.omit_serialization_support, + embed_auditing_info=self._args.embed_auditing_info, ) if self._args.generate_support != "only": @@ -237,4 +246,5 @@ def _generate(self) -> None: is_dryrun=self._args.dry_run, allow_overwrite=not self._args.no_overwrite, omit_serialization_support=self._args.omit_serialization_support, + embed_auditing_info=self._args.embed_auditing_info, ) diff --git a/src/nunavut/jinja/__init__.py b/src/nunavut/jinja/__init__.py index ce93bb87..e0eba659 100644 --- a/src/nunavut/jinja/__init__.py +++ b/src/nunavut/jinja/__init__.py @@ -1,12 +1,13 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ jinja-based :class:`~nunavut.generators.AbstractGenerator` implementation. """ +import abc import datetime import io import logging @@ -15,15 +16,16 @@ import shutil import typing -import nunavut._generators -import nunavut.lang -import nunavut._postprocessors import pydsdl -from nunavut._utilities import ResourceType, YesNoDefault, ResourceSearchPolicy, TEMPLATE_SUFFIX from yaml import Dumper as YamlDumper from yaml import dump as yaml_dump -from .environment import CodeGenEnvironment +import nunavut._generators +import nunavut._postprocessors +import nunavut.lang +from nunavut._utilities import TEMPLATE_SUFFIX, ResourceSearchPolicy, ResourceType, YesNoDefault + +from .environment import CodeGenEnvironmentBuilder from .jinja2 import Template from .loaders import DEFAULT_TEMPLATE_PATH, DSDLTemplateLoader @@ -93,7 +95,7 @@ def __augment_post_processors_with_ln_limit_empty_lines( """ Subroutine of _handle_post_processors method. """ - from nunavut._postprocessors import LimitEmptyLines + from nunavut._postprocessors import LimitEmptyLines # pylint: disable=import-outside-toplevel if post_processors is None: post_processors = [LimitEmptyLines(limit_empty_lines)] @@ -114,7 +116,7 @@ def __augment_post_processors_with_ln_trim_trailing_whitespace( """ Subroutine of _handle_post_processors method. """ - from nunavut._postprocessors import TrimTrailingWhitespace + from nunavut._postprocessors import TrimTrailingWhitespace # pylint: disable=import-outside-toplevel if post_processors is None: post_processors = [TrimTrailingWhitespace()] @@ -193,22 +195,32 @@ def __init__( self._post_processors = self._handle_post_processors(target_language, post_processors) - self._env = CodeGenEnvironment( - lctx=language_context, - loader=self._dsdl_template_loader, - lstrip_blocks=lstrip_blocks, - trim_blocks=trim_blocks, - additional_filters=additional_filters, - additional_tests=additional_tests, - additional_globals=additional_globals, + env_builder = ( + CodeGenEnvironmentBuilder(self._dsdl_template_loader, language_context) + .set_trim_blocks(trim_blocks) + .set_lstrip_blocks(lstrip_blocks) ) + if additional_filters is not None: + env_builder.add_filters(**additional_filters) + if additional_tests is not None: + env_builder.add_tests(**additional_tests) + if additional_globals is not None: + env_builder.add_globals(**additional_globals) + + self._env = env_builder.create() @property def dsdl_loader(self) -> DSDLTemplateLoader: + """ + The template loader used by this generator. + """ return self._dsdl_template_loader @property def language_context(self) -> nunavut.lang.LanguageContext: + """ + The language context used by this generator. + """ return self._namespace.get_language_context() # +-----------------------------------------------------------------------+ @@ -219,7 +231,7 @@ def _handle_overwrite(self, output_path: pathlib.Path, allow_overwrite: bool) -> if allow_overwrite: output_path.chmod(output_path.stat().st_mode | 0o220) else: - raise PermissionError("{} exists and allow_overwrite is False.".format(output_path)) + raise PermissionError("{output_path} exists and allow_overwrite is False.") # +-----------------------------------------------------------------------+ # | AbstractGenerator @@ -234,6 +246,16 @@ def get_templates(self, omit_serialization_support: bool = False) -> typing.Iter """ return self._dsdl_template_loader.get_templates() + @abc.abstractmethod + def generate_all( + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, + ) -> typing.Iterable[pathlib.Path]: + raise NotImplementedError() + # +-----------------------------------------------------------------------+ # | PRIVATE # +-----------------------------------------------------------------------+ @@ -314,12 +336,12 @@ def _generate_code( elif isinstance(pp, nunavut._postprocessors.FilePostProcessor): file_pps.append(pp) else: - raise ValueError("PostProcessor type {} is unknown.".format(type(pp))) + raise ValueError(f"PostProcessor type {type(pp)} is unknown.") logger.debug("Using post-processors: %r %r", line_pps, file_pps) self._handle_overwrite(output_path, allow_overwrite) output_path.parent.mkdir(parents=True, exist_ok=True) - with open(str(output_path), "w") as output_file: + with open(str(output_path), "w", encoding="utf-8") as output_file: if len(line_pps) > 0: # The logic gets much more complex when doing line post-processing. self._generate_with_line_buffer(output_file, template_gen, line_pps) @@ -395,7 +417,7 @@ def filter_type_to_template(self, value: typing.Any) -> str: """ result = self.dsdl_loader.type_to_template(type(value)) if result is None: - raise RuntimeError("No template found for type {}".format(type(value))) + raise RuntimeError(f"No template found for type {value}") return result.name def filter_type_to_include_path(self, value: typing.Any, resolve: bool = False) -> str: @@ -462,7 +484,7 @@ def filter_alignment_prefix(offset: pydsdl.BitLengthSet) -> str: # and template = '{{ B | alignment_prefix }}' - # then ('str' is stropped to 'str_' before the version is suffixed) + # outputs rendered = 'aligned' .. invisible-code-block: python @@ -479,7 +501,7 @@ def filter_alignment_prefix(offset: pydsdl.BitLengthSet) -> str: # and template = '{{ B | alignment_prefix }}' - # then ('str' is stropped to 'str_' before the version is suffixed) + # outputs rendered = 'unaligned' .. invisible-code-block: python @@ -493,7 +515,7 @@ def filter_alignment_prefix(offset: pydsdl.BitLengthSet) -> str: if isinstance(offset, pydsdl.BitLengthSet): return "aligned" if offset.is_aligned_at_byte() else "unaligned" else: # pragma: no cover - raise TypeError("Expected BitLengthSet, got {}".format(type(offset).__name__)) + raise TypeError(f"Expected BitLengthSet, got {type(offset).__name__}") @staticmethod def filter_bit_length_set(values: typing.Optional[typing.Union[typing.Iterable[int], int]]) -> pydsdl.BitLengthSet: @@ -521,7 +543,27 @@ def filter_remove_blank_lines(text: str) -> str: from nunavut.jinja import DSDLCodeGenerator import pydsdl - assert DSDLCodeGenerator.filter_remove_blank_lines('123\n \n\n456\n\t\n\v\f\n789') == '123\n456\n789' + .. code-block:: python + + # Given + text = '''123 + + 456 + \t + \v\f + 789''' + + # and + template = '{{ text | remove_blank_lines }}' + + # then the black lines will be removed leaving... + rendered = '''123 + 456 + 789''' + + .. invisible-code-block: python + + jinja_filter_tester(DSDLCodeGenerator.filter_remove_blank_lines, template, rendered, 'c', text=text) """ return re.sub(r"\n([ \t\f\v]*\n)+", r"\n", text) @@ -545,12 +587,56 @@ def filter_bits2bytes_ceil(n_bits: int) -> int: raise ValueError("The number of bits cannot be negative") return (int(n_bits) + 7) // 8 + @staticmethod + def filter_text_table( + data: typing.Dict, start_each_line: str, column_sep: str = " : ", line_end: str = "\n" + ) -> str: + """ + Create a text table from a dictionary of data. + + .. invisible-code-block: python + + from nunavut.jinja import DSDLCodeGenerator + import pydsdl + + .. code-block:: python + + # Given + table = { + "banana": "yellow", + "apple": "red", + "grape": "purple" + } + + # and + template = ''' + {{ table | text_table("// ", " | ", "\\n") }}''' + + # then + rendered = ''' + // banana | yellow + // apple | red + // grape | purple''' + + .. invisible-code-block: python + + jinja_filter_tester(DSDLCodeGenerator.filter_text_table, template, rendered, 'c', table=table) + + """ + # Find the longest key to set the width of the first column + key_width = max(len(key) for key in data.keys()) + + output = [] + for key, value in data.items(): + output.append(f"{start_each_line}{key:<{key_width}}{column_sep}{value}".rstrip()) + return line_end.join(output) + # +-----------------------------------------------------------------------+ # | JINJA : tests # +-----------------------------------------------------------------------+ @staticmethod - def is_None(value: typing.Any) -> bool: + def is_None(value: typing.Any) -> bool: # pylint: disable=invalid-name """ Tests if a value is ``None`` @@ -692,11 +778,17 @@ def __init__(self, namespace: nunavut.Namespace, **kwargs: typing.Any): # +-----------------------------------------------------------------------+ def generate_all( - self, is_dryrun: bool = False, allow_overwrite: bool = True, omit_serialization_support: bool = False + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> typing.Iterable[pathlib.Path]: generated = [] # type: typing.List[pathlib.Path] self._env.update_nunavut_globals( - *self.language_context.get_target_language().get_support_module(), is_dryrun, omit_serialization_support + *self.language_context.get_target_language().get_support_module(), + omit_serialization_support, + embed_auditing_info, ) provider = self.namespace.get_all_types if self.generate_namespace_types else self.namespace.get_all_datatypes for parsed_type, output_path in provider(): @@ -820,10 +912,16 @@ def get_templates(self, omit_serialization_support: bool = False) -> typing.Iter return files def generate_all( - self, is_dryrun: bool = False, allow_overwrite: bool = True, omit_serialization_support: bool = False + self, + is_dryrun: bool = False, + allow_overwrite: bool = True, + omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> typing.Iterable[pathlib.Path]: target_language = self.language_context.get_target_language() - self._env.update_nunavut_globals(*target_language.get_support_module(), is_dryrun, omit_serialization_support) + self._env.update_nunavut_globals( + *target_language.get_support_module(), omit_serialization_support, embed_auditing_info + ) target_path = pathlib.Path(self.namespace.get_support_output_folder()) / self._sub_folders line_pps = [] # type: typing.List['nunavut._postprocessors.LinePostProcessor'] @@ -895,8 +993,8 @@ def _copy_header_using_line_pps( target: pathlib.Path, line_pps: typing.List["nunavut._postprocessors.LinePostProcessor"], ) -> None: - with open(str(target), "w") as target_file: - with open(str(resource), "r") as resource_file: + with open(str(target), "w", encoding="utf-8") as target_file: + with open(str(resource), "r", encoding="utf-8") as resource_file: for resource_line in resource_file: if len(resource_line) > 1 and resource_line[-2] == "\r": resource_line_tuple = (resource_line[0:-2], "\r\n") diff --git a/src/nunavut/jinja/environment.py b/src/nunavut/jinja/environment.py index 96ac2c99..9bce2fe4 100644 --- a/src/nunavut/jinja/environment.py +++ b/src/nunavut/jinja/environment.py @@ -1,11 +1,19 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # +# cSpell: words loopcontrols +# +""" +Jinja environment for Nunavut code generation. +""" + import datetime import inspect import logging +import platform +import sys import types import typing @@ -16,7 +24,7 @@ from .jinja2 import BaseLoader, Environment, StrictUndefined, select_autoescape from .jinja2.ext import Extension from .jinja2.ext import do as jinja_do -from .jinja2.ext import loopcontrols as loopcontrols +from .jinja2.ext import loopcontrols from .jinja2.filters import FILTERS as JINJA2_FILTERS logger = logging.getLogger(__name__) @@ -70,8 +78,8 @@ class LanguageTemplateNamespace: """ def __init__(self, **kwargs: typing.Any): - for name in kwargs: - setattr(self, name, kwargs[name]) + for name, value in kwargs.items(): + setattr(self, name, value) def __repr__(self) -> str: type_name = type(self).__name__ @@ -79,12 +87,12 @@ def __repr__(self) -> str: star_args = {} for name, value in self._get_kwargs(): if name.isidentifier(): - arg_strings.append("%s=%r" % (name, value)) + arg_strings.append(f"{name}={repr(value)}") else: star_args[name] = value if star_args: - arg_strings.append("**%s" % repr(star_args)) - return "%s(%s)" % (type_name, ", ".join(arg_strings)) + arg_strings.append(f"**{repr(star_args)}") + return f"{type_name}({','.join(arg_strings)})" def _get_kwargs(self) -> typing.List[typing.Any]: return list(self.__dict__.items()) @@ -98,19 +106,184 @@ def __contains__(self, key: str) -> bool: return key in self.__dict__ def update(self, update_from: typing.Mapping[str, typing.Any]) -> None: + """ + update the namespace with the given values. + """ for key, value in update_from.items(): setattr(self, key, value) def items(self) -> typing.ItemsView[str, typing.Any]: + """ + The items in the namespace. + """ return self.__dict__.items() + def keys(self) -> typing.KeysView[typing.Any]: + """ + The values in the namespace. + """ + return self.__dict__.keys() + def values(self) -> typing.ValuesView[typing.Any]: + """ + The values in the namespace. + """ return self.__dict__.values() # +---------------------------------------------------------------------------+ # | JINJA : CodeGenEnvironment # +---------------------------------------------------------------------------+ +class CodeGenEnvironmentBuilder: + """ + Builder class for creating a CodeGenEnvironment object for code generation. + + :param BaseLoader loader: The loader used to load templates. + :param LanguageContext lctx: The language context used for code generation. + """ + + DEFAULT_JINJA_EXTENSIONS = [jinja_do, loopcontrols, JinjaAssert, UseQuery] + + def __init__(self, loader: BaseLoader, lctx: LanguageContext) -> None: + self._loader = loader + self._lctx = lctx + self._trim_blocks = False + self._lstrip_blocks = False + self._additional_filters: typing.Optional[typing.Dict[str, typing.Callable]] = None + self._additional_tests: typing.Optional[typing.Dict[str, typing.Callable]] = None + self._additional_globals: typing.Optional[typing.Dict[str, typing.Any]] = None + self._extensions = self.DEFAULT_JINJA_EXTENSIONS[:] + self._allow_filter_test_or_use_query_overwrite = False + + @property + def loader(self) -> BaseLoader: + """ + The loader. + + :return: The loader. + :rtype: BaseLoader + """ + return self._loader + + @property + def lctx(self) -> LanguageContext: + """ + The language context. + + :return: The language context. + :rtype: LanguageContext + """ + return self._lctx + + def set_trim_blocks(self, trim_blocks: bool) -> "CodeGenEnvironmentBuilder": + """ + Set the trim blocks flag. + + :param bool trim_blocks: The trim blocks flag. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._trim_blocks = trim_blocks + return self + + def set_lstrip_blocks(self, lstrip_blocks: bool) -> "CodeGenEnvironmentBuilder": + """ + Set the lstrip blocks flag. + + :param bool lstrip_blocks: The lstrip blocks flag. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._lstrip_blocks = lstrip_blocks + return self + + def add_filters(self, **additional_filters: typing.Callable) -> "CodeGenEnvironmentBuilder": + """ + Add filters to the created environment. + + :param typing.Dict[str, typing.Callable] additional_filters: The additional filters. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + if self._additional_filters is None: + self._additional_filters = additional_filters + else: + self._additional_filters.update(additional_filters) + return self + + def add_tests(self, **additional_tests: typing.Callable) -> "CodeGenEnvironmentBuilder": + """ + Add tests to the created environment. + + :param typing.Dict[str, typing.Callable] additional_tests: The additional tests. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + if self._additional_tests is None: + self._additional_tests = additional_tests + else: + self._additional_tests.update(additional_tests) + return self + + def add_globals(self, **additional_globals: typing.Any) -> "CodeGenEnvironmentBuilder": + """ + Add globals so the created environment. + + :param typing.Dict[str, typing.Any] additional_globals: The additional globals. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + if self._additional_globals is None: + self._additional_globals = additional_globals + else: + self._additional_globals.update(additional_globals) + return self + + def set_extensions(self, *extensions: Extension) -> "CodeGenEnvironmentBuilder": + """ + Set the extensions. + + :param typing.List[Extension] extensions: The extensions. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._extensions = list(extensions) + return self + + def set_allow_filter_test_or_use_query_overwrite( + self, allow_filter_test_or_use_query_overwrite: bool + ) -> "CodeGenEnvironmentBuilder": + """ + Allow overwriting of built-in filters, tests, or use queries. + + :param bool allow_filter_test_or_use_query_overwrite: Allow overwrite of built-ins. + :return: The CodeGenEnvironmentBuilder object. + :rtype: CodeGenEnvironmentBuilder + """ + self._allow_filter_test_or_use_query_overwrite = allow_filter_test_or_use_query_overwrite + return self + + def create(self) -> "CodeGenEnvironment": + """ + Create a CodeGenEnvironment object. + + :return: A CodeGenEnvironment object. + :rtype: CodeGenEnvironment + """ + return CodeGenEnvironment( + self.loader, + self.lctx, + trim_blocks=self._trim_blocks, + lstrip_blocks=self._lstrip_blocks, + additional_filters=self._additional_filters, + additional_tests=self._additional_tests, + additional_globals=self._additional_globals, + extensions=self._extensions, + allow_filter_test_or_use_query_overwrite=self._allow_filter_test_or_use_query_overwrite, + ) + + +# +---------------------------------------------------------------------------+ class CodeGenEnvironment(Environment): @@ -118,11 +291,13 @@ class CodeGenEnvironment(Environment): Jinja Environment optimized for compile-time generation of source code (i.e. as opposed to dynamically generating webpages). + Do not insatiate directly. Use the :class:`CodeGenEnvironmentBuilder` to create an instance. + .. invisible-code-block: python from nunavut.lang import LanguageContext, LanguageContextBuilder from nunavut.lang._language import Language - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader lctx = LanguageContextBuilder().create() @@ -131,7 +306,7 @@ class CodeGenEnvironment(Environment): template = 'Hello World' - e = CodeGenEnvironment(loader=DictLoader({'test': template}), lctx=lctx) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx).create() assert 'Hello World' == e.get_template('test').render() .. warning:: @@ -142,7 +317,11 @@ class CodeGenEnvironment(Environment): .. code-block:: python try: - CodeGenEnvironment(loader=DictLoader({'test': template}), lctx=lctx, additional_globals={'ln': 'bad_ln'}) + ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_globals(ln='bad_ln') + .create() + ) assert False except RuntimeError: pass @@ -152,19 +331,23 @@ class CodeGenEnvironment(Environment): .. code-block:: python try: - CodeGenEnvironment(loader=DictLoader({'test': template}), - lctx=lctx, - additional_filters={'indent': lambda x: x}) + ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_filters(indent=lambda x: x) + .create() + ) assert False except RuntimeError: pass # You can allow overwrite of built-ins using the ``allow_filter_test_or_use_query_overwrite`` # argument. - e = CodeGenEnvironment(loader=DictLoader({'test': template}), - lctx=lctx, - additional_filters={'indent': lambda x: x}, - allow_filter_test_or_use_query_overwrite=True) + e = ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_filters(indent=lambda x: x) + .set_allow_filter_test_or_use_query_overwrite(True) + .create() + ) assert 'foo' == e.filters['indent']('foo') ...or that user-defined filters or redefined. @@ -177,9 +360,11 @@ class MyFilters: def filter_misnamed(name: str) -> str: return name - e = CodeGenEnvironment(loader=DictLoader({'test': template}), - lctx=lctx, - additional_filters={'filter_misnamed': lambda x: x}) + e = ( + CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) + .add_filters(filter_misnamed=lambda x: x) + .create() + ) try: e.add_conventional_methods_to_environment(MyFilters()) @@ -202,14 +387,14 @@ def __init__( self, loader: BaseLoader, lctx: LanguageContext, - trim_blocks: bool = False, - lstrip_blocks: bool = False, - additional_filters: typing.Optional[typing.Dict[str, typing.Callable]] = None, - additional_tests: typing.Optional[typing.Dict[str, typing.Callable]] = None, - additional_globals: typing.Optional[typing.Dict[str, typing.Any]] = None, - extensions: typing.List[Extension] = [jinja_do, loopcontrols, JinjaAssert, UseQuery], - allow_filter_test_or_use_query_overwrite: bool = False, - ): + trim_blocks: bool, + lstrip_blocks: bool, + additional_filters: typing.Optional[typing.Dict[str, typing.Callable]], + additional_tests: typing.Optional[typing.Dict[str, typing.Callable]], + additional_globals: typing.Optional[typing.Dict[str, typing.Any]], + extensions: typing.Optional[typing.List[Extension]], + allow_filter_test_or_use_query_overwrite: bool, + ): # pylint: disable=too-many-arguments super().__init__( loader=loader, # nosec extensions=extensions, @@ -226,7 +411,7 @@ def __init__( if additional_globals is not None: for global_name, global_value in additional_globals.items(): if global_name in self.RESERVED_GLOBAL_NAMESPACES or global_name in self.RESERVED_GLOBAL_NAMES: - raise RuntimeError('Additional global "{}" uses a reserved global name'.format(global_name)) + raise RuntimeError(f'Additional global "{global_name}" uses a reserved global name') self.globals[global_name] = global_value self._allow_replacements = allow_filter_test_or_use_query_overwrite @@ -260,6 +445,16 @@ def __init__( self._add_each_to_environment(additional_tests.items(), self.tests, supported_languages=supported_languages) def add_conventional_methods_to_environment(self, obj: typing.Any) -> None: + """ + Adds methods using specific naming conventions to the Jinja environment. For example, methods named `filter_*` + are added to the Jinja environment as filters. + + This method iterates over the methods of the given object and adds them to the Jinja environment. + Only methods that are supported by the specified languages are added. + + :param typing.Any obj: The object to add the methods from. + + """ for name, method in inspect.getmembers(obj, inspect.isroutine): try: self._add_conventional_method_to_environment(method, name, supported_languages=self.supported_languages) @@ -270,11 +465,24 @@ def update_nunavut_globals( self, support_namespace: str = "", support_version: typing.Tuple[int, int, int] = (0, 0, 0), - support_module: typing.Optional["types.ModuleType"] = None, - is_dryrun: bool = False, + support_module: typing.Optional["types.ModuleType"] = None, # pylint: disable=unused-argument omit_serialization_support: bool = False, + embed_auditing_info: bool = False, ) -> None: + """ + Update the global properties available to templates as `nunavut`. + :param support_namespace: The name of a generated namespace for support code. Available as + `nunavut.support.namespace` in templates. + :param support_version: The version to report for supporting code. Available as + `nunavut.support.version` in templates. + :param support_module: The python module containing support for the selected language. + :param omit_serialization_support: Boolean flag set on the support object. Available as + `nunavut.support.omit_serialization_support` in templates. + :param embed_auditing_info: Boolean flag available as `nunavut.embed_auditing_info` in templates. + """ nunavut_namespace = self.nunavut_global + setattr(nunavut_namespace, "embed_auditing_info", embed_auditing_info) + setattr(nunavut_namespace, "platform_version", self._create_platform_version(embed_auditing_info)) setattr( nunavut_namespace, @@ -282,43 +490,88 @@ def update_nunavut_globals( {"omit": omit_serialization_support, "namespace": support_namespace, "version": support_version}, ) - if "version" not in nunavut_namespace: - from nunavut import __version__ as nunavut_version + if "template_sets" not in nunavut_namespace: + # pylint: disable=import-outside-toplevel from nunavut.jinja.loaders import DSDLTemplateLoader - setattr(nunavut_namespace, "version", nunavut_version) - setattr(nunavut_namespace, "platform_version", self._create_platform_version()) - if isinstance(self.loader, DSDLTemplateLoader): setattr(nunavut_namespace, "template_sets", self.loader.get_template_sets()) + if "version" not in nunavut_namespace: + # pylint: disable=import-outside-toplevel + from nunavut import __version__ as nunavut_version + + setattr(nunavut_namespace, "version", nunavut_version) + @property def supported_languages(self) -> typing.ValuesView[Language]: + """ + The supported languages in the environment. + + :return: A view of the supported languages. + :rtype: typing.ValuesView[Language] + """ ln_globals = self.globals["ln"] # type: LanguageTemplateNamespace return ln_globals.values() @property def nunavut_global(self) -> LanguageTemplateNamespace: + """ + The `nunavut` global namespace. + + :return: The `nunavut` global namespace. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["nunavut"]) @property def target_language_uses_queries(self) -> LanguageTemplateNamespace: + """ + All `uses_queries` for the target language. + + :return: The uses queries for the target language. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["uses_queries"]) @property def language_options(self) -> LanguageTemplateNamespace: + """ + The language options. + + :return: The language options. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["options"]) @property def language_support(self) -> LanguageTemplateNamespace: + """ + The language support. + + :return: The language support. + :rtype: LanguageTemplateNamespace + """ return typing.cast(LanguageTemplateNamespace, self.globals["ln"]) @property def target_language(self) -> Language: + """ + The target language. + + :return: The target language. + :rtype: Language + """ return self._target_language @property def now_utc(self) -> datetime.datetime: + """ + Get or set the current UTC time. + + :return: The current UTC time. + :rtype: datetime.datetime + """ return typing.cast(datetime.datetime, self.globals["now_utc"]) @now_utc.setter @@ -326,6 +579,13 @@ def now_utc(self, utc_time: datetime.datetime) -> None: self.globals["now_utc"] = utc_time def add_test(self, test_name: str, test_callable: typing.Callable) -> None: + """ + Add a test to the environment. + + :param str test_name: The name of the test. + :param typing.Callable test_callable: The test. + :return: None + """ self._add_to_environment(test_name, test_callable, self.tests) # +----------------------------------------------------------------------------------------------------------------+ @@ -337,22 +597,23 @@ def _resolve_collection( method_name: str, collection_maybe: typing.Optional[typing.Union[LanguageTemplateNamespace, typing.Dict[str, typing.Any]]], ) -> typing.Union[LanguageTemplateNamespace, typing.Dict[str, typing.Any]]: + """ + Resolve the collection to add the item to. If collection_maybe is not None then it is returned otherwise the + collection is resolved based on the method name. + """ if collection_maybe is not None: return collection_maybe if LanguageEnvironment.is_test_name(conventional_method_prefix): return typing.cast(typing.Dict[str, typing.Any], self.tests) - elif LanguageEnvironment.is_filter_name(conventional_method_prefix): + if LanguageEnvironment.is_filter_name(conventional_method_prefix): return typing.cast(typing.Dict[str, typing.Any], self.filters) - elif LanguageEnvironment.is_uses_query_name(conventional_method_prefix): + if LanguageEnvironment.is_uses_query_name(conventional_method_prefix): uses_queries = self.globals["uses_queries"] return typing.cast(LanguageTemplateNamespace, uses_queries) - else: - raise TypeError( - "Tried to add an item {} to the template environment but we don't know what the item is.".format( - method_name - ) - ) + raise TypeError( + f"Tried to add an item {method_name} to the template environment but we don't know what the item is." + ) def _add_to_environment( self, @@ -362,13 +623,13 @@ def _add_to_environment( ) -> None: if item_name in collection: if not self._allow_replacements: - raise RuntimeError("{} was already defined.".format(item_name)) - elif item_name in JINJA2_FILTERS: - logger.info("Replacing Jinja built-in {}".format(item_name)) + raise RuntimeError(f"{item_name} was already defined.") + if item_name in JINJA2_FILTERS: + logger.info("Replacing Jinja built-in %s", item_name) else: - logger.info('Replacing "{}" which was already defined for this environment.'.format(item_name)) + logger.info('Replacing "%s" which was already defined for this environment.', item_name) else: - logger.debug("Adding {} to environment".format(item_name)) + logger.debug("Adding %s to environment", item_name) if isinstance(collection, LanguageTemplateNamespace): setattr(collection, item_name, item) else: @@ -384,16 +645,20 @@ def _add_conventional_method_to_environment( is_target: bool = False, ) -> None: """ + Add a method using specific naming conventions to the Jinja environment. For example, methods named `filter_*` + are added to the Jinja environment as filters. - :param str callable_name: The name of the callable to use in a template. - :param typing.Callable[..., bool] callable: The named callable. - :param typing.Optional[str] callable_namespace: If provided the namespace to prefix to the callable name. - :return: tuple of name and the callable which might be prepared as a partial function based on decorators. - :raises: RuntimeWarning if the callable requested resources that were not available in this environment. + :param typing.Callable[..., bool] method: The named method. + :param str method_name: The name of the callable to use in a template. + :param typing.Optional[typing.Union[LanguageTemplateNamespace, typing.Dict[str, typing.Any]]] collection_maybe: + The collection to add the method to. If None then the collection is resolved based on the method name. + :param typing.Optional[typing.ValuesView[Language]] supported_languages: The supported languages. + :param typing.Optional[Language] method_language: The language of the method. + :param bool is_target: Whether the method is for the target language. .. invisible-code-block: python - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader from nunavut._templates import template_language_test from unittest.mock import MagicMock @@ -409,10 +674,10 @@ def _add_conventional_method_to_environment( def test_test(language): return True - e = CodeGenEnvironment( - loader=DictLoader({'test': 'hello world'}), - additional_tests={'foo': test_test}, - lctx=lctx + e = ( + CodeGenEnvironmentBuilder(DictLoader({'test': 'hello world'}), lctx) + .add_tests(foo=test_test) + .create() ) assert test_test == e.tests['foo'].func assert e.tests['foo']() @@ -422,7 +687,7 @@ def test_test(language): collection = self._resolve_collection(result[0], method_name, collection_maybe) if method_language is not None: - self._add_to_environment("ln.{}.{}".format(method_language.name, result[1]), result[2], collection) + self._add_to_environment(f"ln.{method_language.name}.{result[1]}", result[2], collection) else: self._add_to_environment(result[1], result[2], collection) if is_target: @@ -447,25 +712,25 @@ def _add_each_to_environment( ) @classmethod - def _create_platform_version(cls) -> typing.Dict[str, typing.Any]: - import platform - import sys + def _create_platform_version(cls, embed_auditing_info: bool) -> typing.Dict[str, typing.Any]: platform_version = {} # type: typing.Dict[str, typing.Any] - platform_version["python_implementation"] = platform.python_implementation() platform_version["python_version"] = platform.python_version() - platform_version["python_release_level"] = sys.version_info[3] - platform_version["python_build"] = platform.python_build() - platform_version["python_compiler"] = platform.python_compiler() - platform_version["python_revision"] = platform.python_revision() + if embed_auditing_info: + platform_version["python_implementation"] = platform.python_implementation() + platform_version["python_release_level"] = sys.version_info[3] + platform_version["python_build"] = platform.python_build() + platform_version["python_compiler"] = platform.python_compiler() + platform_version["python_revision"] = platform.python_revision() - try: - platform_version["python_xoptions"] = sys._xoptions - except AttributeError: # pragma: no cover - platform_version["python_xoptions"] = {} + try: + # pylint: disable=protected-access + platform_version["python_xoptions"] = sys._xoptions + except AttributeError: # pragma: no cover + platform_version["python_xoptions"] = {} - platform_version["runtime_platform"] = platform.platform() + platform_version["runtime_platform"] = platform.platform() return platform_version diff --git a/src/nunavut/jinja/extensions.py b/src/nunavut/jinja/extensions.py index 5d208168..38acf439 100644 --- a/src/nunavut/jinja/extensions.py +++ b/src/nunavut/jinja/extensions.py @@ -1,12 +1,14 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # - +""" +Jinja2 extensions for use with the Nunavut code generator. +""" import typing -from nunavut.jinja.jinja2 import Environment, TemplateAssertionError, UndefinedError, nodes +from nunavut.jinja.jinja2 import TemplateAssertionError, UndefinedError, nodes from nunavut.jinja.jinja2.ext import Extension from nunavut.jinja.jinja2.parser import Parser @@ -23,14 +25,14 @@ class JinjaAssert(Extension): .. invisible-code-block: python from nunavut.jinja.jinja2.exceptions import TemplateAssertionError - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader from nunavut.jinja.extensions import JinjaAssert from nunavut.lang import LanguageContextBuilder - e = CodeGenEnvironment(lctx=LanguageContextBuilder().create(), - loader=DictLoader({'test': template}), - extensions=[JinjaAssert]) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), LanguageContextBuilder().create()) \ + .set_extensions(JinjaAssert) \ + .create() try: e.get_template('test').render() # huh. This should have raised a TemplateAssertionError @@ -38,7 +40,7 @@ class JinjaAssert(Extension): except TemplateAssertionError: pass - This extension also support provding an assertion message: + This extension also support providing an assertion message: .. code-block:: python @@ -46,9 +48,9 @@ class JinjaAssert(Extension): .. invisible-code-block: python - e = CodeGenEnvironment(lctx=LanguageContextBuilder().create(), - loader=DictLoader({'test': template}), - extensions=[JinjaAssert]) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), LanguageContextBuilder().create())\ + .set_extensions(JinjaAssert)\ + .create() try: e.get_template('test').render() # huh. This should have raised a TemplateAssertionError @@ -60,9 +62,6 @@ class JinjaAssert(Extension): tags = set(["assert"]) - def __init__(self, environment: Environment): - super().__init__(environment) - def parse(self, parser: Parser) -> nodes.Node: """ See http://jinja.pocoo.org/docs/2.10/extensions/ for help writing @@ -115,7 +114,7 @@ class UseQuery(Extension): .. invisible-code-block: python from nunavut.jinja.jinja2.exceptions import TemplateAssertionError - from nunavut.jinja import CodeGenEnvironment + from nunavut.jinja import CodeGenEnvironmentBuilder from nunavut.jinja.jinja2 import DictLoader from nunavut.jinja.extensions import UseQuery from nunavut.lang import LanguageClassLoader @@ -129,9 +128,9 @@ class UseQuery(Extension): lctx.get_target_language = MagicMock(return_value = ln_c) - e = CodeGenEnvironment(lctx=lctx, - loader=DictLoader({'test': template}), - extensions=[UseQuery]) + e = CodeGenEnvironmentBuilder(DictLoader({'test': template}), lctx) \ + .set_extensions(UseQuery)\ + .create() try: result = e.get_template('test').render() @@ -158,9 +157,6 @@ class UseQuery(Extension): tags = set(["ifuses", "ifnuses"]) - def __init__(self, environment: Environment): - super().__init__(environment) - def parse(self, parser: Parser) -> nodes.Node: """ See http://jinja.pocoo.org/docs/2.10/extensions/ for help writing @@ -195,12 +191,12 @@ def parse(self, parser: Parser) -> nodes.Node: node = nodes.If(lineno=parser.stream.current.lineno) result.elif_.append(node) continue - elif token.test("name:elifnuses"): + if token.test("name:elifnuses"): negate = True node = nodes.If(lineno=parser.stream.current.lineno) result.elif_.append(node) continue - elif token.test("name:else"): + if token.test("name:else"): result.else_ = parser.parse_statements( ( "name:endifuses", @@ -221,11 +217,11 @@ def _use_query_common(self, uses_query_name: str, lineno: int, name: str, filena uses_query = typing.cast( typing.Callable[..., bool], getattr(self.environment.target_language_uses_queries, uses_query_name) ) - except AttributeError: + except AttributeError as e: raise UndefinedError( - 'use query "{}" for language "{}" is not defined ' - "(line={}, name={}, filename={})".format(uses_query_name, target_language.name, lineno, name, filename) - ) + f'use query "{uses_query_name}" for language "{target_language.name}" is not defined ' + "(line={lineno}, name={name}, filename={filename})" + ) from e return uses_query() diff --git a/src/nunavut/jinja/loaders.py b/src/nunavut/jinja/loaders.py index a6736e0a..2d82d617 100644 --- a/src/nunavut/jinja/loaders.py +++ b/src/nunavut/jinja/loaders.py @@ -1,8 +1,12 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # +""" +Contains template loaders for Nunavut's Jinja2 environment. +""" + import collections import importlib import logging @@ -55,7 +59,7 @@ def __init__( package_name_for_templates: typing.Optional[str] = None, builtin_template_path: str = DEFAULT_TEMPLATE_PATH, search_policy: ResourceSearchPolicy = ResourceSearchPolicy.FIND_ALL, - **kwargs: typing.Any + **kwargs: typing.Any, ): super().__init__(**kwargs) self._type_to_template_lookup_cache: typing.Dict[pydsdl.Any, pathlib.Path] = dict() @@ -63,8 +67,8 @@ def __init__( if templates_dirs is not None: for templates_dir_item in templates_dirs: if not templates_dir_item.exists(): - raise ValueError("Templates directory {} did not exist?".format(str(templates_dir_item))) - logger.info("Loading templates from file system at {}".format(templates_dirs)) + raise ValueError(f"Templates directory {str(templates_dir_item)} did not exist?") + logger.info("Loading templates from file system at %s", templates_dirs) self._fsloader = FileSystemLoader((str(d) for d in templates_dirs), followlinks=followlinks) else: self._fsloader = None @@ -221,7 +225,7 @@ def type_to_template(self, value_type: typing.Type) -> typing.Optional[pathlib.P # +----------------------------------------------------------------------------------------------------------------+ @staticmethod def _filter_template_list_by_suffix(files: typing.List[str]) -> typing.List[str]: - return [f for f in files if (pathlib.Path(f).suffix == TEMPLATE_SUFFIX)] + return [f for f in files if pathlib.Path(f).suffix == TEMPLATE_SUFFIX] def _type_to_template_internal( self, value_type: typing.Type, templates: typing.Mapping[str, pathlib.Path] @@ -241,9 +245,9 @@ def _type_to_template_internal( try: logging.debug( - "NunavutTemplateLoader.type_to_template for {}: considering {}...".format( - value_type.__name__, current_search_type.__name__ - ) + "NunavutTemplateLoader.type_to_template for %s: considering %s...", + value_type.__name__, + current_search_type.__name__, ) template_path = templates[current_search_type.__name__] self._type_to_template_lookup_cache[current_search_type] = template_path diff --git a/src/nunavut/lang/__init__.py b/src/nunavut/lang/__init__.py index c4d51c98..22a7946e 100644 --- a/src/nunavut/lang/__init__.py +++ b/src/nunavut/lang/__init__.py @@ -1,9 +1,10 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # -"""Language-specific support in nunavut. +""" +Language-specific support in nunavut. This package contains modules that provide specific support for generating source for various languages using templates. @@ -13,9 +14,8 @@ import pathlib import typing -from ._config import LanguageConfig as LanguageConfig -from ._language import Language as Language -from ._language import LanguageClassLoader as LanguageClassLoader +from ._config import LanguageConfig +from ._language import Language, LanguageClassLoader logger = logging.getLogger(__name__) @@ -25,12 +25,10 @@ class UnsupportedLanguageError(ValueError): Error type raised if an unsupported language type is used. """ - pass - class LanguageContextBuilder: """ - Used to instatiate new :class:`LanguageContext` objects. + Used to instantiate new :class:`LanguageContext` objects. The simplest invocation will always work by using the :data:`LanguageContextBuilder.DEFAULT_TARGET_LANGUAGE` constant: @@ -83,6 +81,9 @@ def get_supported_language_names(self) -> typing.Iterable[str]: @property def config(self) -> LanguageConfig: + """ + The configuration object that will be used to create the language context. + """ return self._ln_loader.config # +-----------------------------------------------------------------------+ @@ -92,7 +93,7 @@ def config(self) -> LanguageConfig: def set_target_language_configuration_override(self, key: str, value: typing.Any) -> "LanguageContextBuilder": """ Stores a key and value to override in the configuration for a language target when a LanguageContext is crated. - These overrides are always set under the language section of the target langauge. + These overrides are always set under the language section of the target language. .. invisible-code-block: python @@ -114,7 +115,7 @@ def set_target_language_configuration_override(self, key: str, value: typing.Any builder.set_target_language_configuration_override(Language.WKCV_DEFINITION_FILE_EXTENSION, ".foo") - ...but that value will not be overriden until you create the target language: + ...but that value will not be overridden until you create the target language: .. code-block:: python @@ -132,7 +133,7 @@ def set_target_language_configuration_override(self, key: str, value: typing.Any assert overridden_c_file_extension == ".foo" - Note that the config is scoped by the builder but is then inherited by the langauge objects created by the + Note that the config is scoped by the builder but is then inherited by the language objects created by the builder: .. code-block:: python @@ -220,15 +221,16 @@ def set_target_language(self, target_language: typing.Optional[str]) -> "Languag self._target_language_name = LanguageClassLoader.to_language_name(target_language) return self - def load_default_config(self, language_standard: str) -> None: - self._ln_loader.config.apply_defaults(language_standard) - - def validate_langauge_options(self) -> None: - self._ln_loader.config.validate_language_options() - def set_additional_config_files( self, additional_config_files: typing.List[pathlib.Path] ) -> "LanguageContextBuilder": + """ + Deprecated. Use :func:`add_config_files` instead. + """ + logger.warning("set_additional_config_files is deprecated. Use add_config_files instead.") + return self.add_config_files(*additional_config_files) + + def add_config_files(self, *additional_config_files: pathlib.Path) -> "LanguageContextBuilder": """ A list of paths to additional yaml files to load as configuration. These will override any values found in the :file:`nunavut.lang.properties.yaml` file and files @@ -241,13 +243,13 @@ def set_additional_config_files( import textwrap from nunavut.lang import LanguageContextBuilder, Language, LanguageClassLoader - overrides_file = gen_paths.out_dir / pathlib.Path("overrides1.yaml") + overrides_file = gen_paths_for_module.out_dir / pathlib.Path("overrides1.yaml") overrides_data = {LanguageClassLoader.to_language_module_name("c"): {Language.WKCV_DEFINITION_FILE_EXTENSION: ".foo"} } - with open(overrides_file, "w") as overrides_handle: + with open(overrides_file, "w", encoding="utf-8") as overrides_handle: yaml.dump(overrides_data, overrides_handle) .. code-block:: python @@ -255,7 +257,7 @@ def set_additional_config_files( target_language_w_overrides = ( LanguageContextBuilder() .set_target_language("c") - .set_additional_config_files([overrides_file]) + .add_config_files(overrides_file) .create() .get_target_language() ) @@ -270,7 +272,7 @@ def set_additional_config_files( assert target_language_w_overrides.extension == ".foo" assert target_language_no_overrides.extension == ".h" - Overrides are applies as unions. For example, given this override data: + Overrides are applied as unions. For example, given this override data: .. code-block:: python @@ -284,8 +286,8 @@ def set_additional_config_files( .. invisible-code-block: python - second_overrides_file = gen_paths.out_dir / pathlib.Path("overrides2.yaml") - with open(second_overrides_file, "w") as overrides_handle: + second_overrides_file = gen_paths_for_module.out_dir / pathlib.Path("overrides2.yaml") + with open(second_overrides_file, "w", encoding="utf-8") as overrides_handle: overrides_handle.write(textwrap.dedent(overrides_data)) .. code-block:: python @@ -293,7 +295,7 @@ def set_additional_config_files( target_language_w_overrides = ( LanguageContextBuilder() .set_target_language("c") - .set_additional_config_files([second_overrides_file]) + .add_config_files(second_overrides_file) .create() .get_target_language() ) @@ -301,16 +303,69 @@ def set_additional_config_files( assert ".foo" == target_language_w_overrides.extension assert "bar" == target_language_w_overrides.get_config_value("non-standard") + .. invisible-code-block: python + + from nunavut import DefaultValue + + # verification of issue #329 fix + with_default = {"enable_serialization_asserts" : DefaultValue(False) } + without_default = {"enable_serialization_asserts" : False } + + # verification of issue #329 fix + overrides_data = ''' + nunavut.lang.c: + options: + enable_serialization_asserts: true + ''' + + issue_329_overrides = gen_paths_for_module.out_dir / pathlib.Path("overrides329.yaml") + with open(issue_329_overrides, "w", encoding="utf-8") as overrides_handle: + overrides_handle.write(textwrap.dedent(overrides_data)) + + target_language_329_no_file_overrides = ( + LanguageContextBuilder() + .set_target_language("c") + .set_target_language_configuration_override(Language.WKCV_LANGUAGE_OPTIONS, with_default) + .create() + .get_target_language() + ) + + target_language_329_file_override = ( + LanguageContextBuilder() + .set_target_language("c") + .add_config_files(issue_329_overrides) + .set_target_language_configuration_override(Language.WKCV_LANGUAGE_OPTIONS, with_default) + .create() + .get_target_language() + ) + + target_language_329_file_override_overridden = ( + LanguageContextBuilder() + .set_target_language("c") + .add_config_files(issue_329_overrides) + .set_target_language_configuration_override(Language.WKCV_LANGUAGE_OPTIONS, without_default) + .create() + .get_target_language() + ) + + # default from command line + assert not target_language_329_no_file_overrides.get_option("enable_serialization_asserts") + + # default from command line overridden by file + assert target_language_329_file_override.get_option("enable_serialization_asserts") + + # command-line overrides file + assert not target_language_329_file_override_overridden.get_option("enable_serialization_asserts") """ for additional_path in additional_config_files: - with open(str(additional_path), "r") as additional_file: - self._ln_loader.config.update_from_file(additional_file) + with open(str(additional_path), "r", encoding="utf-8") as additional_file: + self.config.update_from_yaml_file(additional_file) return self def create(self) -> "LanguageContext": """ - Applies all pending configuration overrides to the internal :class:`LanguageConfig` object and instatiates + Applies all pending configuration overrides to the internal :class:`LanguageConfig` object and instantiates a :class:`LanguageContext` object. """ # First find the target language to use... @@ -337,13 +392,11 @@ def _new_language_w_experimental_handling(self, language_name: str) -> Language: try: language = self._ln_loader.new_language(language_name) except ImportError as e: - logger.debug("Import Error {} when trying to load language {}".format(str(e), language_name)) - raise KeyError("language {} is not a supported language".format(language_name)) + logger.debug("Import Error %s when trying to load language %s", e, language_name) + raise KeyError(f"language {language_name} is not a supported language") from e if not (language.stable_support or self._include_experimental_languages): raise UnsupportedLanguageError( - "{} support is only experimental, but experimental language support is not enabled".format( - language_name - ) + f"{language_name} support is only experimental, but experimental language support is not enabled" ) return language @@ -377,9 +430,8 @@ def _resolve_target_language(self, explicit_value: typing.Optional[str]) -> str: if inferred_target_language_name is None: inferred_target_language_name = self.DEFAULT_TARGET_LANGUAGE logger.info( - "No target language specified and none could be inferred. Using default language, {}".format( - self.DEFAULT_TARGET_LANGUAGE - ) + "No target language specified and none could be inferred. Using default language, %s", + self.DEFAULT_TARGET_LANGUAGE, ) else: logging.info( @@ -472,4 +524,8 @@ def get_supported_languages(self) -> typing.Dict[str, Language]: @property def config(self) -> LanguageConfig: + """ + Returns the :class:`nunavut.lang.LanguageConfig` object that contains the configuration for all + supported languages. This is the same object that is used to instantiate the :class:`nunavut.lang.Language` + """ return self._config diff --git a/src/nunavut/lang/_common.py b/src/nunavut/lang/_common.py index cc0bb289..55733854 100644 --- a/src/nunavut/lang/_common.py +++ b/src/nunavut/lang/_common.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """Language-specific support in nunavut. @@ -19,13 +19,28 @@ from ._language import Language +# +-------------------------------------------------------------------------------------------------------------------+ +# | GENERATORS +# +-------------------------------------------------------------------------------------------------------------------+ + + class IncludeGenerator: + """ + Generates include file paths for a given language and datatype. + """ + def __init__(self, language: Language, t: pydsdl.CompositeType, omit_serialization_support: bool): self._type = t self._language = language self._omit_serialization_support = omit_serialization_support def generate_include_filepart_list(self, output_extension: str, sort: bool) -> typing.List[str]: + """ + Generates a list of include file paths for a given datatype and language. + :param output_extension: The file extension to use for the include file paths. + :param sort: If True the list of include file paths will be sorted. + :return: A list of include file paths. + """ dep_types = self._language.get_dependency_builder(self._type).direct() path_list = [ @@ -43,14 +58,14 @@ def generate_include_filepart_list(self, output_extension: str, sort: bool) -> t prefer_system_includes = self._language.get_config_value_as_bool("prefer_system_includes", False) if prefer_system_includes: - path_list_with_punctuation = ["<{}>".format(p) for p in path_list] + path_list_with_punctuation = [f"<{p}>" for p in path_list] else: - path_list_with_punctuation = ['"{}"'.format(p) for p in path_list] + path_list_with_punctuation = [f'"{p}"' for p in path_list] if sort: return sorted(path_list_with_punctuation + self._language.get_includes(dep_types)) - else: - return path_list_with_punctuation + self._language.get_includes(dep_types) + + return path_list_with_punctuation + self._language.get_includes(dep_types) @classmethod def make_path( @@ -60,7 +75,7 @@ def make_path( output_extension: typing.Optional[str] = None, ) -> pathlib.Path: """ - Common method for createing a relative path to a datatype source file. + Common method for creating a relative path to a datatype source file. .. invisible-code-block: python @@ -94,9 +109,7 @@ def make_path( """ if language is None: - short_name = "{short}_{major}_{minor}".format( - short=dt.short_name, major=dt.version.major, minor=dt.version.minor - ) + short_name = f"{dt.short_name}_{dt.version.major}_{dt.version.minor}" else: short_name = language.filter_short_reference_name(dt, id_type="path") @@ -116,8 +129,10 @@ def make_path( def _make_ns_list(cls, language: typing.Optional[Language], dt: pydsdl.SerializableType) -> typing.List[str]: if language is not None and language.enable_stropping: return [language.filter_id(x, id_type="path") for x in dt.full_namespace.split(".")] - else: - return typing.cast(typing.List[str], dt.full_namespace.split(".")) + return typing.cast(typing.List[str], dt.full_namespace.split(".")) + + +# +-------------------------------------------------------------------------------------------------------------------+ class UniqueNameGenerator: @@ -126,17 +141,23 @@ class UniqueNameGenerator: This should be made available as a private global within each template. """ - _singleton = None # type: typing.Optional['UniqueNameGenerator'] + _singleton: typing.Optional["UniqueNameGenerator"] = None def __init__(self) -> None: - self._index_map = {} # type: typing.Dict[str, typing.Dict[str, int]] + self._index_map: typing.Dict[str, typing.Dict[str, int]] = {} @classmethod def reset(cls) -> None: + """ + Resets the singleton instance of the UniqueNameGenerator. + """ cls._singleton = cls() @classmethod def get_instance(cls) -> "UniqueNameGenerator": + """ + Returns the singleton instance of the UniqueNameGenerator. + """ if cls._singleton is None: raise RuntimeError("No UniqueNameGenerator has been created. Please use reset to create.") return cls._singleton @@ -159,9 +180,12 @@ def __call__(self, key: str, base_token: str, prefix: str, suffix: str) -> str: next_index = 0 keymap[base_token] = 1 - return "{prefix}{base_token}{index}{suffix}".format( - prefix=prefix, base_token=base_token, index=next_index, suffix=suffix - ) + return f"{prefix}{base_token}{next_index}{suffix}" + + +# +-------------------------------------------------------------------------------------------------------------------+ +# | ENCODERS +# +-------------------------------------------------------------------------------------------------------------------+ class TokenEncoder: @@ -329,9 +353,7 @@ def __init__( self._stropping_suffix = language.get_config_value("stropping_suffix", "") self._encoding_prefix = language.get_config_value("encoding_prefix", "") try: - self._whitespace_encoding_char = language.get_config_value( - "whitespace_encoding_char" - ) # type: typing.Optional[str] + self._whitespace_encoding_char: typing.Optional[str] = language.get_config_value("whitespace_encoding_char") except KeyError: self._whitespace_encoding_char = None self._collapse_whitespace_when_encoding = language.get_config_value_as_bool("collapse_whitespace_when_encoding") @@ -370,9 +392,8 @@ def _encode(self, token: str, token_type: str, dry_run: bool) -> str: encoded = token_pattern.sub(self._encoding_filter, encoded) elif token_pattern.match(encoded): raise RuntimeError( - 'Unstable encoding: using prefix "{}" partially encoded token: "{}"'.format( - self._encoding_prefix, encoded - ) + f'Unstable encoding: using prefix "{self._encoding_prefix}" partially encoded token: ' + '"{encoded}"' ) except KeyError: pass @@ -386,8 +407,8 @@ def _strop_by_keyword(self, token: str, token_type: str, dry_run: bool) -> str: stropped = self._stropping_prefix + stropped + self._stropping_suffix else: raise RuntimeError( - 'input token "{}" of type "{}" yielded an illegal token after ' - "stropping: {}".format(stropped, token_type, stropped) + f'input token "{stropped}" of type "{token_type}" yielded an illegal token after ' + "stropping: {stropped}" ) return stropped @@ -402,8 +423,8 @@ def _strop_by_pattern(self, token: str, token_type: str, dry_run: bool) -> str: stropped = self._stropping_prefix + stropped + self._stropping_suffix else: raise RuntimeError( - 'input token "{}" of type "{}" yielded an illegal token after ' - "stropping: {}".format(stropped, token_type, stropped) + f'input token "{stropped}" of type "{token_type}" yielded an illegal token after ' + "stropping: {stropped}" ) return stropped @@ -431,13 +452,21 @@ def _do_for_type_and_all( # +------------------------------------------------------------------------------------------------------------+ def encode_character(self, c: str) -> str: + """ + Encode a character into a string representation. + + :param c: The character to encode. + :return: The string representation of the encoded character. + """ if self._whitespace_encoding_char is not None and c.isspace(): return self._whitespace_encoding_char - else: - return "{}{:04X}".format(self._encoding_prefix, ord(c)) + return f"{self._encoding_prefix}{ord(c):04X}" @functools.lru_cache(maxsize=1024) - def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 + def strop(self, token: str, token_type: str = "any") -> str: + """ + Strops a token such that it is a valid identifier for the given language. + """ token_type_lower = token_type.lower() if token_type_lower == "all": raise ValueError( @@ -461,8 +490,7 @@ def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 except RuntimeError as pending_error: if self._stropping_failure_handler is None: raise pending_error - else: - stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) + stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) # and check that the stropping didn't result in a keyword try: @@ -470,8 +498,7 @@ def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 except RuntimeError as pending_error: if self._stropping_failure_handler is None: raise pending_error - else: - stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) + stropped = self._stropping_failure_handler(self, stropped, token_type, pending_error) # finally, we make sure stropping didn't result in encoding violations try: @@ -479,8 +506,7 @@ def strop(self, token: str, token_type: str = "any") -> str: # noqa: C901 except RuntimeError as pending_error: if self._encoding_failure_handler is None: raise pending_error - else: - stropped = self._encoding_failure_handler(self, stropped, token_type, pending_error) + stropped = self._encoding_failure_handler(self, stropped, token_type, pending_error) return stropped diff --git a/src/nunavut/lang/_config.py b/src/nunavut/lang/_config.py index 12bab95d..6ce76d17 100644 --- a/src/nunavut/lang/_config.py +++ b/src/nunavut/lang/_config.py @@ -1,53 +1,23 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # -"""Logic for parsing language configuration. - +""" +Logic for parsing language configuration. """ import re import types import typing -from enum import auto, Enum - -from yaml import Loader as YamlLoader +from yaml import SafeLoader as YamlLoader from yaml import load as yaml_loader -from nunavut._utilities import deep_update - -NUNAVUT_LANG_CPP = "nunavut.lang.cpp" - - -class ConstructorConvention(Enum): - Default = "default" - UsesLeadingAllocator = "uses-leading-allocator" - UsesTrailingAllocator = "uses-trailing-allocator" - - @staticmethod - def parse_string(s: str) -> typing.Optional[typing.Any]: # annoying mypy cheat due to returning type being defined - for e in ConstructorConvention: - if s == e.value: - return e - return None - - -class SpecialMethod(Enum): - """ - Enum used in the Jinja templates to differentiate different kinds of constructrors - """ - - AllocatorConstructor = auto() - """ Constructor that takes an allocator as its single, required argument """ - - InitializingConstructorWithAllocator = auto() - """ Constructor that takes an initializing value for each field followed by the allocator argument """ - CopyConstructorWithAllocator = auto() - """ Copy constructor that also takes an allocator argument """ +from nunavut._utilities import deep_update, no_default_value - MoveConstructorWithAllocator = auto() - """ Move constructor that also takes an allocator argument """ +# +-------------------------------------------------------------------------------------------------------------------+ +# | LANGUAGE CONFIGURATION +# +-------------------------------------------------------------------------------------------------------------------+ class LanguageConfig: @@ -76,7 +46,7 @@ class LanguageConfig: from nunavut.lang import LanguageConfig config = LanguageConfig() - config.update_from_string(example_yaml) + config.update_from_yaml_string(example_yaml) data = config.sections() assert len(data) == 3 @@ -104,17 +74,17 @@ class LanguageConfig: .. invisible-code-block: python - config.update_from_string(example_yaml) + config.update_from_yaml_string(example_yaml) assert 'a_dictionary' == config.sections()['nunavut.lang.d']['key_one'][2]['list']['is'] """ SECTION_NAME_PATTERN = re.compile( r"^nunavut\.lang\.([a-zA-Z]{1}\w*)$" - ) #: Required pattern for section name identifers. + ) #: Required pattern for section name identifiers. def __init__(self): # type: ignore - self._sections = dict() # type: typing.Dict[str, typing.Dict[str, typing.Any]] + self._sections: typing.Dict[str, typing.Dict[str, typing.Any]] = {} def update(self, configuration: typing.Any) -> None: """ @@ -246,25 +216,46 @@ def update(self, configuration: typing.Any) -> None: raise TypeError("section names must be strings") if not self.SECTION_NAME_PATTERN.match(section_name): raise ValueError( - 'Section name "{}" is invalid. See LanguageConfig documentation for rules.'.format(section_name) + f'Section name "{section_name}" is invalid. See LanguageConfig documentation for rules.' ) self.update_section(section_name, section_data) def update_section(self, section_name: str, configuration: typing.Any) -> None: + """ + Update a section of the configuration. + """ self._sections[section_name] = deep_update(self._sections.get(section_name, {}), configuration) def sections(self) -> typing.Dict[str, typing.Dict[str, typing.Any]]: + """ + Get all sections of the configuration. + """ return self._sections - def update_from_string(self, string: str, context: typing.Optional[str] = None) -> None: + def update_from_yaml_string(self, string: str) -> None: + """ + Update the configuration from a yaml string. + Calls :meth:`update` with the parsed yaml data and will raise the same exceptions. + """ configuration = yaml_loader(string, Loader=YamlLoader) self.update(configuration) - def update_from_file(self, f: typing.TextIO, context: typing.Optional[str] = None) -> None: + def update_from_yaml_file(self, f: typing.TextIO) -> None: + """ + Update the configuration from a yaml file. + Calls :meth:`update` with the parsed yaml data and will raise the same exceptions. + """ configuration = yaml_loader(f, Loader=YamlLoader) self.update(configuration) def set(self, section: str, option: str, value: typing.Any) -> None: + """ + Set a configuration value. + + :param section: The section to set the value in. + :param option: The option to set. + :param value: The value to set. + """ self._sections[section][option] = value def add_section(self, section_name: str) -> None: @@ -306,15 +297,14 @@ def add_section(self, section_name: str) -> None: if not isinstance(section_name, str): raise TypeError("section names must be strings") if not self.SECTION_NAME_PATTERN.match(section_name): - raise ValueError( - 'Section name "{}" is invalid. See LanguageConfig documentation for rules.'.format(section_name) - ) + raise ValueError(f'Section name "{section_name}" is invalid. See LanguageConfig documentation for rules.') if section_name in self._sections: - raise ValueError("Section {} is already defined.".format(section_name)) - self._sections[section_name] = dict() + raise ValueError(f"Section {section_name} is already defined.") + self._sections[section_name] = {} _UNSET = object() # Used internally to allow "None" as a default value. + @no_default_value def _get_config_value_raw(self, section_name: str, key: str, default_value: typing.Any) -> typing.Any: """ .. invisible-code-block: python @@ -361,15 +351,13 @@ def _get_config_value_raw(self, section_name: str, key: str, default_value: typi except KeyError: if default_value is not self._UNSET: return default_value - else: - raise + raise try: return section_data[key] except KeyError: if default_value is not self._UNSET: return default_value - else: - raise + raise def get_config_value(self, section_name: str, key: str, default_value: typing.Optional[str] = None) -> str: """ @@ -486,8 +474,7 @@ def get_config_value_as_bool(self, section_name: str, key: str, default_value: b result = self.get_config_value(section_name, key, default_value="false" if not default_value else "true") if result.lower() == "false" or result == "0": return False - else: - return bool(result) + return bool(result) def get_config_value_as_dict( self, section_name: str, key: str, default_value: typing.Optional[typing.Dict] = None @@ -556,7 +543,7 @@ def get_config_value_as_dict( return raw_value if default_value is None: - raise TypeError("{}.{} exists but is not a dict. (is type {})".format(section_name, key, type(raw_value))) + raise TypeError(f"{section_name}.{key} exists but is not a dict. (is type {type(raw_value)})") return default_value @@ -626,30 +613,10 @@ def get_config_value_as_list( return raw_value if default_value is None: - raise TypeError("{}.{} exists but is not a list. (is type {})".format(section_name, key, type(raw_value))) + raise TypeError(f"{section_name}.{key} exists but is not a list. (is type {type(raw_value)})") return default_value - def apply_defaults(self, language_standard: str) -> None: - defaults_key = f"{language_standard}_options" - if defaults_key in self.sections()[NUNAVUT_LANG_CPP]: - defaults_data = self.get_config_value_as_dict(NUNAVUT_LANG_CPP, defaults_key) - self.update_section(NUNAVUT_LANG_CPP, {"options": defaults_data}) - - def validate_language_options(self) -> None: - options = self.get_config_value_as_dict(NUNAVUT_LANG_CPP, "options") - ctor_convention_str: str = options["ctor_convention"] - ctor_convention = ConstructorConvention.parse_string(ctor_convention_str) - if not ctor_convention: - raise RuntimeError( - f"ctor_convention property '{ctor_convention_str}' is invalid and must be one of " - + (",".join([f"'{e.value}'" for e in ConstructorConvention])) - ) - if ctor_convention != ConstructorConvention.Default and not options["allocator_type"]: - raise RuntimeError( - f"allocator_type property must be specified when ctor_convention is '{ctor_convention_str}'" - ) - # +-------------------------------------------------------------------------------------------------------------------+ # | VersionReader @@ -658,45 +625,59 @@ def validate_language_options(self) -> None: class VersionReader: """ - Helper to read an "x.y.z" semantic version from python modules as a module variable - "__version__" + Helper to read an "x.y.z" semantic version from python modules as a module variable `MODULE_VERSION_ATTRIBUTE_NAME`. + :param module_name: The name of the module to read the version from. """ MODULE_VERSION_ATTRIBUTE_NAME = "__version__" @classmethod def parse_version(cls, version_string: str) -> typing.Optional[typing.Tuple[int, int, int]]: + """ + Parse a version string into a tuple of (major, minor, patch). + :param version_string: The version string to parse. + :return: The version as a tuple of (major, minor, patch) or None if the version string is not in the expected + format. + """ version_array = [int(x) for x in version_string.split(".")] if len(version_array) != 3: return None - else: - return (version_array[0], version_array[1], version_array[2]) + return (version_array[0], version_array[1], version_array[2]) @classmethod def read_version(cls, module: "types.ModuleType") -> typing.Tuple[int, int, int]: - version = getattr(module, cls.MODULE_VERSION_ATTRIBUTE_NAME, "0.0.0") # type: str + """ + Read the version from a module. + + :param module: The module to read the version from. + :return: The version as a tuple of (major, minor, patch). + :raises: ValueError if the version is not in the expected format. + """ + version: str = getattr(module, cls.MODULE_VERSION_ATTRIBUTE_NAME, "0.0.0") version_tuple = cls.parse_version(version) if version_tuple is None: - raise RuntimeError( - 'Invalid {} "{}" for module {} (expected "x.y.z")'.format( - cls.MODULE_VERSION_ATTRIBUTE_NAME, version, module.__name__ - ) + raise ValueError( + f'Invalid {cls.MODULE_VERSION_ATTRIBUTE_NAME} "{version}" for module {module.__name__}' + '(expected "x.y.z")' ) return version_tuple def __init__(self, module_name: str): self._module_name = module_name - self._cached = None # type: typing.Optional[typing.Tuple[int, int, int]] + self._cached: typing.Optional[typing.Tuple[int, int, int]] = None @property def version(self) -> typing.Tuple[int, int, int]: + """ + The version of the module as a tuple of (major, minor, patch). + """ if self._cached is None: self._cached = self._get_version() return self._cached def _get_version(self) -> typing.Tuple[int, int, int]: - import importlib + import importlib # pylint: disable=import-outside-toplevel try: return self.read_version(importlib.import_module(self._module_name)) diff --git a/src/nunavut/lang/_language.py b/src/nunavut/lang/_language.py index 0734e96f..280b0ab2 100644 --- a/src/nunavut/lang/_language.py +++ b/src/nunavut/lang/_language.py @@ -1,7 +1,7 @@ # -# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2022 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Language-specific support in nunavut. @@ -56,21 +56,21 @@ class Language(metaclass=abc.ABCMeta): my_lang = _GenericLanguage("foo", mock_config) # module must be within 'nunavut' assert False - except RuntimeError: + except ValueError: pass try: my_lang = _GenericLanguage("nunavut.foo", mock_config) # module must be within 'nunavut.lang' assert False - except RuntimeError: + except ValueError: pass try: my_lang = _GenericLanguage("not.nunavut.foo", mock_config) # module must be within 'nunavut.lang' assert False - except RuntimeError: + except ValueError: pass my_lang = _GenericLanguage("nunavut.lang.foo", mock_config) @@ -88,6 +88,7 @@ class Language(metaclass=abc.ABCMeta): WKCV_NAMED_TYPES = "named_types" WKCV_NAMED_VALUES = "named_values" WKCV_LANGUAGE_OPTIONS = "options" + WKCV_LANGUAGE_OPTION_DEFAULTS = "defaults" @classmethod def default_filter_id_for_target(cls, instance: typing.Any) -> str: @@ -106,17 +107,23 @@ def default_filter_id_for_target(cls, instance: typing.Any) -> str: # | LIFECYCLE AND DATA MODEL # +-----------------------------------------------------------------------+ - def __init__(self, language_module_name: str, config: LanguageConfig, **kwargs: typing.Any): - self._globals = None # type: typing.Optional[typing.Mapping[str, typing.Any]] + def __init__( + self, language_module_name: str, config: LanguageConfig, **kwargs: typing.Any + ): # pylint: disable=unused-argument + self._globals: typing.Optional[typing.Mapping[str, typing.Any]] = None self._section = language_module_name if not self._section.startswith(LanguageClassLoader.MODULE_PREFIX): - raise RuntimeError("Unknown module name for language: {}".format(self._section)) + raise ValueError(f"Unknown module name for language: {self._section}") self._language_name = LanguageClassLoader.to_language_name(self._section) self._config = config - self._language_options = config.get_config_value_as_dict(self._section, self.WKCV_LANGUAGE_OPTIONS, dict()) - self._filters = dict() # type: typing.Dict[str, typing.Callable] - self._tests = dict() # type: typing.Dict[str, typing.Callable] - self._uses = dict() # type: typing.Dict[str, typing.Callable] + self._filters: typing.Dict[str, typing.Callable] = {} + self._tests: typing.Dict[str, typing.Callable] = {} + self._uses: typing.Dict[str, typing.Callable] = {} + + self._language_options = self._validate_language_options( + config.get_config_value_as_dict(self._section, self.WKCV_LANGUAGE_OPTION_DEFAULTS, {}), + config.get_config_value_as_dict(self._section, self.WKCV_LANGUAGE_OPTIONS, {}), + ) def __getattr__(self, name: str) -> typing.Any: """ @@ -128,7 +135,34 @@ def __getattr__(self, name: str) -> typing.Any: try: return self.get_globals()[name] except KeyError as e: - raise AttributeError(e) + raise AttributeError(e) from e + + def __str__(self) -> str: + return self._language_name + + def _validate_language_options( + self, defaults: typing.Dict[str, typing.Any], options: typing.Dict[str, typing.Any] + ) -> typing.Dict[str, typing.Any]: + # pylint: disable=unused-argument + """ + Subclasses may override this method to validate language options. It will be invoked once + by the base class constructor before setting the language options property. + :param defaults: The a section of the language configuration that contains default values for options. The + format of this section is language-specific + :param options: The options to validate. + :return: The validated or modified options. + :throws: ValueError if the options are invalid. + """ + return options + + def _validate_globals(self, globals_map: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: + """ + Subclasses may override this method to populate additional language-specific globals + :param globals_map: The globals map to validate. + :return: The validated or modified globals map. + :throws: ValueError if the globals are invalid. + """ + return globals_map # +-----------------------------------------------------------------------+ # | PROPERTIES @@ -222,10 +256,6 @@ def named_values(self) -> typing.Mapping[str, str]: # | METHODS # +-----------------------------------------------------------------------+ - def _add_additional_globals(self, globals_map: typing.Dict[str, typing.Any]) -> None: - """Subclasses may override this method to populate additional language-specific globals""" - pass - def get_support_module(self) -> typing.Tuple[str, typing.Tuple[int, int, int], typing.Optional["types.ModuleType"]]: """ Returns the module object for the language support files. @@ -247,7 +277,7 @@ def get_support_module(self) -> typing.Tuple[str, typing.Tuple[int, int, int], t assert support_version[0] == 1 """ - module_name = "{}.support".format(self._section) + module_name = f"{self._section}.support" try: module = importlib.import_module(module_name) @@ -261,6 +291,9 @@ def get_support_module(self) -> typing.Tuple[str, typing.Tuple[int, int, int], t @functools.lru_cache() def get_dependency_builder(self, for_type: pydsdl.Any) -> DependencyBuilder: + """ + Get a dependency builder for the given type. + """ return DependencyBuilder(for_type) @abc.abstractmethod @@ -270,9 +303,8 @@ def get_includes(self, dep_types: Dependencies) -> typing.List[str]: :param Dependencies dep_types: A description of the dependencies includes are needed for. :return: A list of include file paths. The list may be empty if no includes were needed. """ - pass - def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: + def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: # pylint: disable=unused-argument """ Produces a valid identifier in the language for a given object. The encoding may not be reversible. @@ -299,11 +331,10 @@ def filter_short_reference_name( value can be 'typedef', 'macro', 'function', or 'enum'. Use 'any' to apply stropping rules for all identifier types to the instance. """ - short_name = "{short}_{major}_{minor}".format(short=t.short_name, major=t.version.major, minor=t.version.minor) + short_name = f"{t.short_name}_{t.version.major}_{t.version.minor}" if YesNoDefault.test_truth(stropping, self.enable_stropping): return self.filter_id(short_name, id_type) - else: - return short_name + return short_name def get_config_value(self, key: str, default_value: typing.Optional[str] = None) -> str: """ @@ -352,7 +383,7 @@ def get_config_value_as_dict( :param str key: The config value to retrieve. :param default_value: The value to return if the key was not in the configuration. If provided this method - will not raise a KeyError nor a TypeError. + will not raise a KeyError nor a TypeError. :type default_value: typing.Optional[typing.Mapping[str, typing.Any]] :return: Either the value from the config or the default_value if provided. :rtype: typing.Mapping[str, typing.Any] @@ -370,7 +401,7 @@ def get_config_value_as_list( :param str key: The config value to retrieve. :param default_value: The value to return if the key was not in the configuration. If provided this method - will not raise a KeyError nor a TypeError. + will not raise a KeyError nor a TypeError. :type default_value: typing.Optional[typing.List[typing.Any]] :return: Either the value from the config or the default_value if provided. :rtype: typing.List[typing.Any] @@ -408,9 +439,9 @@ def get_support_files( if module is not None: # All language support modules must provide a list_support_files method # to allow the copy generator access to the packaged support files. - list_support_files = getattr( + list_support_files: typing.Callable[[ResourceType], typing.Generator[pathlib.Path, None, None]] = getattr( module, "list_support_files" - ) # type: typing.Callable[[ResourceType], typing.Generator[pathlib.Path, None, None]] + ) return list_support_files(resource_type) else: return empty_list_support_files() @@ -466,16 +497,14 @@ def get_globals(self) -> typing.Mapping[str, typing.Any]: :return: A mapping of global names to global values. """ if self._globals is None: - globals_map = dict() # type: typing.Dict[str, typing.Any] + globals_map: typing.Dict[str, typing.Any] = {} for key, value in self.named_types.items(): - globals_map["typename_{}".format(key)] = value + globals_map[f"typename_{key}"] = value for key, value in self.named_values.items(): - globals_map["valuetoken_{}".format(key)] = value - - self._add_additional_globals(globals_map) + globals_map[f"valuetoken_{key}"] = value - self._globals = globals_map + self._globals = self._validate_globals(globals_map) return self._globals def get_options(self) -> typing.Mapping[str, typing.Any]: @@ -537,7 +566,7 @@ class LanguageClassLoader: def to_language_name(cls, unknown_string: str) -> str: """ Helper method to take a string that is either a language name or a language module name - and always return a langauge name. + and always return a language name. .. invisible-code-block: python from nunavut.lang import LanguageClassLoader @@ -556,7 +585,7 @@ def to_language_name(cls, unknown_string: str) -> str: def to_language_module_name(cls, unknown_string: str) -> str: """ Helper method to take a string that is either a language name or a language module name - and always return a langauge module name. + and always return a language module name. .. invisible-code-block: python from nunavut.lang import LanguageClassLoader @@ -578,14 +607,17 @@ def _load_config(cls) -> LanguageConfig: parser = LanguageConfig() for resource in iter_package_resources(cls.MODULE_NAME, ".yaml"): ini_string = resource.read_text() - parser.update_from_string(ini_string) + parser.update_from_yaml_string(ini_string) return parser def __init__(self) -> None: - self._config = None # type: typing.Optional[LanguageConfig] + self._config: typing.Optional[LanguageConfig] = None @classmethod - def load_language_module(cls, language_name: str) -> "types.ModuleType": + def load_language_module(cls, language_name: str) -> types.ModuleType: + """ + Load a language module by name. + """ module_name = cls.to_language_module_name(language_name) return importlib.import_module(module_name) @@ -621,9 +653,7 @@ def load_language_class(self, language_name: str) -> typing.Tuple[types.ModuleTy language_type = typing.cast(typing.Type["Language"], getattr(ln_module, "Language")) except AttributeError: logging.debug( - "Unable to find a Language object in nunavut.lang.{}. Using a Generic language object".format( - language_name - ) + "Unable to find a Language object in nunavut.lang.%s. Using a Generic language object", language_name ) language_type = _GenericLanguage diff --git a/src/nunavut/lang/c/__init__.py b/src/nunavut/lang/c/__init__.py index 838d83cc..f1df4d44 100644 --- a/src/nunavut/lang/c/__init__.py +++ b/src/nunavut/lang/c/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating C. All filters in this @@ -10,7 +10,6 @@ import enum import fractions -import functools import re import typing @@ -24,7 +23,7 @@ template_language_test, template_volatile_filter, ) -from nunavut._utilities import YesNoDefault +from nunavut._utilities import YesNoDefault, cached_property from nunavut.jinja.environment import Environment from nunavut.lang._common import IncludeGenerator, TokenEncoder, UniqueNameGenerator from nunavut.lang._language import Language as BaseLanguage @@ -52,8 +51,8 @@ def _handle_stropping_failure( # we couldn't help after all. raise the pending error. raise pending_error - @functools.lru_cache(maxsize=None) - def _get_token_encoder(self) -> TokenEncoder: + @cached_property + def _token_encoder(self) -> TokenEncoder: """ Caching getter to ensure we don't have to recompile TokenEncoders for each filter invocation. """ @@ -72,12 +71,12 @@ def get_includes(self, dep_types: Dependencies) -> typing.List[str]: if dep_types.uses_primitive_static_array: # We include this for memset. std_includes.append("string.h") - return ["<{}>".format(include) for include in sorted(std_includes)] + return [f"<{include}>" for include in sorted(std_includes)] def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: raw_name = self.default_filter_id_for_target(instance) - vne = self._get_token_encoder() + vne = self._token_encoder return vne.strop(raw_name, id_type) @@ -286,7 +285,7 @@ def to_c_int(self, is_signed: bool) -> str: return intname def to_c_float(self) -> str: - if self.value == 8 or self.value == 16 or self.value == 32: + if self.value in (8, 16, 32): return "float" else: return "double" diff --git a/src/nunavut/lang/c/support/__init__.py b/src/nunavut/lang/c/support/__init__.py index 41fd4185..1b14f7c3 100644 --- a/src/nunavut/lang/c/support/__init__.py +++ b/src/nunavut/lang/c/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains supporting C headers to distribute with generated types. diff --git a/src/nunavut/lang/c/templates/__init__.py b/src/nunavut/lang/c/templates/__init__.py index 8c8df7c9..b692c232 100644 --- a/src/nunavut/lang/c/templates/__init__.py +++ b/src/nunavut/lang/c/templates/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains the Jinja templates to generate C headers. diff --git a/src/nunavut/lang/c/templates/base.j2 b/src/nunavut/lang/c/templates/base.j2 index 0b76dac1..b985675c 100644 --- a/src/nunavut/lang/c/templates/base.j2 +++ b/src/nunavut/lang/c/templates/base.j2 @@ -9,15 +9,21 @@ // This is an AUTO-GENERATED Cyphal DSDL data type implementation. Curious? See https://opencyphal.org. // You shouldn't attempt to edit this file. // -// Checking this file under version control is not recommended unless it is used as part of a high-SIL -// safety-critical codebase. The typical usage scenario is to generate it as part of the build process. +{%- if nunavut.embed_auditing_info %} +// Checking this file under version control is not recommended since metadata in this header will change for each +// build invocation (do not use --embed-auditing-info option to remove this comment). +{%- endif %} // // To avoid conflicts with definitions given in the source DSDL file, all entities created by the code generator // are named with an underscore at the end, like foo_bar_(). // // Generator: nunavut-{{ nunavut.version }} (serialization was {{ 'not ' * nunavut.support.omit }}enabled) +{%- if nunavut.embed_auditing_info %} // Source file: {{ T.source_file_path.as_posix() }} // Generated at: {{ now_utc }} UTC +{%- else %} +// Source file: {{ T.source_file_path.name }} +{%- endif %} // Is deprecated: {{ T.deprecated and 'yes' or 'no' }} // Fixed port-ID: {{ T.fixed_port_id }} // Full name: {{ T.full_name }} diff --git a/src/nunavut/lang/cpp/__init__.py b/src/nunavut/lang/cpp/__init__.py index 547e6d0d..25133244 100644 --- a/src/nunavut/lang/cpp/__init__.py +++ b/src/nunavut/lang/cpp/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating C++. All filters in this @@ -15,6 +15,7 @@ import re import textwrap import typing +from enum import Enum, auto import pydsdl @@ -25,14 +26,93 @@ template_language_list_filter, template_language_test, ) -from nunavut._utilities import YesNoDefault +from nunavut._utilities import YesNoDefault, cached_property from nunavut.jinja.environment import Environment from nunavut.lang._common import IncludeGenerator, TokenEncoder, UniqueNameGenerator -from nunavut.lang._config import ConstructorConvention, SpecialMethod from nunavut.lang._language import Language as BaseLanguage from nunavut.lang.c import _CFit from nunavut.lang.c import filter_literal as c_filter_literal +# +-------------------------------------------------------------------------------------------------------------------+ +# | ENUMERATIONS +# +-------------------------------------------------------------------------------------------------------------------+ + + +class ConstructorConvention(Enum): + """ + Indicates the convention used for constructors in the target language. + + .. invisible-code-block: python + + from nunavut.lang.cpp import ConstructorConvention + + assert "default" == ConstructorConvention.DEFAULT + assert ConstructorConvention.DEFAULT == ConstructorConvention.from_string("default") + assert ConstructorConvention.USES_LEADING_ALLOCATOR == \ + ConstructorConvention.from_string("uses-leading-allocator") + assert "uses-trailing-allocator" == str(ConstructorConvention.USES_TRAILING_ALLOCATOR) + + from pytest import raises as assert_raises + assert_raises(ValueError, ConstructorConvention.from_string, "not-a-convention") + + """ + + DEFAULT = "default" + USES_LEADING_ALLOCATOR = "uses-leading-allocator" + USES_TRAILING_ALLOCATOR = "uses-trailing-allocator" + + def __str__(self) -> str: + """ + Return a string representation of the ConstructorConvention enum value. + + :return: The string representation of the enum value. + :rtype: str + """ + return self.value + + def __eq__(self, value: object) -> bool: + if isinstance(value, str): + return super().__eq__(ConstructorConvention.from_string(value)) + return super().__eq__(value) + + @staticmethod + def from_string(s: str) -> "ConstructorConvention": + """ + Parse a string into a ConstructorConvention enum value. + + :param s: The string to parse. + :return: The enum value corresponding to the string. + :rtype: ConstructorConvention + :raises: ValueError if the string does not correspond to a valid enum value. + """ + for e in ConstructorConvention: + if s.lower().replace("_", "-") == e.value: + return e + raise ValueError(f"Invalid ConstructorConvention string '{s}'") + + +class SpecialMethod(Enum): + """ + Enum used in the Jinja templates to differentiate different kinds of constructors + """ + + ALLOCATOR_CONSTRUCTOR = auto() + """ Constructor that takes an allocator as its single, required argument """ + + INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR = auto() + """ Constructor that takes an initializing value for each field followed by the allocator argument """ + + COPY_CONSTRUCTOR_WITH_ALLOCATOR = auto() + """ Copy constructor that also takes an allocator argument """ + + MOVE_CONSTRUCTOR_WITH_ALLOCATOR = auto() + """ Move constructor that also takes an allocator argument """ + + +# +-------------------------------------------------------------------------------------------------------------------+ +# | LANGUAGE SUPPORT +# +-------------------------------------------------------------------------------------------------------------------+ + class Language(BaseLanguage): """ @@ -41,10 +121,77 @@ class Language(BaseLanguage): CPP_STD_EXTRACT_NUMBER_PATTERN = re.compile(r"(?:gnu|c)\+\+(\d(?:\w))") + def _validate_language_options( + self, defaults: typing.Dict[str, typing.Any], options: typing.Dict[str, typing.Any] + ) -> typing.Dict[str, typing.Any]: + """ + apply defaults based on language standard + + .. invisible-code-block: python + + from nunavut.lang import LanguageContextBuilder + from pytest import raises as assert_raises + + # test std + language = LanguageContextBuilder(include_experimental_languages=True)\ + .set_target_language("cpp")\ + .set_target_language_configuration_override("options", { "std_flavor":"std"})\ + .create()\ + .get_target_language() + + assert language._validate_language_options( + {}, + { + "std":"c++17", + "ctor_convention": "default" + } + ) == { + "std":"c++17", + "ctor_convention": "default" + } + + assert_raises(ValueError, \ + language._validate_language_options, {}, {"ctor_convention": "default"}) + + assert_raises(ValueError, \ + language._validate_language_options, {}, {"std":"c++17", "ctor_convention": "uses-leading-allocator"}) + + assert_raises(ValueError, \ + language._validate_language_options, {}, \ + {"std":"c++17", "ctor_convention": "uses-leading-allocator", "allocator_type": ""}) + + """ + try: + language_standard = options["std"] + except KeyError as e: + raise ValueError("The 'std' option must be in the language options for the C++ language.") from e + + if language_standard in defaults: + options.update(defaults[language_standard]) + + try: + ctor_convention = ConstructorConvention.from_string(options["ctor_convention"]) + except KeyError as e: + raise ValueError("No constructor convention option in C++ language options. This is required.") from e + + if ctor_convention != ConstructorConvention.DEFAULT and ( + "allocator_type" not in options or not options["allocator_type"] + ): + raise ValueError( + f"allocator_type property must be specified when ctor_convention is '{str(ctor_convention)}'" + ) + return options + + def _validate_globals(self, globals_map: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: + globals_map["ConstructorConvention"] = ConstructorConvention + globals_map["SpecialMethod"] = SpecialMethod + return globals_map + @staticmethod def _handle_stropping_or_encoding_failure( encoder: TokenEncoder, stropped: str, token_type: str, pending_error: RuntimeError ) -> str: + # pylint: disable=unused-argument """ If the generic stropping fails we take one last look to see if there is something c++-specific we can do. """ @@ -57,15 +204,15 @@ def _handle_stropping_or_encoding_failure( if m: # Resolve the conflict between C's global identifier rules and our desire to use # '_' as a stropping prefix: - return "_{}{}".format(m.group(1).lower(), stropped[m.end() :]) + return f"_{m.group(1).lower()}{stropped[m.end() :]}" # we couldn't help after all. raise the pending error. raise pending_error - @functools.lru_cache(maxsize=None) - def _get_token_encoder(self) -> TokenEncoder: + @cached_property + def _token_encoder(self) -> TokenEncoder: """ - Caching getter to ensure we don't have to recompile TokenEncoders for each filter invocation. + Cached property to ensure we don't have to recompile TokenEncoders for each filter invocation. """ return TokenEncoder( self, @@ -73,8 +220,39 @@ def _get_token_encoder(self) -> TokenEncoder: encoding_failure_handler=self._handle_stropping_or_encoding_failure, ) - def _standard_version(self) -> int: + @property + def standard_flavor(self) -> str: + """ + A flavor of the C++ language standard being targeted. + + .. invisible-code-block: python + + from nunavut.lang import LanguageContextBuilder + + # test std + language = LanguageContextBuilder(include_experimental_languages=True)\ + .set_target_language("cpp")\ + .set_target_language_configuration_override("options", { "std_flavor":"std"})\ + .create()\ + .get_target_language() + + assert language.standard_flavor == 'std' + + # test cetl + language = LanguageContextBuilder(include_experimental_languages=True)\ + .set_target_language("cpp")\ + .set_target_language_configuration_override("options", { "std_flavor":"cetl"})\ + .create()\ + .get_target_language() + assert language.standard_flavor == 'cetl' + """ + return str(self.get_option("std_flavor")) + + @property + def standard_version(self) -> int: """ + The numeric version of the C++ language standard being targeted. + .. invisible-code-block: python from nunavut.lang import LanguageContextBuilder @@ -86,7 +264,7 @@ def _standard_version(self) -> int: .create()\ .get_target_language() - assert language._standard_version() == 17 + assert language.standard_version == 17 # test c++14 language = LanguageContextBuilder(include_experimental_languages=True)\ @@ -94,7 +272,7 @@ def _standard_version(self) -> int: .set_target_language_configuration_override("options", { "std":"c++14"})\ .create()\ .get_target_language() - assert language._standard_version() == 14 + assert language.standard_version == 14 # test gnu++20 language = LanguageContextBuilder(include_experimental_languages=True)\ @@ -103,7 +281,7 @@ def _standard_version(self) -> int: .create()\ .get_target_language() - assert language._standard_version() == 20 + assert language.standard_version == 20 """ std = str(self.get_option("std", "")) @@ -111,50 +289,48 @@ def _standard_version(self) -> int: if match is not None and len(match.groups()) >= 1: return int(match.group(1)) - else: - return 0 + return 0 - def _has_variant(self) -> bool: + @property + def has_variant(self) -> bool: """ .. invisible-code-block: python from nunavut.lang import LanguageClassLoader # test c++17 - language = LanguageContextBuilder(include_experimental_languages=True)\ - .set_target_language("cpp")\ - .set_target_language_configuration_override("options", { "std":"c++17"})\ - .create()\ - .get_target_language() + language = ( + LanguageContextBuilder(include_experimental_languages=True) + .set_target_language("cpp") + .set_target_language_configuration_override("options", { "std":"c++17"}) + .create() + .get_target_language() + ) - assert language._has_variant() + assert language.has_variant # test c++14 - language = LanguageContextBuilder(include_experimental_languages=True)\ - .set_target_language("cpp")\ - .set_target_language_configuration_override("options", { "std":"c++14"})\ - .create()\ + language = ( + LanguageContextBuilder(include_experimental_languages=True) + .set_target_language("cpp") + .set_target_language_configuration_override("options", { "std":"c++14"}) + .create() .get_target_language() - - assert not language._has_variant() + ) + assert not language.has_variant # test gnu++20 - language = LanguageContextBuilder(include_experimental_languages=True)\ - .set_target_language("cpp")\ - .set_target_language_configuration_override("options", { "std":"gnu++20"})\ - .create()\ + language = ( + LanguageContextBuilder(include_experimental_languages=True) + .set_target_language("cpp") + .set_target_language_configuration_override("options", { "std":"gnu++20"}) + .create() .get_target_language() + ) - assert language._has_variant() + assert language.has_variant """ - return self._standard_version() >= 17 - - def _add_additional_globals(self, globals_map: typing.Dict[str, typing.Any]) -> None: - """ - Make additional globals available in the cpp jinja templates - """ - globals_map["ConstructorConvention"] = ConstructorConvention - globals_map["SpecialMethod"] = SpecialMethod + return self.standard_version >= 17 def get_includes(self, dep_types: Dependencies) -> typing.List[str]: """ @@ -211,7 +387,7 @@ def do_includes_test(override_vla_include, override_allocator_include): do_includes_test(False, False) do_includes_test(False, True) """ - std_includes = [] # type: typing.List[str] + std_includes: typing.List[str] = [] std_includes.append("limits") # we always include limits to support static assertions if self.get_config_value_as_bool("use_standard_types"): if dep_types.uses_integer: @@ -220,9 +396,9 @@ def do_includes_test(override_vla_include, override_allocator_include): std_includes.append("array") if dep_types.uses_boolean_static_array: std_includes.append("bitset") - if dep_types.uses_union and self._has_variant(): + if dep_types.uses_union and self.has_variant: std_includes.append("variant") - includes_formatted = ["<{}>".format(include) for include in sorted(std_includes)] + includes_formatted = [f"<{include}>" for include in sorted(std_includes)] allocator_include = str(self.get_option("allocator_include", "")) if len(allocator_include) > 0: @@ -238,20 +414,20 @@ def do_includes_test(override_vla_include, override_allocator_include): def filter_id(self, instance: typing.Any, id_type: str = "any") -> str: raw_name = self.default_filter_id_for_target(instance) - return self._get_token_encoder().strop(raw_name, id_type) + return self._token_encoder.strop(raw_name, id_type) - def create_bitset_decl(self, type: str, max_size: int) -> str: - return "std::bitset<{MAX_SIZE}>".format(MAX_SIZE=max_size) + def create_bitset_decl(self, max_size: int) -> str: + return f"std::bitset<{max_size}>" - def create_array_decl(self, type: str, max_size: int) -> str: - return "std::array<{TYPE},{MAX_SIZE}>".format(TYPE=type, MAX_SIZE=max_size) + def create_array_decl(self, data_type: str, max_size: int) -> str: + return f"std::array<{data_type},{max_size}>" - def create_vla_decl(self, type: str, max_size: int) -> str: + def create_vla_decl(self, data_type: str, max_size: int) -> str: variable_array_type_template = self.get_option("variable_array_type_template") if not isinstance(variable_array_type_template, str) or len(variable_array_type_template) == 0: raise RuntimeError("You must specify a value for the 'variable_array_type_template' option.") - rebind_allocator = "std::allocator_traits::rebind_alloc<{TYPE}>".format(TYPE=type) - return variable_array_type_template.format(TYPE=type, MAX_SIZE=max_size, REBIND_ALLOCATOR=rebind_allocator) + rebind_allocator = f"std::allocator_traits::rebind_alloc<{data_type}>" + return variable_array_type_template.format(TYPE=data_type, MAX_SIZE=max_size, REBIND_ALLOCATOR=rebind_allocator) @template_language_test(__name__) @@ -301,7 +477,27 @@ def uses_std_variant(language: Language) -> bool: jinja_filter_tester(None, template, '#include "user_variant.h"', lctx) """ - return language._has_variant() + return language.has_variant + + +@template_language_test(__name__) +def uses_cetl(language: Language) -> bool: + """ + Uses query for Cyphal Embedded Template Library. + + If this is true then CETL is used to ensure compatibility back to C++14. + """ + return language.standard_flavor == "cetl" + + +@template_language_test(__name__) +def uses_pmr(language: Language) -> bool: + """ + Uses query for C++17 Polymorphic Memory Resources. + + If this is true then additional C++ code is generated to support the use of polymorphic memory resources. + """ + return language.standard_flavor == "pmr" @template_language_filter(__name__) @@ -815,8 +1011,7 @@ def filter_short_reference_name(language: Language, t: pydsdl.CompositeType) -> if isinstance(t, pydsdl.ServiceType): if YesNoDefault.test_truth(YesNoDefault.DEFAULT, language.enable_stropping): return language.filter_id(t.short_name) - else: - return str(t.short_name) + return str(t.short_name) return language.filter_short_reference_name(t) @@ -932,14 +1127,14 @@ def filter_explicit_decorator(language: Language, instance: pydsdl.Any, special_ arg_count: int = len(instance.fields_except_padding) + ( 0 if language.get_option("allocator_is_default_constructible") else 1 ) - if special_method == SpecialMethod.InitializingConstructorWithAllocator and arg_count == 1: + if special_method == SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR and arg_count == 1: return f"explicit {name}" - else: - return f"{name}" + return f"{name}" @template_language_filter(__name__) def filter_default_value_initializer(language: Language, instance: pydsdl.Any) -> str: + # pylint: disable=unused-argument """ Emit a default initialization expression for the given instance if primitive, array, or composite. @@ -955,14 +1150,14 @@ def filter_default_value_initializer(language: Language, instance: pydsdl.Any) - def needs_initializing_value(special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" - return special_method == SpecialMethod.InitializingConstructorWithAllocator or needs_rhs(special_method) + return special_method == SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR or needs_rhs(special_method) def needs_rhs(special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" return special_method in ( - SpecialMethod.CopyConstructorWithAllocator, - SpecialMethod.MoveConstructorWithAllocator, + SpecialMethod.COPY_CONSTRUCTOR_WITH_ALLOCATOR, + SpecialMethod.MOVE_CONSTRUCTOR_WITH_ALLOCATOR, ) @@ -975,14 +1170,14 @@ def needs_allocator(instance: pydsdl.Any) -> bool: def needs_vla_init_args(instance: pydsdl.Any, special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" - return special_method == SpecialMethod.AllocatorConstructor and isinstance( + return special_method == SpecialMethod.ALLOCATOR_CONSTRUCTOR and isinstance( instance.data_type, pydsdl.VariableLengthArrayType ) def needs_move(special_method: SpecialMethod) -> bool: """Helper method used by filter_value_initializer()""" - return special_method == SpecialMethod.MoveConstructorWithAllocator + return special_method == SpecialMethod.MOVE_CONSTRUCTOR_WITH_ALLOCATOR def requires_initialization(instance: pydsdl.Any) -> bool: @@ -999,7 +1194,7 @@ def assemble_initializer_expression( ) -> str: """Helper method used by filter_value_initializer()""" if wrap: - rhs = "{}({})".format(wrap, rhs) + rhs = f"{wrap}({rhs})" args = [] if rhs: args.append(rhs) @@ -1031,7 +1226,7 @@ def filter_value_initializer(language: Language, instance: pydsdl.Any, special_m trailing_args.append(constructor_args.format(MAX_SIZE=instance.data_type.capacity)) if needs_allocator(instance): - if language.get_option("ctor_convention") == ConstructorConvention.UsesLeadingAllocator.value: + if language.get_option("ctor_convention") == ConstructorConvention.USES_LEADING_ALLOCATOR.value: leading_args.extend(["std::allocator_arg", "allocator"]) else: trailing_args.append("allocator") @@ -1046,13 +1241,15 @@ def filter_value_initializer(language: Language, instance: pydsdl.Any, special_m @template_language_filter(__name__) def filter_default_construction(language: Language, instance: pydsdl.Any, reference: str) -> str: + """ + Emit a default construction expression for the given instance if it is a composite type. + """ if ( isinstance(instance, pydsdl.CompositeType) - and language.get_option("ctor_convention") != ConstructorConvention.Default.value + and language.get_option("ctor_convention") != ConstructorConvention.DEFAULT.value ): return f"{reference}.get_allocator()" - else: - return "" + return "" @template_language_filter(__name__) @@ -1060,17 +1257,16 @@ def filter_declaration(language: Language, instance: pydsdl.Any) -> str: """ Emit a declaration statement for the given instance. """ - if isinstance(instance, pydsdl.PrimitiveType) or isinstance(instance, pydsdl.VoidType): + if isinstance(instance, (pydsdl.PrimitiveType, pydsdl.VoidType)): return filter_type_from_primitive(language, instance) - elif isinstance(instance, pydsdl.VariableLengthArrayType): + if isinstance(instance, pydsdl.VariableLengthArrayType): return language.create_vla_decl(filter_declaration(language, instance.element_type), instance.capacity) - elif isinstance(instance, pydsdl.ArrayType): + if isinstance(instance, pydsdl.ArrayType): if isinstance(instance.element_type, pydsdl.BooleanType): - return language.create_bitset_decl(filter_declaration(language, instance.element_type), instance.capacity) - else: - return language.create_array_decl(filter_declaration(language, instance.element_type), instance.capacity) - else: - return filter_full_reference_name(language, instance) + return language.create_bitset_decl(instance.capacity) + return language.create_array_decl(filter_declaration(language, instance.element_type), instance.capacity) + + return filter_full_reference_name(language, instance) @template_language_filter(__name__) @@ -1167,8 +1363,7 @@ def filter_to_namespace_qualifier(namespace_list: typing.List[str]) -> str: """ if namespace_list is None or len(namespace_list) == 0: return "" - else: - return "::".join(namespace_list) + "::" + return "::".join(namespace_list) + "::" def filter_to_template_unique_name(base_token: str) -> str: @@ -1232,7 +1427,7 @@ def filter_to_template_unique_name(base_token: str) -> str: else: adj_base_token = base_token - return UniqueNameGenerator.get_instance()("cpp", adj_base_token, "_", "_") + return UniqueNameGenerator.get_instance()("cpp", adj_base_token, "_", "_") # pylint: disable=not-callable def filter_as_boolean_value(value: bool) -> str: @@ -1337,7 +1532,7 @@ def filter_indent_if_not(language: Language, text: str, depth: int = 1) -> str: configured_indent = int(language.get_config_value("indent")) lines = text.splitlines(keepends=True) result = "" - for i in range(0, len(lines)): + for i, line in enumerate(lines): line = lines[i].lstrip() if len(line) == 0: # don't indent blank lines @@ -1417,11 +1612,11 @@ def filter_minimum_required_capacity_bits(t: pydsdl.SerializableType) -> int: @functools.lru_cache(3) -def _make_textwrap(width: int, initial_indent: str, subseqent_indent: str) -> textwrap.TextWrapper: +def _make_textwrap(width: int, initial_indent: str, subsequent_indent: str) -> textwrap.TextWrapper: return textwrap.TextWrapper( width=width, initial_indent=initial_indent, - subsequent_indent=subseqent_indent, + subsequent_indent=subsequent_indent, break_on_hyphens=True, break_long_words=False, replace_whitespace=False, @@ -1429,22 +1624,22 @@ def _make_textwrap(width: int, initial_indent: str, subseqent_indent: str) -> te def _make_block_comment(text: str, prefix: str, comment: str, suffix: str, indent: int, line_length: int) -> str: - doc_lines = text.splitlines() # type: typing.List[str] - indented_comment = "{}{}".format(" " * indent, comment) + doc_lines: typing.List[str] = text.splitlines() + indented_comment = f"{' ' * indent}{comment}" - commented_doc_lines = [] # type: typing.List[str] + commented_doc_lines: typing.List[str] = [] if len(doc_lines) > 0: if len(prefix) > 0: commented_doc_lines.append(prefix) else: commented_doc_lines.extend( - _make_textwrap(width=line_length, initial_indent=comment, subseqent_indent=indented_comment).wrap( + _make_textwrap(width=line_length, initial_indent=comment, subsequent_indent=indented_comment).wrap( doc_lines.pop(0) ) ) - tw = _make_textwrap(width=line_length, initial_indent=indented_comment, subseqent_indent=indented_comment) + tw = _make_textwrap(width=line_length, initial_indent=indented_comment, subsequent_indent=indented_comment) for docline in doc_lines: # The docs for textwrap.TextWrapper.wrap say: @@ -1453,7 +1648,7 @@ def _make_block_comment(text: str, prefix: str, comment: str, suffix: str, inden commented_doc_lines.extend(tw.wrap(docline) if docline.strip() else [indented_comment]) if len(suffix) > 0 and len(commented_doc_lines) > 0: - commented_doc_lines.append("{}{}".format(" " * indent, suffix)) + commented_doc_lines.append(f"{' ' * indent}{suffix}") return "\n".join(commented_doc_lines) @@ -1656,16 +1851,14 @@ def filter_block_comment(language: Language, text: str, style: str, indent: int """ - config_styles = language.get_config_value_as_dict( - "comment_styles" - ) # type: typing.Mapping[str, typing.Mapping[str, str]] + config_styles: typing.Mapping[str, typing.Mapping[str, str]] = language.get_config_value_as_dict("comment_styles") try: config_style = config_styles[style.lower()] - except KeyError: + except KeyError as ke: raise ValueError( - "{} is not a supported comment style. Supported is c, cpp, cpp-doxygen, and javadoc".format(style) - ) + f"{style} is not a supported comment style. Supported is c, cpp, cpp-doxygen, and javadoc" + ) from ke return _make_block_comment( text=text, diff --git a/src/nunavut/lang/cpp/support/__init__.py b/src/nunavut/lang/cpp/support/__init__.py index 9c7e4583..5900178f 100644 --- a/src/nunavut/lang/cpp/support/__init__.py +++ b/src/nunavut/lang/cpp/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains supporting C++ headers to distribute with generated types. diff --git a/src/nunavut/lang/cpp/templates/__init__.py b/src/nunavut/lang/cpp/templates/__init__.py index cbca9c40..f679535b 100644 --- a/src/nunavut/lang/cpp/templates/__init__.py +++ b/src/nunavut/lang/cpp/templates/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Contains the Jinja templates to generate C++ headers. diff --git a/src/nunavut/lang/cpp/templates/_composite_type.j2 b/src/nunavut/lang/cpp/templates/_composite_type.j2 index 8f0bd60c..63fbbe71 100644 --- a/src/nunavut/lang/cpp/templates/_composite_type.j2 +++ b/src/nunavut/lang/cpp/templates/_composite_type.j2 @@ -1,13 +1,13 @@ {#- - # Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. - # Copyright (C) 2021 OpenCyphal Development Team - # This software is distributed under the terms of the MIT License. + # Copyright (C) OpenCyphal Development Team + # Copyright Amazon.com Inc. or its affiliates. + # SPDX-License-Identifier: MIT -#} {%- from '_definitions.j2' import assert -%} {%- ifuses "std_variant" %} // +-------------------------------------------------------------------------------------------------------------------+ // | This implementation uses the C++17 standard library variant type with wrappers for the emplace and -// | get_if methods to support forward-compatibility with the C++14 version of this object. The union_value type +// | get_if methods to support backwards-compatibility with the C++14 version of this object. The union_value type // | extends std::variant and can be used with the entire set of variant methods. Using std::variant directly does mean // | your code will not be backwards compatible with the C++14 version of this object. // +-------------------------------------------------------------------------------------------------------------------+ @@ -33,7 +33,7 @@ struct {% if composite_type.deprecated -%} {%- endif -%} {{composite_type|short_reference_name}} final { -{%- if options.ctor_convention != ConstructorConvention.Default.value %} +{%- if options.ctor_convention != ConstructorConvention.DEFAULT %} using allocator_type = {{ options.allocator_type }}; {%- endif %} @@ -84,7 +84,7 @@ struct {% if composite_type.deprecated -%} {%- endif %} {%- endfor %} }; -{% if options.ctor_convention != ConstructorConvention.Default.value %} +{% if options.ctor_convention != ConstructorConvention.DEFAULT %} {%- if options.allocator_is_default_constructible %} // Default constructor {%- if composite_type.inner_type is UnionType %} @@ -107,7 +107,7 @@ struct {% if composite_type.deprecated -%} union_value{} // can't make use of the allocator with a union {%- else %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.AllocatorConstructor) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.ALLOCATOR_CONSTRUCTOR) }}{%if not loop.last %},{%endif %} {%- endfor %} {%- endif %} { @@ -117,7 +117,7 @@ struct {% if composite_type.deprecated -%} {%- if composite_type.inner_type is not UnionType %} {% if composite_type.fields_except_padding %} // Initializing constructor - {{ composite_type | explicit_decorator(SpecialMethod.InitializingConstructorWithAllocator)}}( + {{ composite_type | explicit_decorator(SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR)}}( {%- for field in composite_type.fields_except_padding %} const _traits_::TypeOf::{{ field | id }}& {{ field | id }}, {%- endfor %} @@ -125,7 +125,7 @@ struct {% if composite_type.deprecated -%} {%- if options.allocator_is_default_constructible %} = allocator_type(){% endif %}) {%- if composite_type.fields_except_padding %} :{% endif %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.InitializingConstructorWithAllocator) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.INITIALIZING_CONSTRUCTOR_WITH_ALLOCATOR) }}{%if not loop.last %},{%endif %} {%- endfor %} { (void)allocator; // avoid unused param warning @@ -143,7 +143,7 @@ struct {% if composite_type.deprecated -%} union_value{rhs.union_value} // can't make use of the allocator with a union {%- else %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.CopyConstructorWithAllocator) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.COPY_CONSTRUCTOR_WITH_ALLOCATOR) }}{%if not loop.last %},{%endif %} {%- endfor %} {% endif %} { @@ -161,7 +161,7 @@ struct {% if composite_type.deprecated -%} union_value{} // can't make use of the allocator with a union {%- else %} {%- for field in composite_type.fields_except_padding %} - {{ field | id }}{{ field | value_initializer(SpecialMethod.MoveConstructorWithAllocator) }}{%if not loop.last %},{%endif %} + {{ field | id }}{{ field | value_initializer(SpecialMethod.MOVE_CONSTRUCTOR_WITH_ALLOCATOR) }}{%if not loop.last %},{%endif %} {%- endfor %} {%- endif %} { diff --git a/src/nunavut/lang/cpp/templates/_fields.j2 b/src/nunavut/lang/cpp/templates/_fields.j2 index d9c02496..1d28b91f 100644 --- a/src/nunavut/lang/cpp/templates/_fields.j2 +++ b/src/nunavut/lang/cpp/templates/_fields.j2 @@ -10,7 +10,7 @@ // +----------------------------------------------------------------------+ {% endif -%} {{ field.doc | block_comment('cpp-doxygen', 4, 120) }} - {% if options.ctor_convention != ConstructorConvention.Default.value -%} + {% if options.ctor_convention != ConstructorConvention.DEFAULT -%} _traits_::TypeOf::{{field.name|id}} {{ field | id }}; {%- else -%} _traits_::TypeOf::{{field.name|id}} {{ field | id }}{{ field.data_type | default_value_initializer }}; diff --git a/src/nunavut/lang/cpp/templates/base.j2 b/src/nunavut/lang/cpp/templates/base.j2 index abfbddfb..66988bba 100644 --- a/src/nunavut/lang/cpp/templates/base.j2 +++ b/src/nunavut/lang/cpp/templates/base.j2 @@ -1,45 +1,53 @@ {#- - # Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. - # Copyright (C) 2021 OpenCyphal Development Team - # This software is distributed under the terms of the MIT License. + # Copyright (C) OpenCyphal Development Team + # Copyright Amazon.com Inc. or its affiliates. + # SPDX-License-Identifier: MIT -#} // // This is an AUTO-GENERATED Cyphal DSDL data type implementation. Curious? See https://opencyphal.org. // You shouldn't attempt to edit this file. +{%- if nunavut.embed_auditing_info %} // // Checking this file under version control is not recommended since metadata in this header will change for each -// build invocation. TODO: add --reproducible option to prevent any volatile metadata from being generated. +// build invocation (do not use --embed-auditing-info option to remove this comment). +{%- endif %} +// +// Generator : nunavut-{{ nunavut.version }} (serialization was {{ 'not ' * nunavut.support.omit }}enabled) +{%- if nunavut.embed_auditing_info %} +// Source file : {{ T.source_file_path.as_posix() }} +// Generated at : {{ now_utc }} UTC +{%- else %} +// Source file : {{ T.source_file_path.name }} +{%- endif %} +// Is deprecated : {{ T.deprecated and 'yes' or 'no' }} +// Fixed port-ID : {{ T.fixed_port_id }} +// Full name : {{ T.full_name }} +// Type Version : {{ T.version.major }}.{{ T.version.minor }} // -// Generator: nunavut-{{ nunavut.version }} (serialization was {{ 'not ' * nunavut.support.omit }}enabled) -// Source file: {{ T.source_file_path.as_posix() }} -// Generated at: {{ now_utc }} UTC -// Is deprecated: {{ T.deprecated and 'yes' or 'no' }} -// Fixed port-ID: {{ T.fixed_port_id }} -// Full name: {{ T.full_name }} -// Type Version: {{ T.version.major }}.{{ T.version.minor }} // Support {%- if nunavut.support.omit %} // (support file generation disabled) {%- else %} -// Support Namespace: {{ nunavut.support.namespace }} -// Support Version: {{ nunavut.support.version }} +// Support Namespace : {{ nunavut.support.namespace }} +// Support Version : {{ nunavut.support.version }} {%- endif %} {%- for template_set in nunavut.template_sets %} +// // Template Set ({{ template_set[0] }}) -// priority: {{ loop.index0 }} -// package: {{ template_set[1] }} -// version: {{ template_set[2] }} +// priority : {{ loop.index0 }} +// package : {{ template_set[1] }} +// version : {{ template_set[2] }} {%- endfor %} +// // Platform -{%- for key, value in nunavut.platform_version.items() %} -// {{ key }}: {{ value }} -{%- endfor %} +{{ nunavut.platform_version | text_table("// ") }} +// // Language Options -{%- for key, value in options.items() %} -// {{ key }}: {{ value }} -{%- endfor %} +{{ options | text_table("// ") }} +// // Uses Language Features // Uses std_variant: {%- ifuses "std_variant" -%}yes{%- else -%}no{%- endifuses -%} +// {%- if T.deprecated and options.std | int < 14 %} {#- Courtesy http://patorjk.com/software/taag/#p=display&f=Big&t=DEPRECATED #} {#- [[deprecated]] becomes available in c++14 #} diff --git a/src/nunavut/lang/html/__init__.py b/src/nunavut/lang/html/__init__.py index 2bb4894d..982093ae 100644 --- a/src/nunavut/lang/html/__init__.py +++ b/src/nunavut/lang/html/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating docs. All filters in this @@ -24,20 +24,29 @@ def filter_extent(instance: pydsdl.Any) -> int: + """ + Filter that returns the dsdl extend property of a given type. + """ try: return instance.extent or 0 except TypeError as e: - raise TemplateAssertionError(e) + raise TemplateAssertionError(e) from None def filter_max_bit_length(instance: pydsdl.Any) -> int: + """ + Filter that returns the dsdl max bit length property of a given type. + """ try: return instance.bit_length_set.max or 0 except TypeError as e: - raise TemplateAssertionError(e) + raise TemplateAssertionError(e) from None def filter_tag_id(instance: pydsdl.Any) -> str: + """ + Emit a tag id for a given type. + """ if isinstance(instance, pydsdl.ArrayType): return "{}_array".format(str(instance.element_type).replace(".", "_").replace(" ", "_")) else: @@ -49,6 +58,9 @@ def filter_tag_id(instance: pydsdl.Any) -> str: def filter_url_from_type(instance: pydsdl.Any) -> str: + """ + Emit a path to the documentation for a given type. + """ root_ns = instance.root_namespace tag_id = "{}_{}_{}".format(instance.full_name.replace(".", "_"), instance.version[0], instance.version[1]) return "../{}/#{}".format(root_ns, tag_id) @@ -121,6 +133,9 @@ def filter_make_unique(_: typing.Any, base_token: str) -> str: def filter_namespace_doc(ns: nunavut.Namespace) -> str: + """ + Generate HTML documentation for a namespace. + """ result = "" for t, _ in ns.get_nested_types(): if t.short_name == "_": @@ -130,6 +145,9 @@ def filter_namespace_doc(ns: nunavut.Namespace) -> str: def filter_display_type(instance: pydsdl.Any) -> str: + """ + Deprecated. Don't use this filter. Needs refactoring. + """ # TODO: this whole thing needs to be in the template. if isinstance(instance, pydsdl.FixedLengthArrayType): capacity = '[{}]'.format(instance.capacity) diff --git a/src/nunavut/lang/html/support/__init__.py b/src/nunavut/lang/html/support/__init__.py index f5c35ff9..e4c95a6b 100644 --- a/src/nunavut/lang/html/support/__init__.py +++ b/src/nunavut/lang/html/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Empty python package to ensure the support generator doesn't explode. @@ -16,4 +16,9 @@ def list_support_files(resource_type: ResourceType = ResourceType.ANY) -> typing.Generator[pathlib.Path, None, None]: + """ + Get a list of HTML support files embedded in this package. + :param resource_type: A type of support file to list. + """ + # pylint: disable=unused-argument return empty_list_support_files() diff --git a/src/nunavut/lang/js/__init__.py b/src/nunavut/lang/js/__init__.py index 10ddf3b8..482de62a 100644 --- a/src/nunavut/lang/js/__init__.py +++ b/src/nunavut/lang/js/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating javascript. All filters in this diff --git a/src/nunavut/lang/js/support/__init__.py b/src/nunavut/lang/js/support/__init__.py index 9f0dfc3c..8dae41c0 100644 --- a/src/nunavut/lang/js/support/__init__.py +++ b/src/nunavut/lang/js/support/__init__.py @@ -1,7 +1,7 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Empty python package to ensure the support generator doesn't explode. @@ -16,4 +16,9 @@ def list_support_files(resource_type: ResourceType = ResourceType.ANY) -> typing.Generator[pathlib.Path, None, None]: + """ + Get a list of javascript support files embedded in this package. + :param resource_type: A type of support file to list. + """ + # pylint: disable=unused-argument return empty_list_support_files() diff --git a/src/nunavut/lang/properties.yaml b/src/nunavut/lang/properties.yaml index 07243fce..f42a02a0 100644 --- a/src/nunavut/lang/properties.yaml +++ b/src/nunavut/lang/properties.yaml @@ -1,8 +1,8 @@ %YAML 1.2 # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # --- nunavut.lang.c: @@ -194,6 +194,7 @@ nunavut.lang.c: - defined - __has_include - __has_cpp_attribute + - pascal token_encoding_rules_by_identifier_type: all: - '\s+' @@ -311,6 +312,7 @@ nunavut.lang.cpp: enable_serialization_asserts: false enable_override_variable_array_capacity: false std: c++14 + std_flavor: std cast_format: "static_cast<{type}>({value})" # Provide non-empty values to override the type used for variable-length arrays in C++ types. variable_array_type_include: "" @@ -320,22 +322,27 @@ nunavut.lang.cpp: allocator_type: "" allocator_is_default_constructible: true ctor_convention: "default" - cetl++14-17_options: - variable_array_type_include: '"cetl/variable_length_array.hpp"' - variable_array_type_template: "cetl::VariableLengthArray<{TYPE}, {REBIND_ALLOCATOR}>" - variable_array_type_constructor_args: "{MAX_SIZE}" - allocator_include: '"cetl/pf17/sys/memory_resource.hpp"' - allocator_type: "cetl::pf17::pmr::polymorphic_allocator" - allocator_is_default_constructible: false - ctor_convention: "uses-trailing-allocator" - c++17-pmr_options: - variable_array_type_include: "" - variable_array_type_template: "std::vector<{TYPE}, {REBIND_ALLOCATOR}>" - variable_array_type_constructor_args: "" - allocator_include: "" - allocator_type: "std::pmr::polymorphic_allocator" - allocator_is_default_constructible: true - ctor_convention: "uses-trailing-allocator" + defaults: + cetl++14-17: + std: c++14 + std_flavor: cetl + variable_array_type_include: '"cetl/variable_length_array.hpp"' + variable_array_type_template: "cetl::VariableLengthArray<{TYPE}, {REBIND_ALLOCATOR}>" + variable_array_type_constructor_args: "{MAX_SIZE}" + allocator_include: '"cetl/pf17/sys/memory_resource.hpp"' + allocator_type: "cetl::pf17::pmr::polymorphic_allocator" + allocator_is_default_constructible: false + ctor_convention: "uses-trailing-allocator" + c++17-pmr: + std: c++17 + std_flavor: pmr + variable_array_type_include: "" + variable_array_type_template: "std::vector<{TYPE}, {REBIND_ALLOCATOR}>" + variable_array_type_constructor_args: "" + allocator_include: "" + allocator_type: "std::pmr::polymorphic_allocator" + allocator_is_default_constructible: true + ctor_convention: "uses-trailing-allocator" nunavut.lang.py: diff --git a/src/nunavut/lang/py/__init__.py b/src/nunavut/lang/py/__init__.py index 5b7b9e69..4624fbf6 100644 --- a/src/nunavut/lang/py/__init__.py +++ b/src/nunavut/lang/py/__init__.py @@ -1,21 +1,22 @@ # -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2021 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # """ Filters for generating python. All filters in this module will be available in the template's global namespace as ``py``. """ from __future__ import annotations + +import base64 import builtins import functools -import keyword -import base64 import gzip -import pickle import itertools -from typing import Any, Iterable +import keyword +import pickle +from typing import Any, Dict, Iterable import pydsdl @@ -27,6 +28,7 @@ template_language_int_filter, template_language_list_filter, ) +from nunavut._utilities import cached_property from nunavut.lang import Language as BaseLanguage from nunavut.lang._common import TokenEncoder, UniqueNameGenerator @@ -38,12 +40,13 @@ class Language(BaseLanguage): PYTHON_RESERVED_IDENTIFIERS: list[str] = sorted(list(map(str, list(keyword.kwlist) + dir(builtins)))) - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self._language_options["enable_serialization_asserts"] = True + def _validate_language_options(self, defaults: Dict[str, Any], options: Dict[str, Any]) -> Dict[str, Any]: + # pylint: disable=unused-argument + options["enable_serialization_asserts"] = True # always enable serialization asserts for python + return options - @functools.lru_cache(maxsize=None) - def _get_token_encoder(self) -> TokenEncoder: + @cached_property + def _token_encoder(self) -> TokenEncoder: """ Caching getter to ensure we don't have to recompile TokenEncoders for each filter invocation. """ @@ -56,11 +59,12 @@ def get_includes(self, dep_types: Dependencies) -> list[str]: def filter_id(self, instance: Any, id_type: str = "any") -> str: raw_name = self.default_filter_id_for_target(instance) - return self._get_token_encoder().strop(raw_name, id_type) + return self._token_encoder.strop(raw_name, id_type) @template_context_filter def filter_to_template_unique_name(context: SupportsTemplateContext, base_token: str) -> str: + # pylint: disable=unused-argument """ Filter that takes a base token and forms a name that is very likely to be unique within the template the filter is invoked. This diff --git a/src/nunavut/lang/py/templates/base.j2 b/src/nunavut/lang/py/templates/base.j2 index d0592c22..7e8d2dea 100644 --- a/src/nunavut/lang/py/templates/base.j2 +++ b/src/nunavut/lang/py/templates/base.j2 @@ -5,10 +5,15 @@ -#} # AUTOGENERATED, DO NOT EDIT. # +{%- if nunavut.embed_auditing_info %} # Source file: # {{ T.source_file_path }} # # Generated at: {{ now_utc }} UTC +{%- else %} +# Source file: +# {{ T.source_file_path.name }} +{%- endif %} # Is deprecated: {{ T.deprecated and 'yes' or 'no' }} # Fixed port ID: {{ T.fixed_port_id }} # Full name: {{ T.full_name }} diff --git a/test/gentest_dsdl/test_dsdl.py b/test/gentest_dsdl/test_dsdl.py index 2d7033d3..49c34b4d 100644 --- a/test/gentest_dsdl/test_dsdl.py +++ b/test/gentest_dsdl/test_dsdl.py @@ -18,7 +18,7 @@ ("cpp", True, {"std": "c++17"}), ("c", False, {}), ("c", True, {}), - ("py", False, {}), + ("py", True, {}), ("html", False, {}), ], ) diff --git a/test/gentest_namespaces/test_namespaces.py b/test/gentest_namespaces/test_namespaces.py index 5bc120f4..b4900b90 100644 --- a/test/gentest_namespaces/test_namespaces.py +++ b/test/gentest_namespaces/test_namespaces.py @@ -9,24 +9,30 @@ from pathlib import Path import pytest +from pydsdl import Any, CompositeType, read_namespace + from nunavut import Namespace, build_namespace_tree from nunavut._utilities import YesNoDefault from nunavut.jinja import DSDLCodeGenerator from nunavut.lang import Language, LanguageContext, LanguageContextBuilder -from pydsdl import Any, CompositeType, read_namespace class DummyType(Any): """Fake dsdl 'any' type for testing.""" def __init__(self, namespace: str = "uavcan", name: str = "Dummy"): - self._full_name = "{}.{}".format(namespace, name) + self._full_name = f"{namespace}.{name}" # +-----------------------------------------------------------------------+ - # | DUCK TYPEING: CompositeType + # | DUCK TYPING: CompositeType # +-----------------------------------------------------------------------+ @property def full_name(self) -> str: + """ + Returns the full name of the object. + + :return: The full name as a string. + """ return self._full_name # +-----------------------------------------------------------------------+ @@ -36,8 +42,7 @@ def full_name(self) -> str: def __eq__(self, other: object) -> bool: if isinstance(other, DummyType): return self._full_name == other._full_name - else: - return False + return False def __str__(self) -> str: return self.full_name @@ -49,6 +54,17 @@ def __hash__(self) -> int: def gen_test_namespace( gen_paths: typing.Any, language_context: LanguageContext ) -> typing.Tuple[Namespace, str, typing.List[CompositeType]]: + """ + Generate a test namespace. + + :param gen_paths (typing.Any): The paths for generating the namespace. + :param language_context (LanguageContext): The language context for generating the namespace. + + :return (typing.Tuple[Namespace, str, typing.List[CompositeType]]): + A tuple containing the generated namespace, + the root namespace path, and a list of composite types. + + """ root_namespace_path = str(gen_paths.dsdl_dir / Path("scotec")) includes = [str(gen_paths.dsdl_dir / Path("uavcan"))] compound_types = read_namespace(root_namespace_path, includes, allow_unregulated_fixed_port_id=True) @@ -85,7 +101,7 @@ def test_get_all_types(gen_paths): # type: ignore """Verify the get_all_namespaces method in Namespace""" language_context = LanguageContextBuilder(include_experimental_languages=True).set_target_language("js").create() namespace, _, _ = gen_test_namespace(gen_paths, language_context) - index = dict() + index = {} for ns, path in namespace.get_all_types(): index[path] = ns @@ -140,7 +156,7 @@ def test_namespace_namespace_template(gen_paths): # type: ignore def test_namespace_generation(gen_paths): # type: ignore - """Test actually generating a namepace file.""" + """Test actually generating a namespace file.""" language_context = ( LanguageContextBuilder(include_experimental_languages=True) .set_target_language("js") @@ -162,7 +178,7 @@ def test_namespace_generation(gen_paths): # type: ignore assert outfile is not None - with open(str(outfile), "r") as json_file: + with open(str(outfile), "r", encoding="utf-8") as json_file: json_blob = json.load(json_file) assert json_blob is not None @@ -180,34 +196,38 @@ def test_build_namespace_tree_from_nothing(gen_paths): # type: ignore @pytest.mark.parametrize( - "language_key,expected_file_ext,expected_stropp_part_0,expected_stropp_part_1", + "language_key,expected_file_ext,expected_strop_part_0,expected_strop_part_1", [("c", ".h", "_typedef", "str"), ("py", ".py", "typedef", "str_")], ) # type: ignore def test_namespace_stropping( - gen_paths, language_key, expected_file_ext, expected_stropp_part_0, expected_stropp_part_1 + gen_paths, + language_key: str, + expected_file_ext: str, + expected_strop_part_0: str, + expected_strop_part_1: str, ): """Test generating a namespace that uses a reserved keyword for a given language.""" language_context = ( LanguageContextBuilder(include_experimental_languages=True).set_target_language(language_key).create() ) - namespace, root_namespace_path, compound_types = gen_test_namespace(gen_paths, language_context) + namespace, _, compound_types = gen_test_namespace(gen_paths, language_context) assert len(compound_types) == 2 generator = DSDLCodeGenerator( namespace, generate_namespace_types=YesNoDefault.YES, templates_dir=gen_paths.templates_dir / Path("default") ) generator.generate_all() - expected_stropped_ns = "scotec.{}.{}".format(expected_stropp_part_0, expected_stropp_part_1) + expected_stropped_ns = f"scotec.{expected_strop_part_0}.{expected_strop_part_1}" outfile = gen_paths.find_outfile_in_namespace(expected_stropped_ns, namespace) assert outfile is not None - with open(str(outfile), "r") as json_file: + with open(str(outfile), "r", encoding="utf-8") as json_file: json_blob = json.load(json_file) assert json_blob is not None output_path_for_stropped = namespace.find_output_path_for_type(compound_types[1]) expected_stable_path = gen_paths.out_dir / "scotec" - expected_path_and_file = expected_stable_path / expected_stropp_part_0 / expected_stropp_part_1 / "ATOMIC_TYPE_0_1" + expected_path_and_file = expected_stable_path / expected_strop_part_0 / expected_strop_part_1 / "ATOMIC_TYPE_0_1" assert expected_path_and_file.with_suffix(expected_file_ext) == output_path_for_stropped diff --git a/test/gentest_nnvg/test_nnvg.py b/test/gentest_nnvg/test_nnvg.py index 0cfeeeb5..8197781f 100644 --- a/test/gentest_nnvg/test_nnvg.py +++ b/test/gentest_nnvg/test_nnvg.py @@ -1,8 +1,9 @@ # -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2019 OpenCyphal Development Team -# This software is distributed under the terms of the MIT License. +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT # +# cSpell:ignore scotec, herringtec import json import os import pathlib @@ -19,7 +20,7 @@ @pytest.mark.parametrize("env_var_name", ["DSDL_INCLUDE_PATH"]) def test_DSDL_INCLUDE_PATH(gen_paths: typing.Any, run_nnvg: typing.Callable, env_var_name: str) -> None: """ - Verify that supported environment variables are used by nnvg. + Verify that the DSDL_INCLUDE_PATH environment variable and any aliases are used by nnvg. """ nnvg_args0 = [ @@ -38,7 +39,7 @@ def test_DSDL_INCLUDE_PATH(gen_paths: typing.Any, run_nnvg: typing.Callable, env scotec_path = (gen_paths.dsdl_dir / pathlib.Path("scotec")).as_posix() herringtec_path = (gen_paths.dsdl_dir / pathlib.Path("herringtec")).as_posix() - env = {env_var_name: "{}{}{}".format(herringtec_path, os.pathsep, scotec_path)} + env = {env_var_name: f"{herringtec_path}{os.pathsep}{scotec_path}"} run_nnvg(gen_paths, nnvg_args0, env=env) @@ -94,7 +95,7 @@ def test_list_inputs(gen_paths: typing.Any, run_nnvg: typing.Callable, generate_ (gen_paths.dsdl_dir / pathlib.Path("scotec")).as_posix(), "--list-inputs", (gen_paths.dsdl_dir / pathlib.Path("uavcan")).as_posix(), - "--generate-support={}".format(generate_support), + f"--generate-support={generate_support}", ] if generate_support == "only": diff --git a/test/gettest_properties/test_properties.py b/test/gettest_properties/test_properties.py index 9551ef2f..2d9c36c3 100644 --- a/test/gettest_properties/test_properties.py +++ b/test/gettest_properties/test_properties.py @@ -1,16 +1,15 @@ # -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Copyright (C) 2018-2020 OpenCyphal Development Team +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright (C) 2018-2021 OpenCyphal Development Team # This software is distributed under the terms of the MIT License. # import pathlib +import re from pathlib import Path import pydsdl -import pytest import yaml -import re from nunavut import build_namespace_tree from nunavut.jinja import DSDLCodeGenerator @@ -30,7 +29,7 @@ def test_issue_277(gen_paths): # type: ignore "allocator_include": '"MyCrazyAllocator.hpp"', "allocator_type": "MyCrazyAllocator", "allocator_is_default_constructible": True, - "ctor_convention": "uses-leading-allocator" + "ctor_convention": "uses-leading-allocator", } vla_decl_pattern = re.compile(r"\b|^MyCrazyArray\B") @@ -44,7 +43,7 @@ def test_issue_277(gen_paths): # type: ignore LanguageClassLoader.to_language_module_name("cpp"): {Language.WKCV_LANGUAGE_OPTIONS: override_language_options} } - with open(overrides_file, "w") as overrides_handle: + with open(overrides_file, "w", encoding="utf-8") as overrides_handle: yaml.dump(overrides_data, overrides_handle) root_namespace = str(gen_paths.dsdl_dir / Path("proptest")) @@ -52,7 +51,7 @@ def test_issue_277(gen_paths): # type: ignore language_context = ( LanguageContextBuilder(include_experimental_languages=True) .set_target_language("cpp") - .set_additional_config_files([overrides_file]) + .add_config_files(overrides_file) .create() ) namespace = build_namespace_tree(compound_types, root_namespace, gen_paths.out_dir, language_context) @@ -64,12 +63,11 @@ def test_issue_277(gen_paths): # type: ignore assert outfile is not None - found_vla_decl = False found_vla_include = False found_alloc_include = False found_vla_constructor_args = False - with open(str(outfile), "r") as header_file: + with open(str(outfile), "r", encoding="utf-8") as header_file: for line in header_file: if not found_vla_decl and vla_decl_pattern.search(line): found_vla_decl = True diff --git a/tox.ini b/tox.ini index 4686c7c3..52e005ea 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,13 @@ # The standard version to develop against is 3.10. # [tox] -envlist = {py37,py38,py39,py310,py311}-{test,nnvg,doctest,rstdoctest},lint,report,docs +envlist = {py37,py38,py39,py310,py311,py312}-{test,nnvg,doctest,rstdoctest},lint,report,docs [base] deps = - Sybil >= 3.0.0, < 4.0.0 - pytest >= 5.4.3, < 7.0.0 + Sybil + pytest pytest-timeout coverage types-PyYAML @@ -19,19 +19,21 @@ deps = autopep8 rope isort + nox # +---------------------------------------------------------------------------+ # | CONFIGURATION # +---------------------------------------------------------------------------+ -[flake8] -max-complexity = 10 +[pylint] max-line-length = 120 -doctests = True -statistics = True -show-source = True -ignore = E203, W503 +max-args = 8 +max-attributes = 12 +ignore-paths = .*/(jinja2|markupsafe)/.* +min-public-methods = 0 +source-roots = src +disable = no-else-return,invalid-name [pytest] @@ -39,15 +41,11 @@ log_file = pytest.log log_level = DEBUG log_cli = true log_cli_level = WARNING -addopts: --keep-generated norecursedirs = submodules .* build* verification .tox -# The fill fixtures deprecation warning comes from Sybil, which we don't have any control over. Remove when updated. -filterwarnings = - error - ignore:A private pytest class or function was used.:DeprecationWarning - +addopts = -p no:doctest [coverage:run] +data_file = build/coverage-py/.coverage branch=True parallel=True include = @@ -70,7 +68,7 @@ source = [coverage:report] -exclude_lines = +exclude_also = pragma: no cover def __repr__ raise AssertionError @@ -78,7 +76,7 @@ exclude_lines = assert False if False: if __name__ == .__main__.: - +omit = *.j2 [doc8] max-line-length = 120 @@ -96,6 +94,7 @@ warn_redundant_casts = True warn_unused_ignores = True show_error_context = True mypy_path = src +exclude = (jinja2|markupsafe) [mypy-pydsdl] ignore_missing_imports = True @@ -140,8 +139,8 @@ commands = nnvg: -O {envtmpdir} \ nnvg: --target-language cpp \ nnvg: --experimental-languages \ + nnvg: --language-standard c++17-pmr \ nnvg: -v \ - nnvg: --dry-run \ nnvg: {toxinidir}/submodules/public_regulated_data_types/uavcan test: coverage run \ @@ -163,34 +162,13 @@ commands = [testenv:docs] deps = -rrequirements.txt - sphinx ~= 6.2.1 + sphinx sphinx-rtd-theme readthedocs-sphinx-ext commands = sphinx-build -W -b html {toxinidir} {envtmpdir} -[testenv:gen-apidoc] -allowlist_externals = rm -deps = - sphinx-autoapi - -commands = - rm -rf {toxinidir}/docs/api - sphinx-apidoc \ - --doc-project library \ - --output-dir {toxinidir}/docs/api \ - --ext-autodoc \ - --ext-intersphinx \ - --templatedir={toxinidir}/docs/sphinx_templates/apidoc \ - --tocfile=library \ - --module-first \ - src \ - "**/conftest.py" \ - "src/nunavut/jinja/jinja2/**" \ - "src/nunavut/jinja/markupsafe/**" - - [testenv:report] deps = coverage skip_install = true @@ -205,7 +183,7 @@ basepython = python3.10 deps = {[dev]deps} black - flake8 + pylint doc8 Pygments mypy @@ -214,15 +192,19 @@ deps = types-PyYAML commands = - flake8 --benchmark --tee --output-file={envtmpdir}/flake8.txt --filename=*.py --exclude=**/jinja2/*,**/markupsafe/* src + pylint --reports=y \ + --rcfile={toxinidir}/tox.ini \ + --output={envtmpdir}/pylint.txt \ + --output-format=json2 \ + --clear-cache-post-run=y \ + --confidence=HIGH \ + {toxinidir}/src/nunavut black --check --line-length 120 --force-exclude '(/jinja2/|/markupsafe\/)' src doc8 {toxinidir}/docs - mypy -m nunavut \ - -m nunavut.jinja \ - -p nunavut.lang \ - --cache-dir {envtmpdir} \ - --txt-report {envtmpdir}/mypy-report-lib \ - --config-file {toxinidir}/tox.ini + mypy -p nunavut \ + --cache-dir {envtmpdir} \ + --txt-report {envtmpdir}/mypy-report-lib \ + --config-file {toxinidir}/tox.ini [testenv:package] @@ -247,9 +229,4 @@ deps = {[testenv:docs]deps} {[testenv:lint]deps} commands = - mypy -m nunavut \ - -m nunavut.jinja \ - -p nunavut.lang \ - --config-file {toxinidir}/tox.ini \ - --install-types \ - --non-interactive + python --version diff --git a/verification/.devcontainer/devcontainer.json b/verification/.devcontainer/devcontainer.json index b2de9e38..533a16a2 100644 --- a/verification/.devcontainer/devcontainer.json +++ b/verification/.devcontainer/devcontainer.json @@ -20,8 +20,7 @@ "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", - "ms-python.flake8", - "ms-python.autopep8" + "ms-python.pylint" ] } }, diff --git a/verification/CMakeLists.txt b/verification/CMakeLists.txt index a970e48e..63fbfcd6 100644 --- a/verification/CMakeLists.txt +++ b/verification/CMakeLists.txt @@ -285,7 +285,7 @@ endif() if(NUNAVUT_VERIFICATION_LANG_STANDARD STREQUAL "cetl++14-17" OR NUNAVUT_VERIFICATION_LANG_STANDARD STREQUAL "c++17-pmr") # - # Generate serialization support headers with config matching the langauge standard + # Generate serialization support headers with config matching the language standard # create_dsdl_target(nunavut-support-array-with-allocator ${NUNAVUT_VERIFICATION_LANG} @@ -301,7 +301,7 @@ if(NUNAVUT_VERIFICATION_LANG_STANDARD STREQUAL "cetl++14-17" OR NUNAVUT_VERIFICA "only") # - # Generate additional types with config matching the langauge standard + # Generate additional types with config matching the language standard # create_dsdl_target(dsdl-test-array-with-allocator ${NUNAVUT_VERIFICATION_LANG}