From 8e75de55b4063f41b3e87900de9deca3b5027b6b Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen <43357585+CasperWA@users.noreply.github.com> Date: Fri, 23 Feb 2024 09:20:37 +0100 Subject: [PATCH] Update documentation preparation (#209) Update documentation handling. Use SINTEF/ci-cd pre-commit hooks. Remove the invoke tasks. Make the new link in the README relative in the docs. Update strategies in documentation. Ensure everything is shown, even if it has no explicit doc-string by updating options for the documentation file that showcases all the strategies. --- .github/workflows/ci_cd_updated_master.yml | 8 +- .github/workflows/ci_tests.yml | 6 +- .pre-commit-config.yaml | 29 ++-- README.md | 3 +- docs/all_strategies.md | 3 + docs/api_reference/models/session.md | 2 +- docs/api_reference/strategies/convert.md | 2 + docs/api_reference/strategies/filter.md | 2 + docs/api_reference/strategies/generate.md | 2 + docs/api_reference/strategies/mapping.md | 2 + docs/api_reference/strategies/parse.md | 2 + docs/api_reference/strategies/parse_excel.md | 2 + docs/api_reference/strategies/parse_image.md | 2 + docs/api_reference/strategies/serialise.md | 2 + docs/index.md | 3 +- mkdocs.yml | 3 - tasks.py | 145 ------------------- 17 files changed, 44 insertions(+), 174 deletions(-) diff --git a/.github/workflows/ci_cd_updated_master.yml b/.github/workflows/ci_cd_updated_master.yml index 97e77b5a..8bbf4a7d 100644 --- a/.github/workflows/ci_cd_updated_master.yml +++ b/.github/workflows/ci_cd_updated_master.yml @@ -104,16 +104,16 @@ jobs: - name: Check API Reference and landing page if: env.RELEASE_RUN == 'false' run: | - invoke create-api-reference-docs --pre-clean - invoke create-docs-index + pre-commit run --all-files docs-api-reference + pre-commit run --all-files docs-landing-page if [ -n "$(git status --porcelain docs/api_reference docs/index.md)" ]; then echo -e "\u274c Discrepancies found !" echo -e "The following files in the documentation must be committed:" git status --porcelain docs/api_reference docs/index.md echo -e "\nRun:\n" - echo " invoke create-api-reference-docs --pre-clean" - echo -e " invoke create-docs-index\n" + echo " pre-commit run --all-files docs-api-reference" + echo -e " pre-commit run --all-files docs-landing-page\n" echo "And commit the changed files." exit 1 else diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index cb04b9d9..69c160a8 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -196,11 +196,11 @@ jobs: run: | python -m pip install -U pip pip install -U setuptools wheel - pip install -U -r requirements_docs.txt + pip install -U -r requirements_docs.txt -r requirements_dev.txt pip install -e . - name: Build run: | - invoke create-api-reference-docs --pre-clean - invoke create-docs-index + pre-commit run --all-files docs-api-reference + pre-commit run --all-files docs-landing-page mkdocs build --strict diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6bb9edd1..28714d25 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,19 @@ repos: - id: mypy exclude: ^tests/.*$ + - repo: https://github.com/SINTEF/ci-cd + rev: v2.7.2 + hooks: + - id: docs-api-reference + args: + - --package-dir=oteapi_dlite + - --full-docs-folder=models + - --full-docs-folder=strategies + - id: docs-landing-page + args: + - --replacement=(LICENSE),(LICENSE.md) + - --replacement=https://EMMC-ASBL.github.io/oteapi-dlite/latest/, + - repo: local hooks: - id: pylint @@ -67,19 +80,3 @@ repos: types: [python] require_serial: true files: ^tests/.*$ - - id: update-docs-api-reference - name: Update API Reference in Documentation - entry: invoke - args: [create-api-reference-docs, --pre-clean, --pre-commit] - language: python - pass_filenames: false - files: ^oteapi_dlite/.*\.py$ - description: Update the API Reference documentation, whenever a Python file is touched in the code base. - - id: update-docs-index - name: Update Landing Page for Documentation - entry: invoke - args: [create-docs-index] - language: python - pass_filenames: false - files: ^README.md$ - description: Update the landing page for the documentation if the source file (README.md) is changed. diff --git a/README.md b/README.md index 357b47ba..2f26260f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ [![DOI](https://zenodo.org/badge/455070691.svg)](https://zenodo.org/badge/latestdoi/455070691) - # OTEAPI DLite Plugin All strategies implemented in this plugin (except for `create_collection`) assumes that you have the UUID of a DLite collection with the key `collection_id` in the `session`: @@ -21,6 +20,8 @@ Further reading: - [OTE-API Services Documentation](https://github.com/EMMC-ASBL/oteapi-services) - [DLite](https://github.com/SINTEF/dlite) +A list and documentation for all the strategies in this plugin can be found [here](https://EMMC-ASBL.github.io/oteapi-dlite/latest/all_strategies). + ## License and copyright The OTEAPI DLite Plugin is released under the [MIT license](LICENSE) with copyright © SINTEF. diff --git a/docs/all_strategies.md b/docs/all_strategies.md index 33bcd9bb..13e60eda 100644 --- a/docs/all_strategies.md +++ b/docs/all_strategies.md @@ -5,3 +5,6 @@ This page provides documentation for the `oteapi_dlite.strategies` submodule, wh These strategies will be available when setting up a server in an environment with oteapi-dlite installed. ::: oteapi_dlite.strategies + options: + show_submodules: true + show_if_no_docstring: true diff --git a/docs/api_reference/models/session.md b/docs/api_reference/models/session.md index 87449dc4..5b56f9b8 100644 --- a/docs/api_reference/models/session.md +++ b/docs/api_reference/models/session.md @@ -1,5 +1,5 @@ # session ::: oteapi_dlite.models.session - rendering: + options: show_if_no_docstring: true diff --git a/docs/api_reference/strategies/convert.md b/docs/api_reference/strategies/convert.md index ebffb8b1..b938a010 100644 --- a/docs/api_reference/strategies/convert.md +++ b/docs/api_reference/strategies/convert.md @@ -1,3 +1,5 @@ # convert ::: oteapi_dlite.strategies.convert + options: + show_if_no_docstring: true diff --git a/docs/api_reference/strategies/filter.md b/docs/api_reference/strategies/filter.md index 35f18295..fca8a482 100644 --- a/docs/api_reference/strategies/filter.md +++ b/docs/api_reference/strategies/filter.md @@ -1,3 +1,5 @@ # filter ::: oteapi_dlite.strategies.filter + options: + show_if_no_docstring: true diff --git a/docs/api_reference/strategies/generate.md b/docs/api_reference/strategies/generate.md index bfb698c1..2c555359 100644 --- a/docs/api_reference/strategies/generate.md +++ b/docs/api_reference/strategies/generate.md @@ -1,3 +1,5 @@ # generate ::: oteapi_dlite.strategies.generate + options: + show_if_no_docstring: true diff --git a/docs/api_reference/strategies/mapping.md b/docs/api_reference/strategies/mapping.md index ba19a443..1f4ec563 100644 --- a/docs/api_reference/strategies/mapping.md +++ b/docs/api_reference/strategies/mapping.md @@ -1,3 +1,5 @@ # mapping ::: oteapi_dlite.strategies.mapping + options: + show_if_no_docstring: true diff --git a/docs/api_reference/strategies/parse.md b/docs/api_reference/strategies/parse.md index 28760162..3e0ced35 100644 --- a/docs/api_reference/strategies/parse.md +++ b/docs/api_reference/strategies/parse.md @@ -1,3 +1,5 @@ # parse ::: oteapi_dlite.strategies.parse + options: + show_if_no_docstring: true diff --git a/docs/api_reference/strategies/parse_excel.md b/docs/api_reference/strategies/parse_excel.md index 4277ff07..c3669697 100644 --- a/docs/api_reference/strategies/parse_excel.md +++ b/docs/api_reference/strategies/parse_excel.md @@ -1,3 +1,5 @@ # parse_excel ::: oteapi_dlite.strategies.parse_excel + options: + show_if_no_docstring: true diff --git a/docs/api_reference/strategies/parse_image.md b/docs/api_reference/strategies/parse_image.md index fc51228f..2e153f5e 100644 --- a/docs/api_reference/strategies/parse_image.md +++ b/docs/api_reference/strategies/parse_image.md @@ -1,3 +1,5 @@ # parse_image ::: oteapi_dlite.strategies.parse_image + options: + show_if_no_docstring: true diff --git a/docs/api_reference/strategies/serialise.md b/docs/api_reference/strategies/serialise.md index 4ade9c04..0fc56820 100644 --- a/docs/api_reference/strategies/serialise.md +++ b/docs/api_reference/strategies/serialise.md @@ -1,3 +1,5 @@ # serialise ::: oteapi_dlite.strategies.serialise + options: + show_if_no_docstring: true diff --git a/docs/index.md b/docs/index.md index cd70c6ca..d25d6816 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,5 @@ [![DOI](https://zenodo.org/badge/455070691.svg)](https://zenodo.org/badge/latestdoi/455070691) - # OTEAPI DLite Plugin All strategies implemented in this plugin (except for `create_collection`) assumes that you have the UUID of a DLite collection with the key `collection_id` in the `session`: @@ -21,6 +20,8 @@ Further reading: - [OTE-API Services Documentation](https://github.com/EMMC-ASBL/oteapi-services) - [DLite](https://github.com/SINTEF/dlite) +A list and documentation for all the strategies in this plugin can be found [here](all_strategies). + ## License and copyright The OTEAPI DLite Plugin is released under the [MIT license](LICENSE.md) with copyright © SINTEF. diff --git a/mkdocs.yml b/mkdocs.yml index 56e2fe01..4d42dc87 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -78,9 +78,6 @@ plugins: docstring_style: google docstring_options: replace_admonitions: true - # setup_commands: - # - import os - # - os.environ["MKDOCS_BUILD"] = "1" - awesome-pages nav: diff --git a/tasks.py b/tasks.py index 127a0d9b..48712c46 100644 --- a/tasks.py +++ b/tasks.py @@ -14,8 +14,6 @@ if TYPE_CHECKING: # pragma: no cover from typing import Optional, Tuple - from invoke import Context, Result - TOP_DIR = Path(__file__).parent.resolve() @@ -56,146 +54,3 @@ def setver(_, version=""): ) print(f"Bumped version to {version}.") - - -@task( - help={ - "pre-clean": ( - "Remove the 'api_reference' sub directory prior to (re)creation." - ), - "pre-commit": "Return a non-zero error code if changes were made.", - } -) # pylint: disable=too-many-branches -def create_api_reference_docs(context, pre_clean=False, pre_commit=False): - """Create the API Reference in the documentation.""" - import os - import shutil - - def write_file(full_path: Path, content: str) -> None: - """Write file with `content` to `full_path`""" - if full_path.exists(): - cached_content = full_path.read_text(encoding="utf8") - if content == cached_content: - del cached_content - return - del cached_content - full_path.write_text(content, encoding="utf8") - - package_dir = TOP_DIR / "oteapi_dlite" - docs_api_ref_dir = TOP_DIR / "docs" / "api_reference" - - unwanted_subdirs = ("__pycache__",) - unwanted_files = ("__init__.py",) - - pages_template = 'title: "{name}"\n' - md_template = "# {name}\n\n::: {py_path}\n" - models_template = ( - md_template - + f"{' ' * 4}rendering:\n{' ' * 6}show_if_no_docstring: true\n" - ) - - if docs_api_ref_dir.exists() and pre_clean: - shutil.rmtree(docs_api_ref_dir, ignore_errors=True) - if docs_api_ref_dir.exists(): - sys.exit(f"{docs_api_ref_dir} should have been removed!") - docs_api_ref_dir.mkdir(exist_ok=True) - - write_file( - full_path=docs_api_ref_dir / ".pages", - content=pages_template.format(name="API Reference"), - ) - - for dirpath, dirnames, filenames in os.walk(package_dir): - for unwanted_dir in unwanted_subdirs: - if unwanted_dir in dirnames: - # Avoid walking into or through unwanted directories - dirnames.remove(unwanted_dir) - - relpath = Path(dirpath).relative_to(package_dir) - - if not (package_dir.name / relpath / "__init__.py").exists(): - # Avoid paths that are not included in the public Python API - print("does not exist:", package_dir.name / relpath / "__init__.py") - continue - - # Create `.pages` - docs_sub_dir = docs_api_ref_dir / relpath - docs_sub_dir.mkdir(exist_ok=True) - print(docs_sub_dir) - if str(relpath) != ".": - write_file( - full_path=docs_sub_dir / ".pages", - content=pages_template.format( - name=str(relpath).rsplit("/", maxsplit=1)[-1] - ), - ) - - # Create markdown files - for filename in filenames: - if ( - re.match(r".*\.py$", filename) is None - or filename in unwanted_files - ): - # Not a Python file: We don't care about it! - # Or filename is in the tuple of unwanted files: - # We don't want it! - continue - - basename = filename[: -len(".py")] - py_path = ( - f"{package_dir.name}/{relpath}/{basename}".replace("/", ".") - if str(relpath) != "." - else f"{package_dir.name}/{basename}".replace("/", ".") - ) - md_filename = filename.replace(".py", ".md") - - # For models we want to include EVERYTHING, even if it doesn't - # have a doc-string - template = ( - models_template if str(relpath) == "models" else md_template - ) - - write_file( - full_path=docs_sub_dir / md_filename, - content=template.format(name=basename, py_path=py_path), - ) - - if pre_commit: - # Check if there have been any changes. - # List changes if yes. - if TYPE_CHECKING: # pragma: no cover - context: "Context" = context # type: ignore - - # NOTE: grep returns an exit code of 1 if it doesn't find anything - # (which will be good in this case). - # Concerning the weird last grep command see: - # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html - result: "Result" = context.run( # type: ignore - "git status --porcelain docs/api_reference | " - "grep -E '^[? MARC][?MD]' || exit 0", - hide=True, - ) - if result.stdout: - sys.exit( - "The following files have been changed/added, please stage " - f"them:\n\n{result.stdout}" - ) - - -@task -def create_docs_index(_): - """Create the documentation index page from README.md""" - readme = TOP_DIR / "README.md" - docs_index = TOP_DIR / "docs" / "index.md" - - content = readme.read_text(encoding="utf8") - - replacement_mapping = [ - ("docs/", ""), - ("(LICENSE)", "(LICENSE.md)"), - ] - - for old, new in replacement_mapping: - content = content.replace(old, new) - - docs_index.write_text(content, encoding="utf8")