Update documentation preparation (#209)

Update documentation handling.
Use SINTEF/ci-cd pre-commit hooks.
Remove the invoke tasks.

Make the new link in the README relative in the docs.

Update strategies in documentation.
Ensure everything is shown, even if it has no explicit doc-string
by updating options for the documentation file that showcases
all the strategies.

.github/workflows/ci_cd_updated_master.yml

@@ -104,16 +104,16 @@ jobs:
     - name: Check API Reference and landing page
       if: env.RELEASE_RUN == 'false'
       run: |
-        invoke create-api-reference-docs --pre-clean
-        invoke create-docs-index
+        pre-commit run --all-files docs-api-reference
+        pre-commit run --all-files docs-landing-page
 
         if [ -n "$(git status --porcelain docs/api_reference docs/index.md)" ]; then
           echo -e "\u274c Discrepancies found !"
           echo -e "The following files in the documentation must be committed:"
           git status --porcelain docs/api_reference docs/index.md
           echo -e "\nRun:\n"
-          echo "    invoke create-api-reference-docs --pre-clean"
-          echo -e "    invoke create-docs-index\n"
+          echo "    pre-commit run --all-files docs-api-reference"
+          echo -e "    pre-commit run --all-files docs-landing-page\n"
           echo "And commit the changed files."
           exit 1
         else

.github/workflows/ci_tests.yml

@@ -196,11 +196,11 @@ jobs:
       run: |
         python -m pip install -U pip
         pip install -U setuptools wheel
-        pip install -U -r requirements_docs.txt
+        pip install -U -r requirements_docs.txt -r requirements_dev.txt
         pip install -e .
 
     - name: Build
       run: |
-        invoke create-api-reference-docs --pre-clean
-        invoke create-docs-index
+        pre-commit run --all-files docs-api-reference
+        pre-commit run --all-files docs-landing-page
         mkdocs build --strict

.pre-commit-config.yaml

@@ -43,6 +43,19 @@ repos:
     - id: mypy
       exclude: ^tests/.*$
 
+  - repo: https://github.com/SINTEF/ci-cd
+    rev: v2.7.2
+    hooks:
+    - id: docs-api-reference
+      args:
+      - --package-dir=oteapi_dlite
+      - --full-docs-folder=models
+      - --full-docs-folder=strategies
+    - id: docs-landing-page
+      args:
+      - --replacement=(LICENSE),(LICENSE.md)
+      - --replacement=https://EMMC-ASBL.github.io/oteapi-dlite/latest/,
+
   - repo: local
     hooks:
     - id: pylint
@@ -67,19 +80,3 @@ repos:
       types: [python]
       require_serial: true
       files: ^tests/.*$
-    - id: update-docs-api-reference
-      name: Update API Reference in Documentation
-      entry: invoke
-      args: [create-api-reference-docs, --pre-clean, --pre-commit]
-      language: python
-      pass_filenames: false
-      files: ^oteapi_dlite/.*\.py$
-      description: Update the API Reference documentation, whenever a Python file is touched in the code base.
-    - id: update-docs-index
-      name: Update Landing Page for Documentation
-      entry: invoke
-      args: [create-docs-index]
-      language: python
-      pass_filenames: false
-      files: ^README.md$
-      description: Update the landing page for the documentation if the source file (README.md) is changed.

README.md

@@ -1,6 +1,5 @@
 [![DOI](https://zenodo.org/badge/455070691.svg)](https://zenodo.org/badge/latestdoi/455070691)
 
-
 # OTEAPI DLite Plugin
 
 All strategies implemented in this plugin (except for `create_collection`) assumes that you have the UUID of a DLite collection with the key `collection_id` in the `session`:
@@ -21,6 +20,8 @@ Further reading:
 - [OTE-API Services Documentation](https://github.com/EMMC-ASBL/oteapi-services)
 - [DLite](https://github.com/SINTEF/dlite)
 
+A list and documentation for all the strategies in this plugin can be found [here](https://EMMC-ASBL.github.io/oteapi-dlite/latest/all_strategies).
+
 ## License and copyright
 
 The OTEAPI DLite Plugin is released under the [MIT license](LICENSE) with copyright © SINTEF.

docs/all_strategies.md

@@ -5,3 +5,6 @@ This page provides documentation for the `oteapi_dlite.strategies` submodule, wh
 These strategies will be available when setting up a server in an environment with oteapi-dlite installed.
 
 ::: oteapi_dlite.strategies
+    options:
+      show_submodules: true
+      show_if_no_docstring: true

docs/api_reference/models/session.md

@@ -1,5 +1,5 @@
 # session
 
 ::: oteapi_dlite.models.session
-    rendering:
+    options:
       show_if_no_docstring: true

docs/api_reference/strategies/convert.md

@@ -1,3 +1,5 @@
 # convert
 
 ::: oteapi_dlite.strategies.convert
+    options:
+      show_if_no_docstring: true

docs/api_reference/strategies/filter.md

@@ -1,3 +1,5 @@
 # filter
 
 ::: oteapi_dlite.strategies.filter
+    options:
+      show_if_no_docstring: true

docs/api_reference/strategies/generate.md

@@ -1,3 +1,5 @@
 # generate
 
 ::: oteapi_dlite.strategies.generate
+    options:
+      show_if_no_docstring: true

docs/api_reference/strategies/mapping.md

@@ -1,3 +1,5 @@
 # mapping
 
 ::: oteapi_dlite.strategies.mapping
+    options:
+      show_if_no_docstring: true

docs/api_reference/strategies/parse.md

@@ -1,3 +1,5 @@
 # parse
 
 ::: oteapi_dlite.strategies.parse
+    options:
+      show_if_no_docstring: true

docs/api_reference/strategies/parse_excel.md

@@ -1,3 +1,5 @@
 # parse_excel
 
 ::: oteapi_dlite.strategies.parse_excel
+    options:
+      show_if_no_docstring: true

docs/api_reference/strategies/parse_image.md

@@ -1,3 +1,5 @@
 # parse_image
 
 ::: oteapi_dlite.strategies.parse_image
+    options:
+      show_if_no_docstring: true

docs/api_reference/strategies/serialise.md

@@ -1,3 +1,5 @@
 # serialise
 
 ::: oteapi_dlite.strategies.serialise
+    options:
+      show_if_no_docstring: true

docs/index.md

@@ -1,6 +1,5 @@
 [![DOI](https://zenodo.org/badge/455070691.svg)](https://zenodo.org/badge/latestdoi/455070691)
 
-
 # OTEAPI DLite Plugin
 
 All strategies implemented in this plugin (except for `create_collection`) assumes that you have the UUID of a DLite collection with the key `collection_id` in the `session`:
@@ -21,6 +20,8 @@ Further reading:
 - [OTE-API Services Documentation](https://github.com/EMMC-ASBL/oteapi-services)
 - [DLite](https://github.com/SINTEF/dlite)
 
+A list and documentation for all the strategies in this plugin can be found [here](all_strategies).
+
 ## License and copyright
 
 The OTEAPI DLite Plugin is released under the [MIT license](LICENSE.md) with copyright © SINTEF.

mkdocs.yml

@@ -78,9 +78,6 @@ plugins:
             docstring_style: google
             docstring_options:
               replace_admonitions: true
-          # setup_commands:
-          #   - import os
-          #   - os.environ["MKDOCS_BUILD"] = "1"
   - awesome-pages
 
 nav:

tasks.py

@@ -14,8 +14,6 @@
 if TYPE_CHECKING:  # pragma: no cover
     from typing import Optional, Tuple
 
-    from invoke import Context, Result
-
 
 TOP_DIR = Path(__file__).parent.resolve()
 
@@ -56,146 +54,3 @@ def setver(_, version=""):
     )
 
     print(f"Bumped version to {version}.")
-
-
-@task(
-    help={
-        "pre-clean": (
-            "Remove the 'api_reference' sub directory prior to (re)creation."
-        ),
-        "pre-commit": "Return a non-zero error code if changes were made.",
-    }
-)  # pylint: disable=too-many-branches
-def create_api_reference_docs(context, pre_clean=False, pre_commit=False):
-    """Create the API Reference in the documentation."""
-    import os
-    import shutil
-
-    def write_file(full_path: Path, content: str) -> None:
-        """Write file with `content` to `full_path`"""
-        if full_path.exists():
-            cached_content = full_path.read_text(encoding="utf8")
-            if content == cached_content:
-                del cached_content
-                return
-            del cached_content
-        full_path.write_text(content, encoding="utf8")
-
-    package_dir = TOP_DIR / "oteapi_dlite"
-    docs_api_ref_dir = TOP_DIR / "docs" / "api_reference"
-
-    unwanted_subdirs = ("__pycache__",)
-    unwanted_files = ("__init__.py",)
-
-    pages_template = 'title: "{name}"\n'
-    md_template = "# {name}\n\n::: {py_path}\n"
-    models_template = (
-        md_template
-        + f"{' ' * 4}rendering:\n{' ' * 6}show_if_no_docstring: true\n"
-    )
-
-    if docs_api_ref_dir.exists() and pre_clean:
-        shutil.rmtree(docs_api_ref_dir, ignore_errors=True)
-        if docs_api_ref_dir.exists():
-            sys.exit(f"{docs_api_ref_dir} should have been removed!")
-    docs_api_ref_dir.mkdir(exist_ok=True)
-
-    write_file(
-        full_path=docs_api_ref_dir / ".pages",
-        content=pages_template.format(name="API Reference"),
-    )
-
-    for dirpath, dirnames, filenames in os.walk(package_dir):
-        for unwanted_dir in unwanted_subdirs:
-            if unwanted_dir in dirnames:
-                # Avoid walking into or through unwanted directories
-                dirnames.remove(unwanted_dir)
-
-        relpath = Path(dirpath).relative_to(package_dir)
-
-        if not (package_dir.name / relpath / "__init__.py").exists():
-            # Avoid paths that are not included in the public Python API
-            print("does not exist:", package_dir.name / relpath / "__init__.py")
-            continue
-
-        # Create `.pages`
-        docs_sub_dir = docs_api_ref_dir / relpath
-        docs_sub_dir.mkdir(exist_ok=True)
-        print(docs_sub_dir)
-        if str(relpath) != ".":
-            write_file(
-                full_path=docs_sub_dir / ".pages",
-                content=pages_template.format(
-                    name=str(relpath).rsplit("/", maxsplit=1)[-1]
-                ),
-            )
-
-        # Create markdown files
-        for filename in filenames:
-            if (
-                re.match(r".*\.py$", filename) is None
-                or filename in unwanted_files
-            ):
-                # Not a Python file: We don't care about it!
-                # Or filename is in the tuple of unwanted files:
-                # We don't want it!
-                continue
-
-            basename = filename[: -len(".py")]
-            py_path = (
-                f"{package_dir.name}/{relpath}/{basename}".replace("/", ".")
-                if str(relpath) != "."
-                else f"{package_dir.name}/{basename}".replace("/", ".")
-            )
-            md_filename = filename.replace(".py", ".md")
-
-            # For models we want to include EVERYTHING, even if it doesn't
-            # have a doc-string
-            template = (
-                models_template if str(relpath) == "models" else md_template
-            )
-
-            write_file(
-                full_path=docs_sub_dir / md_filename,
-                content=template.format(name=basename, py_path=py_path),
-            )
-
-    if pre_commit:
-        # Check if there have been any changes.
-        # List changes if yes.
-        if TYPE_CHECKING:  # pragma: no cover
-            context: "Context" = context  # type: ignore
-
-        # NOTE: grep returns an exit code of 1 if it doesn't find anything
-        # (which will be good in this case).
-        # Concerning the weird last grep command see:
-        # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html
-        result: "Result" = context.run(  # type: ignore
-            "git status --porcelain docs/api_reference | "
-            "grep -E '^[? MARC][?MD]' || exit 0",
-            hide=True,
-        )
-        if result.stdout:
-            sys.exit(
-                "The following files have been changed/added, please stage "
-                f"them:\n\n{result.stdout}"
-            )
-
-
-@task
-def create_docs_index(_):
-    """Create the documentation index page from README.md"""
-    readme = TOP_DIR / "README.md"
-    docs_index = TOP_DIR / "docs" / "index.md"
-
-    content = readme.read_text(encoding="utf8")
-
-    replacement_mapping = [
-        ("docs/", ""),
-        ("(LICENSE)", "(LICENSE.md)"),
-    ]
-
-    for old, new in replacement_mapping:
-        content = content.replace(old, new)
-
-    docs_index.write_text(content, encoding="utf8")