Merge pull request #133 from lsst-sqre/tickets/DM-36137

DM-36137: Support setting extensions, linkcheck and nitpick settings in documenteer.toml

Author: jonathansick

docs/_rst_epilog.rst

@@ -32,12 +32,12 @@
 .. _breathe: http://breathe.readthedocs.io/en/latest/index.html
 .. _conda-forge: https://conda-forge.org
 .. _conda: https://conda.io/en/latest/index.html
-.. _doxylink: https://pythonhosted.org/sphinxcontrib-doxylink/
 .. _isort: https://pycqa.github.io/isort/
 .. _numpydoc: https://numpydoc.readthedocs.io/en/latest/index.html
 .. _pre-commit: https://pre-commit.com
 .. _pytest: https://pytest.org
 .. _toctree: http://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-toctree
+.. _linkcheck: https://www.sphinx-doc.org/en/master/usage/configuration.html?#options-for-the-linkcheck-builder
 
 .. Internal links
 

docs/conf.py

@@ -1,22 +1,5 @@
 from documenteer.conf.guide import *
 
-# Intersphinx
-
-intersphinx_mapping = {
-    "python": ("https://docs.python.org/3/", None),
-    "requests": ("https://requests.readthedocs.io/en/latest/", None),
-    "developer": ("https://developer.lsst.io/", None),
-    "pybtex": ("https://docs.pybtex.org/", None),
-    "sphinx": ("https://www.sphinx-doc.org/en/master/", None),
-}
-
-# Warnings to ignore
-nitpick_ignore = [
-    # This link to the base pybtex still never resolves because it is not
-    # in pybtex's intersphinx'd API reference.
-    ("py:class", "pybtex.style.formatting.plain.Style"),
-]
-
 # Automodapi
 # https://sphinx-automodapi.readthedocs.io/en/latest/automodapi.html
 automodapi_toctreedirnm = "dev/api/contents"

docs/documenteer.toml

@@ -9,3 +9,19 @@ package = "documenteer"
 
 [sphinx]
 rst_epilog_file = "_rst_epilog.rst"
+nitpicky = true
+nitpick_ignore = [
+  ["py:class", "pybtex.style.formatting.plain.Style"],
+  ["py:obj", "sphinx_automodapi.automodapi"],
+]
+nitpick_ignore_regex = [
+  ['py:.*', 'docutils.*'],
+  ['py:.*', 'pydantic.*'],
+]
+
+[sphinx.intersphinx.projects]
+python = "https://docs.python.org/3/"
+requests = "https://requests.readthedocs.io/en/latest/"
+developer = "https://developer.lsst.io/"
+pybtex = "https://docs.pybtex.org/"
+sphinx = "https://www.sphinx-doc.org/en/master/"

docs/index.rst

@@ -9,6 +9,7 @@ Documenteer
 This documentation is for version |version|. `Find docs for other versions. <https://documenteer.lsst.io/v>`__
 Documenteer is developed on GitHub at https://github.com/lsst-sqre/documenteer.
 
+.. _pip-install:
 .. _installation:
 
 Installation

docs/project-guides/guides/extend-conf-py.rst

@@ -6,42 +6,29 @@ The basic configurations, through :file:`documenteer.toml` and |documenteer.conf
 You will likely need to extend that configuration, though.
 This page describes a few common scenarios.
 
-In general, structure your customizations so that they add to the configuration presets from |documenteer.conf.guide|.
-If the configuration variable is a list or a dictionary, try to append to that list or dictionary rather than reassigning the whole variable.
+Using the [sphinx] table in documenteer.toml
+============================================
 
-Adding a package to Intersphinx
--------------------------------
+Before editing :file:`conf.py`, check the :doc:`toml-reference` to see if there's support for a particular Sphinx configuration.
+For example, the ``[sphinx]`` table includes support for adding Sphinx extensions, adding projects to the Intersphinx_ mapping, among other capabilities.
 
-One scenario is adding additional projects to the Intersphinx_ configuration.
-For example, to add the Python standard library so that built-in Python APIs can be referenced:
+If the configuration isn't supported by :file:`documenteer.toml`, you can edit the Sphinx :file:`conf.py` configuration file directly.
 
-.. code-block:: python
-   :caption: conf.py
-
-   from documenteer.conf.guide import *
-
-   intersphinx_mapping["python"] = ("https://docs.python.org/3", None)
-
-To additionally add the LSST Science Pipelines:
-
-.. code-block:: python
-   :caption: conf.py
-
-   from documenteer.conf.guide import *
-
-   intersphinx_mapping["python"] = ("https://pipelines.lsst.io", None)
+Editing conf.py
+===============
 
-Adding a Sphinx extension
--------------------------
+When Sphinx runs, the :file:`conf.py` configuration file is "executed" so that all variables in the global namespace become Sphinx configurations.
+The |documenteer.conf.guide| configuration presets populate the global namespace.
+Therefore you can add or edit those configurations by setting variables after the import of |documenteer.conf.guide|.
 
-You can add additional `Sphinx extensions`_ to your Sphinx build to make use of custom reStructuredText directives and roles.
-To add a new extension, append to the ``extensions`` list:
+For example, to change the number of times the linkcheck builder will try a URL:
 
 .. code-block:: python
    :caption: conf.py
 
    from documenteer.conf.guide import *
 
-   extensions.extend(["sphinx-click"])
+   linkcheck_retries = 2
 
-Remember that additional packages may need to be added to your project's Python dependencies (such as in a ``requirements.txt`` or ``pyproject.toml`` file).
+See the `list of Sphinx configuration in the Sphinx documentation <https://www.sphinx-doc.org/en/master/usage/configuration.html>`__.
+Extensions can also declare additional configurations, see their documentation for listings.

docs/project-guides/guides/toml-reference.rst

@@ -131,6 +131,79 @@ If your GitHub repository's URL is associated with a different field label, set
 
 This ``[sphinx]`` table allows you to set a number of Sphinx configurations that you would normally set through the :file:`conf.py` file.
 
+extensions
+----------
+
+|optional|
+
+A list of Sphinx extensions to append to the extensions included in the Documenteer configuration preset (see |documenteer.conf.guide|).
+Duplicate extensions are ignored.
+
+Remember that additional packages may need to be added to your project's Python dependencies (such as in a ``requirements.txt`` or ``pyproject.toml`` file).
+
+nitpicky
+--------
+
+|optional|
+
+Set to ``true`` to escalate Sphinx warnings to errors, which is useful for leveraging CI to notify you of any syntax errors.
+The default is ``false``.
+
+.. code-block:: toml
+
+   [sphinx]
+   nitpicky = true
+
+See ``nitpick_ignore`` and ``nitpick_ignore_regex`` for ways to suppress unavoidable errors.
+
+nitpick_ignore
+--------------
+
+|optional|
+
+A list of Sphinx warnings to ignore.
+Each item is a tuple of two items:
+
+1. ``type``, often the reStructuredText role or directive creating the error/warning.
+2. ``target``, often the argument to the reStructuredText role.
+
+.. code-block:: toml
+
+   [sphinx]
+   nitpick_ignore = [
+     ["py:class", "fastapi.applications.FastAPI"],
+     ["py:class", "httpx.AsyncClient"],
+     ["py:class", "pydantic.main.BaseModel"],
+   ]
+
+This configuration extends the Sphinx ``nitpick_ignore`` configuration.
+
+nitpick_ignore_regex
+--------------------
+
+|optional|
+
+A list of Sphinx warnings to ignore, formatted as regular expressions.
+Each item is a tuple of two items:
+
+1. ``type``, a regular expression of the warning type.
+2. ``target``, a regular expression of the warning target.
+
+.. code-block:: toml
+
+   [sphinx]
+   nitpick_ignore_regex = [
+     ['py:.*', 'fastapi.*'],
+     ['py:.*', 'httpx.*'],
+     ['py:.*', 'pydantic*'],
+   ]
+
+.. tip::
+
+   Use single quotes for literal strings in TOML.
+
+This configuration extends the Sphinx ``nitpick_ignore_regex`` configuration.
+
 rst_epilog_file
 ---------------
 
@@ -153,3 +226,43 @@ If set, the file is also included in the Sphinx source ignore list to prevent it
 
    .. |required| replace:: :bdg-primary-line:`Required`
    .. |optional| replace:: :bdg-secondary-line:`Optional`
+
+[sphinx.intersphinx]
+====================
+
+|optional|
+
+Configurations related to Intersphinx_ for linking to other Sphinx projects.
+
+[sphinx.intersphinx.projects]
+=============================
+
+|optional|
+
+A table of Sphinx projects.
+The labels are targets for the :external+sphinx:rst:role:`external` role.
+The values are URLs to the root of Sphinx documentation projects.
+
+.. code-block:: toml
+
+   [sphinx.intersphinx.projects]
+   sphinx = "https://www.sphinx-doc.org/en/master/"
+   documenteer = "https://documenteer.lsst.io"
+   python = "https://docs.python.org/3/"
+
+See the Intersphinx_ documentation for details on linking to other Sphinx projects.
+
+[sphinx.linkcheck]
+==================
+
+|optional|
+
+Configurations related to Sphinx's linkcheck_ builder.
+
+ignore
+------
+
+|optional|
+
+List of URL regular expressions patterns to ignore checking.
+These are appended to the ``linkcheck_ignore`` configuration.

src/documenteer/conf/_toml.py

@@ -7,7 +7,7 @@
 import sys
 from dataclasses import dataclass
 from email.message import Message
-from typing import Optional, cast
+from typing import Dict, List, MutableMapping, Optional, Tuple, Union, cast
 
 if sys.version_info < (3, 8):
     from importlib_metadata import PackageNotFoundError, metadata
@@ -100,6 +100,23 @@ class ProjectModel(BaseModel):
     python: Optional[PythonPackageModel]
 
 
+class IntersphinxModel(BaseModel):
+    """Model for Intersphinx configurations in documenteer.toml."""
+
+    projects: Dict[str, HttpUrl] = Field(
+        description="Mapping of projects and their URLs.", default_factory=dict
+    )
+
+
+class LinkCheckModel(BaseModel):
+    """Model for linkcheck builder configurations in documenteer.toml."""
+
+    ignore: List[str] = Field(
+        description="Regular expressions of URLs to skip checking links",
+        default_factory=list,
+    )
+
+
 class SphinxModel(BaseModel):
     """Model for Sphinx configurations in documenteer.toml."""
 
@@ -110,6 +127,35 @@ class SphinxModel(BaseModel):
         )
     )
 
+    extensions: List[str] = Field(
+        description="Additional Sphinx extension.", default_factory=list
+    )
+
+    nitpicky: bool = Field(
+        False, description="Escalate warnings to build errors."
+    )
+
+    nitpick_ignore: List[Tuple[str, str]] = Field(
+        description=(
+            "Errors to ignore. First item is the type (like a role or "
+            "directive) and the second is the target (like the argument to "
+            "the role)."
+        ),
+        default_factory=list,
+    )
+
+    nitpick_ignore_regex: List[Tuple[str, str]] = Field(
+        description=(
+            "Same as ``nitpick_ignore``, but both type and target are "
+            "interpreted as regular expressions."
+        ),
+        default_factory=list,
+    )
+
+    intersphinx: Optional[IntersphinxModel]
+
+    linkcheck: Optional[LinkCheckModel]
+
 
 class ConfigRoot(BaseModel):
     """The root model for a documenteer.toml configuration file."""
@@ -157,7 +203,7 @@ def base_url(self) -> str:
 
         1. The ``base_url`` field of the ``[project]`` table in
            documenteer.toml.
-        2. From importlib.metadata if `[project.python]` is set in
+        2. From importlib.metadata if ``[project.python]`` is set in
            documenteer.toml.
         3. Default is "".
         """
@@ -260,3 +306,50 @@ def _get_pyproject_url(
                 if value.startswith(prefix):
                     return value[len(prefix) :]
         return None
+
+    def append_extensions(self, extensions: List[str]) -> None:
+        """Append user-configured extensions to an existing list."""
+        if self.conf.sphinx:
+            for new_ext in self.conf.sphinx.extensions:
+                if new_ext not in extensions:
+                    extensions.append(new_ext)
+
+    def extend_intersphinx_mapping(
+        self, mapping: MutableMapping[str, Tuple[str, Union[str, None]]]
+    ) -> None:
+        """Extend the ``intersphinx_mapping`` dictionary with configured
+        projects.
+        """
+        if (
+            self.conf.sphinx
+            and self.conf.sphinx.intersphinx
+            and self.conf.sphinx.intersphinx.projects
+        ):
+            for project, url in self.conf.sphinx.intersphinx.projects.items():
+                mapping[project] = (str(url), None)
+
+    def append_linkcheck_ignore(self, link_patterns: List[str]) -> None:
+        """Append URL patterns for sphinx.linkcheck.ignore to existing
+        patterns.
+        """
+        if self.conf.sphinx and self.conf.sphinx.linkcheck:
+            link_patterns.extend(self.conf.sphinx.linkcheck.ignore)
+
+    def append_nitpick_ignore(
+        self, nitpick_ignore: List[Tuple[str, str]]
+    ) -> None:
+        if self.conf.sphinx and self.conf.sphinx.nitpick_ignore:
+            nitpick_ignore.extend(self.conf.sphinx.nitpick_ignore)
+
+    def append_nitpick_ignore_regex(
+        self, nitpick_ignore_regex: List[Tuple[str, str]]
+    ) -> None:
+        if self.conf.sphinx and self.conf.sphinx.nitpick_ignore_regex:
+            nitpick_ignore_regex.extend(self.conf.sphinx.nitpick_ignore_regex)
+
+    @property
+    def nitpicky(self) -> bool:
+        if self.conf.sphinx:
+            return self.conf.sphinx.nitpicky
+        else:
+            return False