Adjust downloader and config to pulpcore CI

- Default downloader pulls from a branch (not latest release)
- Tested with fork

pyproject.toml

@@ -7,11 +7,10 @@ name = "pulp-docs"
 version = "0.0.1"
 dependencies = [
     "mkdocs-material",
-    "mkdocs-material[imaging]",
     "mkdocstrings[python]",
     "mkdocs-macros-plugin",
-    "Jinja2",
-    "Faker",
+    "importlib_resources",
+    "httpx",
 ]
 
 [project.scripts]

src/pulp_docs/data/repolist.yml

@@ -5,12 +5,8 @@ meta:
 repos:
   core:
     - name: pulpcore
-      title: Common
-  content:
-    - name: pulp_rpm
-      title: Rpm Package!!!
-    - name: pulp_deb
-      title: Debian Package !!!
-    - name: pulp_maven
-      title: Maven
+      owner: pulp
+      title: Pulp Core
+      branch: main
+  content: []
   other: []

src/pulp_docs/main.py

@@ -4,6 +4,7 @@
 It defines its interface.
 """
 import subprocess
+import os
 import sys
 from pathlib import Path
 
@@ -19,10 +20,17 @@ def get_abspath(name: str) -> Path:
 
 
 class Config:
+    """
+    Params:
+        mkdocs_file: the base mkdocs used in serving/building
+        repolist: the configuration repositories (which and how to fetch)
+    """
+
     def __init__(self):
         self.verbose = False
         self.workdir = Path()
         self.mkdocs_file = files("pulp_docs").joinpath("data/mkdocs.yml")
+        self.repolist = files("pulp_docs").joinpath("data/repolist.yml")
 
 
 pass_config = click.make_pass_decorator(Config, ensure=True)
@@ -40,20 +48,31 @@ def main(config: Config):
 @pass_config
 def serve(config: Config):
     """Run mkdocs server"""
+    env = os.environ.copy()
+    env.update({
+        "PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())
+    })
+
     options = (
         ("--config-file", config.mkdocs_file),
     )
     cmd = ["mkdocs", "serve"]
+
     for opt in options:
         cmd.extend(opt)
     print("Running:", " ".join(str(s) for s in cmd))
-    subprocess.run(cmd)
+    subprocess.run(cmd, env=env)
 
 
 @main.command()
 @pass_config
 def build(config: Config):
     """Build mkdocs site"""
+    env = os.environ.copy()
+    env.update({
+        "PULPDOCS_BASE_REPOLIST": str(config.repolist.absolute())
+    })
+
     options = (
         ("--config-file", config.mkdocs_file),
         ("--site-dir", str(Path("site").absolute())),
@@ -62,7 +81,7 @@ def build(config: Config):
     for opt in options:
         cmd.extend(opt)
     print("Building:", " ".join(str(s) for s in cmd))
-    subprocess.run(cmd)
+    subprocess.run(cmd, env=env)
 
 
 @main.command()

src/pulp_docs/mkdocs_macros.py

@@ -14,15 +14,22 @@
     2. Configure mkdocstrings plugin to find the source code
 
 """
+import logging
 import os
 import shutil
 import tempfile
 import typing as t
 from pathlib import Path
 
 from importlib_resources import as_file, files
+
 from pulp_docs.plugin_repos import Repos
 
+# the name of the docs in the source repositories
+SRC_DOCS_DIRNAME = "staging_docs"
+
+log = logging.getLogger('mkdocs')
+
 
 def create_clean_tmpdir(custom_tmpdir: t.Optional[Path] = None):
     tmpdir_basepath = Path(custom_tmpdir) if custom_tmpdir else Path(
@@ -77,37 +84,57 @@ def prepare_repositories(TMPDIR: Path, repos: Repos):
     for repo in repos.all:
         # 1. Download repo (copy locally or fetch from GH)
         this_src_dir = repo_sources / repo.name
-        print("Downloading '{}', at '{}'".format(repo.name, this_src_dir))
-        repo.download(dest_dir=this_src_dir, from_local=True)
+        log.info(
+            "[pulp-docs] Downloading '{}', at '{}'".format(repo.name, this_src_dir))
+        repo.download(dest_dir=this_src_dir)
 
         # 2. Isolate docs dir from codebase (makes mkdocs happy)
         this_docs_dir = repo_docs / repo.name
-        print("Moving doc related files:\nfrom '{}'\nto '{}'".format(
+        log.info("Moving doc files:\nfrom '{}'\nto '{}'".format(
             this_src_dir, this_docs_dir))
-        shutil.copytree(this_src_dir / "docs", this_docs_dir / "docs")
-        shutil.copy(this_src_dir / "CHANGELOG.md",
-                    this_docs_dir / "CHANGELOG.md")
-        shutil.copy(this_src_dir / "README.md", this_docs_dir / "README.md")
+        shutil.copytree(this_src_dir / SRC_DOCS_DIRNAME,
+                        this_docs_dir / "docs")
+
+        try:
+            shutil.copy(this_src_dir / "CHANGELOG.md",
+                        this_docs_dir / "CHANGELOG.md")
+        except FileNotFoundError:
+            log.warn("CHANGELOG.md does not exist. Keep going")
+
+        try:
+            shutil.copy(this_src_dir / "README.md",
+                        this_docs_dir / "README.md")
+        except FileNotFoundError:
+            log.warn("README.md does not exist. Keep going")
 
         # 3. Generate REST Api pages (workaround)
+        log.info("Generating REST_API page")
         rest_api_page = this_docs_dir / "docs" / "rest_api.md"
         rest_api_page.touch()
         md_title = f"# {repo.title} REST Api"
         md_body = f"[{repo.rest_api_link}]({repo.rest_api_link})"
         rest_api_page.write_text(f"{md_title}\n\n{md_body}")
 
     # Copy template-files (from this plugin) to tmpdir
+    log.info("[pulp-docs] Moving pulp-docs /docs to final destination")
     data_file_docs = files("pulp_docs").joinpath("data/docs")
     with as_file(data_file_docs) as _docs:
         shutil.copytree(_docs, repo_docs / "pulp-docs")
-    shutil.copy(repo_sources / repos.core.name / "docs" /
+    shutil.copy(repo_sources / repos.core.name / SRC_DOCS_DIRNAME /
                 "index.md", repo_docs / "index.md")
+    log.info("[pulp-docs] Done downloading sources.")
     return (repo_docs, repo_sources)
 
 
+def create_no_content_page(tmpdir: Path):
+    """Create placeholder.md file to be used when section is empty"""
+    placeholder_page = Path(tmpdir / "placeholder.md")
+    placeholder_page.write_text("# Placeholder Page\n\nNo content here yet.")
+    return placeholder_page
+
+
 def get_navigation(tmpdir: Path, repos: Repos):
     """The dynamic generated 'nav' section of mkdocs.yml"""
-
     # {repo}/docs/{persona}/{content-type}/*md
     # {repo}/docs/reference/*md
     def get_children(path: t.Union[str, Path]):
@@ -138,42 +165,47 @@ def expand_reference(template_str: str):
             _nav[repo.title] = reference_section
         return _nav
 
+    def from_core(url: str):
+        corename = "pulpcore"
+        return f"{corename}/{url}"
+
     getting_started = [
-        {"Overview": "core/docs/sections/getting_started/index.md"},
+        {"Overview": from_core("docs/sections/getting_started/index.md")},
         {"Quickstart": get_children(
-            "core/docs/sections/getting_started/quickstart")},
+            from_core("docs/sections/getting_started/quickstart"))},
         {"Fundamentals": get_children(
-            "core/docs/sections/getting_started/fundamentals")}
+            from_core("docs/sections/getting_started/fundamentals"))}
     ]
-    guides = [
-        {"Overview": "core/docs/sections/guides/index.md"},
+    guides= [
+        {"Overview": from_core("docs/sections/guides/index.md")},
         {"For Content-Management": expand_repos(
             "{repo}/docs/content-manager/guides")},
         {"For Sys-Admins": expand_repos("{repo}/docs/sys-admin/guides")},
     ]
-    learn = [
-        {"Overview": "core/docs/sections/learn/index.md"},
+    learn= [
+        {"Overview": from_core("docs/sections/learn/index.md")},
         {"For Content-Management": expand_repos(
             "{repo}/docs/content-manager/learn")},
         {"For Sys-Admins": expand_repos("{repo}/docs/sys-admin/learn")},
     ]
-    reference = [
-        {"Overview": "core/docs/sections/reference/index.md"},
-        {"Repository Map": "core/docs/sections/reference/01-repository-map.md"},
-        {"Glossary": "core/docs/sections/reference/02-glossary.md"},
+    reference= [
+        {"Overview": from_core("docs/sections/reference/index.md")},
+        {"Repository Map": from_core(
+            "docs/sections/reference/01-repository-map.md")},
+        {"Glossary": from_core("docs/sections/reference/02-glossary.md")},
         {"Repositories": expand_reference("{repo}/docs/reference")},
     ]
-    development = [
-        {"Overview": "core/docs/sections/development/index.md"},
+    development= [
+        {"Overview": from_core("docs/sections/development/index.md")},
         {"Quickstart": get_children(
-            "core/docs/sections/development/quickstart/")},
+            from_core("docs/sections/development/quickstart/"))},
         {"Onboarding": get_children(
-            "core/docs/sections/development/onboarding/")},
+            from_core("docs/sections/development/onboarding/"))},
         {"Guides": get_children("core/docs/sections/development/guides/")},
     ]
 
     # main navigation
-    navigation = [
+    navigation= [
         {"Home": "index.md"},
         {"Getting Started": getting_started},
         {"Guides": guides},
@@ -186,19 +218,31 @@ def expand_reference(template_str: str):
 
 def define_env(env):
     """The mkdocs-marcros 'on_configuration' hook. Used to setup the project."""
-    # Load repository listing
-    base_repolist = os.environ.get("PULPDOCS_BASE_REPOLIST", None)
-    repos = Repos.from_yaml(
-        Path(base_repolist)) if base_repolist else Repos.test_fixtures()
-
-    # Create tmp_dir with desired repos
-    TMPDIR = create_clean_tmpdir()
-    docs_dir, source_dir = prepare_repositories(TMPDIR, repos)
-
-    # Configure mkdocstrings
-    code_sources = [str(source_dir / repo.name) for repo in repos.all]
-    env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"]["paths"] = code_sources
-
-    # Configure mkdocs navigation
-    env.conf["docs_dir"] = docs_dir
-    env.conf["nav"] = get_navigation(docs_dir, repos)
+    # ===
+    log.info("[pulp-docs] Loading configuration from environ")
+    base_repolist= os.environ.get("PULPDOCS_BASE_REPOLIST", None)
+    log.info(f"{base_repolist=}\n")
+
+    log.info("[pulp-docs] Loading repolist file")
+    if base_repolist == "testing":
+        repos= Repos.test_fixtures()
+    else:
+        repos= Repos.from_yaml(Path(base_repolist))
+    log.info(f"{repos}")
+
+    # ===
+    log.info("[pulp-docs] Preparing repositories")
+    TMPDIR= create_clean_tmpdir()
+    docs_dir, source_dir= prepare_repositories(TMPDIR, repos)
+
+    # ===
+    log.info("[pulp-docs] Configuring mkdocstrings")
+    code_sources= [str(source_dir / repo.name) for repo in repos.all]
+    env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"]["paths"]= code_sources
+
+    # ===
+    log.info("[pulp-docs] Configuring navigation")
+    env.conf["docs_dir"]= docs_dir
+    env.conf["nav"]= get_navigation(docs_dir, repos)
+
+    log.info("[pulp-docs] Done with pulp-docs.")