From e2b63898a6e24988663a69d315ea04356c68e251 Mon Sep 17 00:00:00 2001 From: Levente Hunyadi Date: Mon, 30 Sep 2024 11:16:43 +0200 Subject: [PATCH] Fix miscellaneous issues --- .dockerignore | 7 ++++ .gitignore | 7 ++-- Dockerfile | 24 +++++++++++--- README.md | 20 +++++++++--- md2conf/application.py | 2 +- md2conf/mermaid.py | 60 ++++++++++++++++++++++++----------- md2conf/puppeteer-config.json | 8 +++++ package.sh | 8 ++++- setup.cfg | 1 + test.dockerfile | 31 ++++++++++++++---- 10 files changed, 129 insertions(+), 39 deletions(-) create mode 100644 .dockerignore create mode 100644 md2conf/puppeteer-config.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e29fbef --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +__pycache__ +/.env +/.envrc +/.idea +/.mypy_cache +/.venv +/*.egg-info diff --git a/.gitignore b/.gitignore index 08f46a4..10b58d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ __pycache__ /.env +/.envrc +/.idea +/.mypy_cache +/.venv /dist /*.egg-info -.envrc -.venv -.idea diff --git a/Dockerfile b/Dockerfile index aabd0fb..2fc82e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,14 +4,27 @@ FROM python:${PYTHON_VERSION}-alpine as builder COPY ./ ./ -RUN python3 -m pip install --upgrade pip && \ - pip install build && \ - python -m build --wheel +RUN PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --upgrade pip && \ + pip install build +RUN python -m build --wheel FROM python:${PYTHON_VERSION}-alpine as host -RUN apk add --update nodejs npm && \ - npm install -g @mermaid-js/mermaid-cli +# set environment for @mermaid-js/mermaid-cli +# https://github.com/mermaid-js/mermaid-cli/blob/master/Dockerfile +ENV CHROME_BIN="/usr/bin/chromium-browser" \ + PUPPETEER_SKIP_DOWNLOAD="true" + +# install dependencies for @mermaid-js/mermaid-cli +# https://github.com/mermaid-js/mermaid-cli/blob/master/install-dependencies.sh +RUN apk add chromium font-noto-cjk font-noto-emoji terminus-font ttf-dejavu ttf-freefont ttf-font-awesome ttf-inconsolata ttf-linux-libertine \ + && fc-cache -f + +RUN apk add --update nodejs npm +RUN addgroup md2conf && adduser -D -G md2conf md2conf +USER md2conf +WORKDIR /home/md2conf +RUN npm install @mermaid-js/mermaid-cli FROM host as runner @@ -19,4 +32,5 @@ COPY --from=builder /dist/*.whl dist/ RUN python3 -m pip install `ls -1 dist/*.whl` +WORKDIR /data ENTRYPOINT ["python3", "-m", "md2conf"] diff --git a/README.md b/README.md index 2bc62b9..cdcd686 100644 --- a/README.md +++ b/README.md @@ -164,15 +164,27 @@ options: --webui-links Enable Confluence Web UI links. ``` -### Using the docker container +### Using the Docker container -You can run the docker container via `docker run` or via `Dockerfile`. Either can accept the environment variables or arguments similar to the Python options. The final argument `./` corresponds to `mdpath` in the command-line utility. +You can run the Docker container via `docker run` or via `Dockerfile`. Either can accept the environment variables or arguments similar to the Python options. The final argument `./` corresponds to `mdpath` in the command-line utility. + +With `docker run`, you can pass Confluence domain, user, API and space key directly to `docker run`: ```sh -docker run --rm --name md2conf hunyadi/md2conf -d instructure.atlassian.net -u levente.hunyadi@instructure.com -a 0123456789abcdef -s DAP ./ +docker run --rm --name md2conf -v $(pwd):/data hunyadi/md2conf -d instructure.atlassian.net -u levente.hunyadi@instructure.com -a 0123456789abcdef -s DAP ./ ``` -Note that the entry point for the docker container's base image is `ENTRYPOINT ["python3", "-m", "md2conf"]`. +Alternatively, you can use a separate file `.env` to pass these parameters as environment variables: + +```sh +docker run --rm --env-file .env --name md2conf -v $(pwd):/data hunyadi/md2conf ./ +``` + +In each case, `-v $(pwd):/data` maps the current directory to Docker container's `WORKDIR` such *md2conf* can scan files and directories in the local file system. + +Note that the entry point for the Docker container's base image is `ENTRYPOINT ["python3", "-m", "md2conf"]`. + +With the `Dockerfile` approach, you can extend the base image: ```Dockerfile FROM hunyadi/md2conf:latest diff --git a/md2conf/application.py b/md2conf/application.py index 6979d42..d47fe81 100644 --- a/md2conf/application.py +++ b/md2conf/application.py @@ -141,7 +141,7 @@ def _get_or_create_page( else: if parent_id is None: raise ValueError( - "expected: Confluence page ID to act as parent for Markdown files with no linked Confluence page" + f"expected: parent page ID for Markdown file with no linked Confluence page: {absolute_path}" ) confluence_page = self._create_page( diff --git a/md2conf/mermaid.py b/md2conf/mermaid.py index a1a626b..63c447f 100644 --- a/md2conf/mermaid.py +++ b/md2conf/mermaid.py @@ -1,17 +1,37 @@ +import logging import os import os.path import shutil import subprocess from typing import Literal +LOGGER = logging.getLogger(__name__) + + +def is_docker() -> bool: + "True if the application is running in a Docker container." + + return ( + os.environ.get("CHROME_BIN") == "/usr/bin/chromium-browser" + and os.environ.get("PUPPETEER_SKIP_DOWNLOAD") == "true" + ) + + +def get_mmdc() -> str: + "Path to the Mermaid diagram converter." + + if is_docker(): + return "/home/md2conf/node_modules/.bin/mmdc" + elif os.name == "nt": + return "mmdc.cmd" + else: + return "mmdc" + def has_mmdc() -> bool: "True if Mermaid diagram converter is available on the OS." - if os.name == "nt": - executable = "mmdc.cmd" - else: - executable = "mmdc" + executable = get_mmdc() return shutil.which(executable) is not None @@ -20,20 +40,21 @@ def render(source: str, output_format: Literal["png", "svg"] = "png") -> bytes: filename = f"tmp_mermaid.{output_format}" - if os.name == "nt": - executable = "mmdc.cmd" - else: - executable = "mmdc" + cmd = [ + get_mmdc(), + "--input", + "-", + "--output", + filename, + "--outputFormat", + output_format, + ] + if is_docker(): + cmd.extend( + ["-p", os.path.join(os.path.dirname(__file__), "puppeteer-config.json")] + ) + LOGGER.debug(f"Executing: {' '.join(cmd)}") try: - cmd = [ - executable, - "--input", - "-", - "--output", - filename, - "--outputFormat", - output_format, - ] proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -41,10 +62,11 @@ def render(source: str, output_format: Literal["png", "svg"] = "png") -> bytes: stderr=subprocess.PIPE, text=False, ) - proc.communicate(input=source.encode("utf-8")) + stdout, stderr = proc.communicate(input=source.encode("utf-8")) if proc.returncode: raise RuntimeError( - f"failed to convert Mermaid diagram; exit code: {proc.returncode}" + f"failed to convert Mermaid diagram; exit code: {proc.returncode}, " + f"output:\n{stdout.decode('utf-8')}\n{stderr.decode('utf-8')}" ) with open(filename, "rb") as image: return image.read() diff --git a/md2conf/puppeteer-config.json b/md2conf/puppeteer-config.json new file mode 100644 index 0000000..7387794 --- /dev/null +++ b/md2conf/puppeteer-config.json @@ -0,0 +1,8 @@ +{ + "executablePath": "/usr/bin/chromium-browser", + "args": [ + "--no-sandbox", + "--disable-gpu", + "--disable-setuid-sandbox" + ] +} diff --git a/package.sh b/package.sh index 5208432..a7d5cda 100755 --- a/package.sh +++ b/package.sh @@ -1,4 +1,7 @@ set -e +# +# Builds a Python package and runs unit tests in Docker +# PYTHON=python3 @@ -9,11 +12,14 @@ if [ -d *.egg-info ]; then rm -rf *.egg-info; fi # create PyPI package for distribution $PYTHON -m build +docker build -f Dockerfile -t md2conf-image . +docker run -i -t --rm --env-file .env --name md2conf -v $(pwd):/data md2conf-image sample/index.md --ignore-invalid-url + # test PyPI package with various Python versions # pass environment variables from the file `.env` for PYTHON_VERSION in 3.8 3.9 3.10 3.11 3.12 do docker build -f test.dockerfile -t py-$PYTHON_VERSION-image --build-arg PYTHON_VERSION=$PYTHON_VERSION . - docker run -it --env-file .env --rm py-$PYTHON_VERSION-image python3 -m unittest discover tests + docker run -i -t --rm --env-file .env py-$PYTHON_VERSION-image python3 -m unittest discover tests docker rmi py-$PYTHON_VERSION-image done diff --git a/setup.cfg b/setup.cfg index f7a838f..da2f566 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ exclude = [options.package_data] md2conf = entities.dtd + puppeteer-config.json py.typed [options.entry_points] diff --git a/test.dockerfile b/test.dockerfile index f2014b8..10d74b9 100644 --- a/test.dockerfile +++ b/test.dockerfile @@ -1,9 +1,28 @@ ARG PYTHON_VERSION=3.9 -FROM python:${PYTHON_VERSION}-alpine -RUN PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --upgrade pip + +FROM python:${PYTHON_VERSION}-alpine as host + +# set environment for @mermaid-js/mermaid-cli +# https://github.com/mermaid-js/mermaid-cli/blob/master/Dockerfile +ENV CHROME_BIN="/usr/bin/chromium-browser" \ + PUPPETEER_SKIP_DOWNLOAD="true" + +# install dependencies for @mermaid-js/mermaid-cli +# https://github.com/mermaid-js/mermaid-cli/blob/master/install-dependencies.sh +RUN apk add chromium font-noto-cjk font-noto-emoji terminus-font ttf-dejavu ttf-freefont ttf-font-awesome ttf-inconsolata ttf-linux-libertine \ + && fc-cache -f + +RUN apk add --update nodejs npm +RUN addgroup md2conf && adduser -D -G md2conf md2conf +USER md2conf +WORKDIR /home/md2conf +RUN npm install @mermaid-js/mermaid-cli + +FROM host as runner + COPY dist/*.whl dist/ RUN python3 -m pip install `ls -1 dist/*.whl` -COPY sample/ sample/ -COPY tests/*.py tests/ -COPY tests/source/*.md tests/source/ -COPY tests/target/*.xml tests/target/ +COPY --chown=md2conf:md2conf sample/ sample/ +COPY --chown=md2conf:md2conf tests/*.py tests/ +COPY --chown=md2conf:md2conf tests/source/*.md tests/source/ +COPY --chown=md2conf:md2conf tests/target/*.xml tests/target/