From fac8205ee80e6427a5a64894a5572245463432a4 Mon Sep 17 00:00:00 2001 From: Sida Say Date: Fri, 23 Feb 2024 16:43:36 +0700 Subject: [PATCH] update --- .github/actions/docker-common/action.yml | 20 ++++- .github/workflows/pr.yml | 75 +++++++++++++++- .github/workflows/release.yml | 86 ++++++++++++++++++- .gitignore | 8 +- Dockerfile | 83 ++++++------------ Makefile | 2 +- README.md | 75 ++++++++-------- example/docker-compose.yml | 16 ++-- .../opt/requirements.txt => requirements.txt | 4 +- rootfs/etc/supervisor/supervisord.conf | 18 ---- rootfs/etc/uwsgi/uwsgi.ini | 19 ---- rootfs/opt/privacyidea/gunicorn_conf.py | 67 +++++++++++++++ .../opt/templates/nginx-pi-ssl.conf.template | 22 ----- rootfs/opt/templates/nginx-pi.conf.template | 11 --- rootfs/opt/templates/nginx.conf.template | 19 ---- rootfs/opt/templates/pi-config.template | 10 +-- rootfs/usr/local/bin/configure_nginx.sh | 86 ------------------- ...rivacyidea.sh => configure_privacyidea.sh} | 40 ++++----- .../usr/local/bin/privacyidea_entrypoint.sh | 7 +- structure-tests.yaml | 32 +------ 20 files changed, 352 insertions(+), 348 deletions(-) rename prebuildfs/opt/requirements.txt => requirements.txt (56%) delete mode 100644 rootfs/etc/supervisor/supervisord.conf delete mode 100644 rootfs/etc/uwsgi/uwsgi.ini create mode 100644 rootfs/opt/privacyidea/gunicorn_conf.py delete mode 100644 rootfs/opt/templates/nginx-pi-ssl.conf.template delete mode 100644 rootfs/opt/templates/nginx-pi.conf.template delete mode 100644 rootfs/opt/templates/nginx.conf.template delete mode 100755 rootfs/usr/local/bin/configure_nginx.sh rename rootfs/usr/local/bin/{start_privacyidea.sh => configure_privacyidea.sh} (74%) diff --git a/.github/actions/docker-common/action.yml b/.github/actions/docker-common/action.yml index cc9c4b1..d67536a 100644 --- a/.github/actions/docker-common/action.yml +++ b/.github/actions/docker-common/action.yml @@ -26,6 +26,16 @@ inputs: default: '' required: false description: GitHub Container Registry token + python_base_image: + type: string + default: 3.8.18-slim-bookworm + required: false + description: Base image for python + is_default_version: + type: boolean + default: false + required: false + description: Is this the default versions runs: using: "composite" steps: @@ -52,11 +62,13 @@ runs: khalibre/privacyidea ghcr.io/Khalibre/privacyidea tags: | + type=raw,value={{branch}}-python-${{ inputs.python_version }},enable=${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + type=raw,value={{branch}},enable=${{ github.event_name == 'push' && inputs.is_default_version || github.event_name == 'workflow_dispatch' }} + type=raw,value={{tag}}-python-${{ inputs.python_version }},enable=${{ github.event_name == 'push' }} + type=raw,value={{tag}},enable=${{ github.event_name == 'push' && inputs.is_default_version }} + type=raw,value=latest,enable=${{is_default_branch && inputs.is_default_version }} type=ref,event=pr,enable=${{ github.event_name == 'pull_request' }} - type=raw,value=latest,enable={{is_default_branch}} type=sha,enable=${{ github.event_name == 'push' }} - type=raw,value={{tag}},enable=${{ github.event_name == 'push' }} - type=raw,value={{branch}},enable=${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -70,6 +82,7 @@ runs: platforms: linux/amd64 tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} + build-args: BASE_IMAGE_TAG=${{ inputs.python_base_image }} - name: Container Structure Tests shell: bash run: | @@ -88,3 +101,4 @@ runs: platforms: ${{ inputs.platforms }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} + build-args: BASE_IMAGE_TAG=${{ inputs.python_base_image }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ce34655..f48463e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -7,8 +7,8 @@ on: workflow_dispatch: jobs: - build: - name: Build container images + python-38-bookworm: + name: Python 3.8/Bookworm permissions: pull-requests: write runs-on: ubuntu-latest @@ -20,3 +20,74 @@ jobs: with: platforms: linux/amd64 push: false + python_base_image: 3.8.18-slim-bookworm + python-39-bookworm: + name: Python 3.9/Bookworm + permissions: + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + platforms: linux/amd64 + push: false + python_base_image: 3.9.18-slim-bookworm + python-310-bookworm: + name: Python 3.10/Bookworm + permissions: + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + platforms: linux/amd64 + push: false + python_base_image: 3.10.13-slim-bookworm + python-38-bullseye: + name: Python 3.8/Bullseye + permissions: + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + platforms: linux/amd64 + push: false + python_base_image: 3.8.18-slim-bullseye + python-39-bullseye: + name: Python 3.9/Bullseye + permissions: + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + platforms: linux/amd64 + push: false + python_base_image: 3.9.18-slim-bullseye + python-310-bullseye: + name: Python 3.10/Bullseye + permissions: + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + platforms: linux/amd64 + push: false + python_base_image: 3.10.13-slim-bullseye diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f5a127e..44f6ab9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,8 +8,8 @@ on: - 'v3.*' workflow_dispatch: jobs: - build: - name: Build release container images + python-38-bookworm: + name: Python 3.8/Bookworm runs-on: ubuntu-latest steps: - name: Checkout @@ -17,8 +17,90 @@ jobs: - name: Docker common actions uses: ./.github/actions/docker-common with: + docker_hub_token: ${{ secrets.DOCKERHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKERHUB_USERNAME }} + ghcr_token: ${{ secrets.GITHUB_TOKEN }} + platforms: linux/amd64,linux/arm64 + push: true + python_base_image: 3.8.18-slim-bookworm + is_default_version: true + python-39-bookworm: + name: Python 3.9/Bookworm + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + docker_hub_token: ${{ secrets.DOCKERHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKERHUB_USERNAME }} + ghcr_token: ${{ secrets.GITHUB_TOKEN }} + platforms: linux/amd64,linux/arm64 + push: true + python_base_image: 3.9.18-slim-bookworm + is_default_version: false + python-310-bookworm: + name: Python 3.10/Bookworm + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + docker_hub_token: ${{ secrets.DOCKERHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKERHUB_USERNAME }} + ghcr_token: ${{ secrets.GITHUB_TOKEN }} + platforms: linux/amd64,linux/arm64 + push: true + python_base_image: 3.10.13-slim-bookworm + is_default_version: false + python-38-bullseye: + name: Python 3.8/Bullseye + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + docker_hub_token: ${{ secrets.DOCKERHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKERHUB_USERNAME }} + ghcr_token: ${{ secrets.GITHUB_TOKEN }} platforms: linux/amd64,linux/arm64 push: true + python_base_image: 3.8.18-slim-bullseye + is_default_version: false + python-39-bullseye: + name: Python 3.9/Bullseye + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: + docker_hub_token: ${{ secrets.DOCKERHUB_TOKEN }} docker_hub_username: ${{ secrets.DOCKERHUB_USERNAME }} + ghcr_token: ${{ secrets.GITHUB_TOKEN }} + platforms: linux/amd64,linux/arm64 + push: true + python_base_image: 3.9.18-slim-bullseye + is_default_version: false + python-310-bullseye: + name: Python 3.10/Bullseye + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Docker common actions + uses: ./.github/actions/docker-common + with: docker_hub_token: ${{ secrets.DOCKERHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKERHUB_USERNAME }} ghcr_token: ${{ secrets.GITHUB_TOKEN }} + platforms: linux/amd64,linux/arm64 + push: true + python_base_image: 3.10.13-slim-bullseye + is_default_version: false diff --git a/.gitignore b/.gitignore index 6125dcf..a6b732d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ +.github +.gitignore *.swp -secretkey +example +LICENSE +Makefile pipepper +README +secretkey structure-tests-report.xml diff --git a/Dockerfile b/Dockerfile index 5efcf48..b1178fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,64 +1,31 @@ -FROM python:3.8.18-bookworm +ARG BASE_IMAGE_TAG=3.8.18-slim-bookworm +FROM python:$BASE_IMAGE_TAG as builder +ENV VIRTUAL_ENV=/opt/privacyidea +WORKDIR $VIRTUAL_ENV +RUN apt-get update && apt-get install -y python3-dev gcc libpq-dev +COPY requirements.txt requirements.txt +RUN python3 -m venv "$VIRTUAL_ENV" && . $VIRTUAL_ENV/bin/activate && pip3 install -r requirements.txt + +FROM python:$BASE_IMAGE_TAG LABEL maintainer="Sida Say " +ENV PI_SKIP_BOOTSTRAP=false \ + PI_DB_VENDOR=sqlite \ + PI_HOME=/opt/privacyidea \ + PI_DATA_DIR=/data/privacyidea \ + PI_CFG_DIR=/etc/privacyidea \ + PI_CFG_FILE=pi.cfg COPY prebuildfs / - SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN install_packages ca-certificates gettext-base nginx tini tree jq && \ - apt-get clean - -# Create directories and user for PrivacyIdea and set ownership -RUN mkdir -p /data/privacyidea/keys \ - /var/log/privacyidea \ - /etc/privacyidea && \ - adduser --gecos "PrivacyIdea User" \ - --disabled-password \ - --home /home/privacyidea \ - --uid 1001 \ - privacyidea && \ - usermod -g 1001 privacyidea && \ - chown -R privacyidea:privacyidea /var/log/privacyidea /data/privacyidea /etc/privacyidea - -# Set environment variables for uWSGI and Nginx -ENV UWSGI_INI=/etc/uwsgi/uwsgi.ini \ - UWSGI_CHEAPER=2 \ - UWSGI_PROCESSES=16 \ - NGINX_MAX_UPLOAD=1m \ - NGINX_WORKER_PROCESSES=auto \ - NGINX_SERVER_TOKENS=off \ - NGINX_WORKER_CONNECTIONS=1024 \ - NGINX_LISTEN_PORT=80 \ - NGINX_LISTEN_SSL_PORT=443 \ - NGINX_SSL_ENABLED=true \ - PI_SKIP_BOOTSTRAP=false \ - DB_VENDOR=sqlite \ - PI_HOME=/opt/privacyidea \ - VIRTUAL_ENV=/opt/privacyidea - -# Set environment variables for Python -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -# Set the PrivacyIdea version to install -ARG PI_VERSION=3.9.1 - -# Create a virtual environment for PrivacyIdea and install its dependencies -RUN python3 -m venv $VIRTUAL_ENV && \ - pip3 install --upgrade pip && \ - pip3 install wheel && \ - pip3 install -r /opt/requirements.txt && \ - rm -rf /root/.cache - -# Copy the rootfs directory to the root of the container filesystem -COPY rootfs / - -# Expose ports 80 and 443 -EXPOSE 80/tcp -EXPOSE 443/tcp - -# Set the entrypoint to the privacyidea_entrypoint.sh script +RUN install_packages ca-certificates gettext-base tini tree jq && \ + mkdir -p "$PI_DATA_DIR" "$PI_CFG_DIR" && \ + chown -R nobody:nogroup "$PI_DATA_DIR" "$PI_CFG_DIR" +USER nobody +WORKDIR "$PI_HOME" +COPY --from=builder /opt/privacyidea . +COPY --chown=nobody:nogroup rootfs / +ENV PATH="$PI_HOME/bin:$PATH" +EXPOSE 8080/tcp +VOLUME [ "$PI_DATA_DIR" ] ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/privacyidea_entrypoint.sh"] - -WORKDIR /opt/privacyidea - -VOLUME [ "/data/privacyidea" ] diff --git a/Makefile b/Makefile index 598f744..2394bca 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ push: ## Push image docker push khalibre/privacyidea:dev run: cleanup create_volume secretkey pipepper ## Run test - docker run -p 80:80 -p 443:443 -ti --name=privacyidea-dev --env-file=secretkey --env-file=pipepper khalibre/privacyidea:dev + docker run -p 8080:8080 -p 8443:8443 -ti --name=privacyidea-dev --env-file=secretkey --env-file=pipepper khalibre/privacyidea:dev create_volume: mkdir $(LOCAL_DATA_VOLUME) diff --git a/README.md b/README.md index 59939ff..3c1a11b 100644 --- a/README.md +++ b/README.md @@ -61,46 +61,49 @@ The Khalibre privacyIDEA container can create a default admin user by setting th - `PI_ADMIN_USER`: Administrator default user. Default: **admin**. - `PI_ADMIN_PASSWORD`: Administrator default password. Default: **privacyidea** -### Connecting to database - -The Khalibre privacyIDEA requires a database to work. This is configured with the following environment variables: - -- `DB_VENDOR`: Database vendor (support mysql, mariadb or posgresql) Default **sqlite**. -- `DB_USER`: Database user. No defaults. -- `DB_PASSWORD`: Database. No defaults. -- `DB_NAME`: Database name. No defaults. -- `DB_HOST`: Database host. No defaults. - -### NGINX configuration - -- `NGINX_LISTEN_PORT`: Get the listen port for Nginx, default to **80** -- `NGINX_LISTEN_SSL_PORT`: Get the secured listen port for Nginx, default to **443** -- `NGINX_MAX_UPLOAD`: Get the maximum upload file size for Nginx, default to **100Mb** -- `NGINX_SERVER_TOKENS`: Hide Nginx server version on error pages and in the “Server HTTP” response header field. Default **off** -- `NGINX_SSL_CERT`: Path to SSL certificate. Default to **/etc/nginx/certs/pi-server-cert.pem** -- `NGINX_SSL_ENABLED`: Set to true to enable SSL, Default **false** -- `NGINX_SSL_KEY`: Path to SSL key, default **/etc/nginx/certs/pi-server-key.pem** -- `NGINX_WORKER_CONNECTIONS`: Set the max number of connections per worker for Nginx, if requested. -- `NGINX_WORKER_PROCESSES`: Get the number of workers for Nginx, default to 1 - -### privacyIDEA configuration - -- `CACHE_TYPE`: privacyIDEA cache type. Default simple. -- `PI_PEPPER`: This is used to encrypt the admin passwords. No defaults. -- `PI_AUDIT_NO_SIGN`: If you by any reason want to avoid signing audit entries set it **true**. -- `PI_AUDIT_KEY_PRIVATE_PATH`: This is used to sign the audit log -- `PI_AUDIT_KEY_PUBLIC_PATH`: This is used to sign the audit log -- `PI_ENCFILE`: This is used to encrypt the token data and token passwords. No defaults. -- `PI_HSM`: privacyIDEA HSM. Default **default** -- `PI_LOGFILE`: privacyIDEA log file location. Default **/var/log/privacyidea/privacyidea.log** -- `PI_LOGLEVEL`: privacyIDEA log level. Default **INFO** -- `SECRET_KEY`: This is used to encrypt the auth_token. No defaults. -- `SUPERUSER_REALM`: The realm, where users are allowed to login as administrators in comma separated value. Default **administrator** -- `PI_SKIP_BOOTSTRAP`: Set this to **true** to prevent the container to run setup again. Default **false** +## Environment variables + +### PrivacyIDEA Environment Variables + +| Environment Variable | Description | Default | +| :------------------- | :---------- | :------ | +| PI_ADMIN_USER | Initial admin user for privacyIDEA login | admin | +| PI_ADMIN_PASSWORD | Initial admin password | privacyidea | +| PI_DB_VENDOR | Database vendor | sqlite | +| PI_DB_USER | Database user | | +| PI_DB_PASSWORD | Database password | | +| PI_DB_NAME | Database name | | +| PI_DB_HOST | Database host | | +| PI_CACHE_TYPE | privacyIDEA cache type | simple | +| PI_PEPPER | This is used to encrypt the admin passwords | | +| PI_AUDIT_NO_SIGN | If you by any reason want to avoid signing audit entries set it true | false | +| PI_AUDIT_KEY_PRIVATE_PATH | This is used to sign the audit log | | +| PI_AUDIT_KEY_PUBLIC_PATH | This is used to sign the audit log | | +| PI_ENCFILE | This is used to encrypt the token data and token passwords | | +| PI_HSM | privacyIDEA HSM | default | +| PI_LOGFILE | privacyIDEA log file location | /var/log/privacyidea/privacyidea.log | +| PI_LOGLEVEL | privacyIDEA log level | INFO | +| PI_SECRET_KEY | This is used to encrypt the auth_token | | +| PI_SUPERUSER_REALM | The realm, where users are allowed to login as administrators in comma separated value | administrator | +| PI_SKIP_BOOTSTRAP | Set this to true to prevent the container to run setup again | false |<|endofmiddle|> > [!WARNING] > Be careful and setting `PI_SKIP_BOOTSTRAP` to **true** after first initialization. This will prevent the container to run setup again or your data such as admin credentials, secret keys, etc will be overwritten. +### gunicorn environment variables + +| Environment Variable | Description | Default | +| :------------------- | :---------- | :------ | +| GUNICORN_ACCESS_LOGFILE | Gunicorn access log file location | stdout | +| GUNICORN_ERROR_LOGFILE | Gunicorn error log file location | stderr | +| GUNICORN_WORKER_CLASS | Gunicorn worker class | sync | +| GUNICORN_WORKERS | Gunicorn workers | 1 | +| GUNICORN_BIND | Gunicorn bind address if not set GUNICORN_HOST and GUNICORN_PORT will be used | None | +| GUNICORN_HOST | Gunicorn host will be ingored if GUNICORN_BIND is set | 0.0.0.0 | +| GUNICORN_PORT | Gunicorn port will be ingored if GUNICORN_BIND is set | 8080 | +| GUNICORN_LOGLEVEL | Gunicorn log level | INFO | +| GUNICORN_TIMEOUT | Gunicorn timeout | 60 | + ## Providing Files to the Container The privacyIDEA container uses the files you provide to execute the following use cases: diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 8a8d5de..2643cbe 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -14,17 +14,17 @@ services: pi: image: 'khalibre/privacyidea:dev' ports: - - '8080:80' + - '8080:8080' environment: - - DB_VENDOR=mariadb - - DB_NAME=privacyidea - - DB_HOST=pi-db - - DB_USER=privacyidea - - DB_PASSWORD=privacyidea@123 - - SECRET_KEY=suppersecretkey + - PI_DB_VENDOR=mariadb + - PI_DB_NAME=privacyidea + - PI_DB_HOST=pi-db + - PI_DB_USER=privacyidea + - PI_DB_PASSWORD=privacyidea@123 + - PI_SECRET_KEY=suppersecretkey - PI_PEPPER=secretworduseforadminencrypt - PI_PAGE_TITLE=pivacyIDEA - - SUPERUSER_REALM="administrator,api" + - PI_SUPERUSER_REALM="administrator,api" depends_on: - pi-db wp-db: diff --git a/prebuildfs/opt/requirements.txt b/requirements.txt similarity index 56% rename from prebuildfs/opt/requirements.txt rename to requirements.txt index 379c796..2423166 100644 --- a/prebuildfs/opt/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ privacyIDEA==3.9.2 psycopg2==2.9.9 -supervisor==4.2.5 -uWSGI==2.0.24 grpcio==1.62.0 +grpcio-tools==1.62.0 +gunicorn==21.2.0 diff --git a/rootfs/etc/supervisor/supervisord.conf b/rootfs/etc/supervisor/supervisord.conf deleted file mode 100644 index 2c08023..0000000 --- a/rootfs/etc/supervisor/supervisord.conf +++ /dev/null @@ -1,18 +0,0 @@ -[supervisord] -nodaemon=true - -[program:uwsgi] -command=/opt/privacyidea/bin/uwsgi --ini /etc/uwsgi/uwsgi.ini -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 - -[program:nginx] -command=/usr/sbin/nginx -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -# Graceful stop, see http://nginx.org/en/docs/control.html -stopsignal=QUIT diff --git a/rootfs/etc/uwsgi/uwsgi.ini b/rootfs/etc/uwsgi/uwsgi.ini deleted file mode 100644 index a822ad1..0000000 --- a/rootfs/etc/uwsgi/uwsgi.ini +++ /dev/null @@ -1,19 +0,0 @@ -[uwsgi] -wsgi-file=/opt/privacyidea/etc/privacyidea/privacyideaapp.wsgi -buffer-size=8192 -socket = 127.0.0.1:8080 -# Graceful shutdown on SIGTERM, see https://github.com/unbit/uwsgi/issues/849#issuecomment-118869386 -hook-master-start = unix_signal:15 gracefully_kill_them_all -need-app = true -die-on-term = true -# For debugging and testing -# For debugging and testing -show-config = false -# enable thread -disable-logging = true -enable-threads = true -lazy-apps=true -log-4xx = true -log-5xx = true -master = true -single-interpreter = true diff --git a/rootfs/opt/privacyidea/gunicorn_conf.py b/rootfs/opt/privacyidea/gunicorn_conf.py new file mode 100644 index 0000000..838a0c9 --- /dev/null +++ b/rootfs/opt/privacyidea/gunicorn_conf.py @@ -0,0 +1,67 @@ +import json +import multiprocessing +import os + +workers_per_core_str = os.getenv("GUNICORN_WORKERS_PER_CORE", "1") +max_workers_str = os.getenv("GUNICORN_MAX_WORKERS") +use_max_workers = None +if max_workers_str: + use_max_workers = int(max_workers_str) +web_concurrency_str = os.getenv("GUNICORN_WEB_CONCURRENCY", None) + +host = os.getenv("GUNICORN_HOST", "0.0.0.0") +port = os.getenv("GUNICORN_PORT", "8080") +bind_env = os.getenv("GUNICORN_BIND", None) +use_loglevel = os.getenv("GUNICORN_LOG_LEVEL", "info") +if bind_env: + use_bind = bind_env +else: + use_bind = f"{host}:{port}" + +cores = multiprocessing.cpu_count() +workers_per_core = float(workers_per_core_str) +default_web_concurrency = workers_per_core * cores +if web_concurrency_str: + web_concurrency = int(web_concurrency_str) + assert web_concurrency > 0 +else: + web_concurrency = max(int(default_web_concurrency), 2) + if use_max_workers: + web_concurrency = min(web_concurrency, use_max_workers) +accesslog_var = os.getenv("GUNICORN_ACCESS_LOG", "-") +use_accesslog = accesslog_var or None +errorlog_var = os.getenv("GUNICORN_ERROR_LOG", "-") +use_errorlog = errorlog_var or None +graceful_timeout_str = os.getenv("GUNICORN_GRACEFUL_TIMEOUT", "120") +timeout_str = os.getenv("GUNICORN_TIMEOUT", "120") +keepalive_str = os.getenv("GUNICORN_KEEP_ALIVE", "5") + +# Gunicorn config variables +loglevel = use_loglevel +workers = web_concurrency +bind = use_bind +errorlog = use_errorlog +worker_tmp_dir = "/dev/shm" +accesslog = use_accesslog +graceful_timeout = int(graceful_timeout_str) +timeout = int(timeout_str) +keepalive = int(keepalive_str) + + +# For debugging and testing +log_data = { + "loglevel": loglevel, + "workers": workers, + "bind": bind, + "graceful_timeout": graceful_timeout, + "timeout": timeout, + "keepalive": keepalive, + "errorlog": errorlog, + "accesslog": accesslog, + # Additional, non-gunicorn variables + "workers_per_core": workers_per_core, + "use_max_workers": use_max_workers, + "host": host, + "port": port, +} +print(json.dumps(log_data)) diff --git a/rootfs/opt/templates/nginx-pi-ssl.conf.template b/rootfs/opt/templates/nginx-pi-ssl.conf.template deleted file mode 100644 index 3be05ca..0000000 --- a/rootfs/opt/templates/nginx-pi-ssl.conf.template +++ /dev/null @@ -1,22 +0,0 @@ -server { - listen $NGINX_LISTEN_SSL_PORT ssl; - listen [::]:$NGINX_LISTEN_SSL_PORT ssl; - - ssl_certificate $NGINX_SSL_CERT; - ssl_certificate_key $NGINX_SSL_KEY; - - ssl_session_cache shared:SSL:1m; - ssl_session_timeout 5m; - - ssl_ciphers HIGH:!aNULL:!MD5; - ssl_prefer_server_ciphers on; - - proxy_http_version 1.1; - - client_max_body_size $NGINX_MAX_UPLOAD; - location / { - - include uwsgi_params; - uwsgi_pass 127.0.0.1:8080; - } -} diff --git a/rootfs/opt/templates/nginx-pi.conf.template b/rootfs/opt/templates/nginx-pi.conf.template deleted file mode 100644 index 68c4a80..0000000 --- a/rootfs/opt/templates/nginx-pi.conf.template +++ /dev/null @@ -1,11 +0,0 @@ -server { - listen $NGINX_LISTEN_PORT; - listen [::]:$NGINX_LISTEN_PORT; - - client_max_body_size $NGINX_MAX_UPLOAD; - proxy_http_version 1.1; - location / { - include uwsgi_params; - uwsgi_pass 127.0.0.1:8080; - } -} diff --git a/rootfs/opt/templates/nginx.conf.template b/rootfs/opt/templates/nginx.conf.template deleted file mode 100644 index ef09110..0000000 --- a/rootfs/opt/templates/nginx.conf.template +++ /dev/null @@ -1,19 +0,0 @@ -worker_processes $NGINX_WORKER_PROCESSES; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; -events { - worker_connections $NGINX_WORKER_CONNECTIONS; -} -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - keepalive_timeout 65; - server_tokens $NGINX_SERVER_TOKENS; - include /etc/nginx/conf.d/*.conf; -} -daemon off; diff --git a/rootfs/opt/templates/pi-config.template b/rootfs/opt/templates/pi-config.template index 53d3cf3..cd5057a 100644 --- a/rootfs/opt/templates/pi-config.template +++ b/rootfs/opt/templates/pi-config.template @@ -2,7 +2,7 @@ import os import logging import sys -SECRET_KEY = os.environ.get('SECRET_KEY') +SECRET_KEY = os.environ.get('PI_SECRET_KEY') if SECRET_KEY is None: print("SECRET_KEY not set! Refusing to start") sys.exit(1) @@ -13,7 +13,7 @@ if PI_PEPPER is None: sys.exit(1) # The realm, where users are allowed to login as administrators -SUPERUSER_REALM = os.environ.get('SUPERUSER_REALM','administrator').split(',') +SUPERUSER_REALM = os.environ.get('PI_SUPERUSER_REALM','administrator').split(',') SQLALCHEMY_DATABASE_URI = "$SQLALCHEMY_DATABASE_URI" PI_ENCFILE = os.environ.get("PI_ENCFILE", "/data/privacyidea/keys/encfile") @@ -24,10 +24,10 @@ PI_AUDIT_MODULE = os.environ.get("", "privacyidea.lib.auditmodules.sqlaudit") PI_AUDIT_KEY_PRIVATE = os.environ.get("PI_AUDIT_KEY_PRIVATE_PATH", "/data/privacyidea/keys/private.pem") # PI_AUDIT_KEY_PUBLIC will be used only when PI_AUDIT_NO_SIGN is True PI_AUDIT_KEY_PUBLIC = os.environ.get("PI_AUDIT_KEY_PUBLIC_PATH", "/data/privacyidea/keys/public.pem") -PI_LOGFILE = os.environ.get("PI_LOGFILE", "/var/log/privacyidea/privacyidea.log") -PI_LOGLEVEL = logging.getLevelName(os.environ.get("PI_LOGLEVEL", "INFO")) +PI_LOGFILE = os.environ.get("PI_LOGFILE", "/dev/stdout") +PI_LOGLEVEL = logging.getLevelName(os.environ.get("PI_LOGLEVEL", 20)) PI_NODE = os.environ.get("HOSTNAME", "localnode") -CACHE_TYPE = os.environ.get("CACHE_TYPE", "simple") +CACHE_TYPE = os.environ.get("PI_CACHE_TYPE", "simple") PI_EXTERNAL_LINKS = os.environ.get("PI_EXTERNAL_LINKS", "True").lower() == "true" PI_VASCO_LIBRARY = None PI_ENGINE_REGISTRY_CLASS = os.environ.get("PI_ENGINE_REGISTRY_CLASS", "shared") diff --git a/rootfs/usr/local/bin/configure_nginx.sh b/rootfs/usr/local/bin/configure_nginx.sh deleted file mode 100755 index 8b9b6fa..0000000 --- a/rootfs/usr/local/bin/configure_nginx.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -set -e - -function generate_cert { - - # Create certificate directory - mkdir -p /etc/nginx/certs - - # [ global parameters ] - # certificate configuration - readonly CERT_DAYS=36500 - readonly RSA_STR_LEN=4096 - readonly PREFIX=pi- - readonly CERT_DIR=/etc/nginx/certs - readonly KEY_DIR=/etc/nginx/certs - # certificate content definition - readonly ADDRESS_COUNTRY_CODE=KH - readonly ADDRESS_PREFECTURE=PI - readonly ADDRESS_CITY='Phnom Penh' - readonly COMPANY_NAME=Khalibre - readonly COMPANY_SECTION=DevOps - readonly CERT_PASSWORD= # no password - # - server - readonly SERVER_DOMAIN=localhost - readonly SERVER_EMAIL=server@email.address - - # [ functions ] - echo_cert_params() { - local company_domain="$1" - local company_email="$2" - - echo $ADDRESS_COUNTRY_CODE - echo $ADDRESS_PREFECTURE - echo $ADDRESS_CITY - echo $COMPANY_NAME - echo $COMPANY_SECTION - echo $company_domain - echo $company_email - echo $CERT_PASSWORD # password - echo $CERT_PASSWORD # password (again) - } - echo_server_cert_params() { - echo_cert_params "$SERVER_DOMAIN" "$SERVER_EMAIL" - } - - # [ main ] - # generate certificates - # - server - echo_server_cert_params | \ - openssl req -newkey rsa:$RSA_STR_LEN -days $CERT_DAYS -nodes -keyout $KEY_DIR/${PREFIX}server-key.pem -out $CERT_DIR/${PREFIX}server-req.pem > /dev/null - openssl rsa -in $KEY_DIR/${PREFIX}server-key.pem -out $KEY_DIR/${PREFIX}server-key.pem > /dev/null - openssl x509 -req -in $CERT_DIR/${PREFIX}server-req.pem -days $CERT_DAYS -signkey $KEY_DIR/${PREFIX}server-key.pem -out $CERT_DIR/${PREFIX}server-cert.pem > /dev/null - - # clean up (before permission changed) - rm $CERT_DIR/${PREFIX}server-req.pem > /dev/null - - # validate permission - chmod 400 $KEY_DIR/${PREFIX}server-key.pem > /dev/null - - # verify certificate - openssl x509 -in $CERT_DIR/${PREFIX}server-cert.pem -noout -text > /dev/null -} - -function main { - # Generate certificate if SSL is enabled and no certificate/key paths are provided - if [ "$NGINX_SSL_ENABLED" = true ]; then - if [ -z "$NGINX_SSL_CERT" ] && [ -z "$NGINX_SSL_KEY" ]; - then - echo "[WARNING] SSL enabled but NGINX_SSL_CERT and NGINX_SSL_KEY are not defined, using seflf-signed certificate" - echo "" - generate_cert - export NGINX_SSL_CERT=/etc/nginx/certs/pi-server-cert.pem - export NGINX_SSL_KEY=/etc/nginx/certs/pi-server-key.pem - echo "[INFO] A seflf-signed certificate has been generated at $NGINX_SSL_CERT and $NGINX_SSL_KEY" - echo "" - fi - envsubst < /opt/templates/nginx-pi-ssl.conf.template > /etc/nginx/conf.d/pi-ssl.conf - fi - - # Substitute environment variables in nginx configuration files - envsubst < /opt/templates/nginx.conf.template > /etc/nginx/nginx.conf - envsubst < /opt/templates/nginx-pi.conf.template > /etc/nginx/conf.d/pi.conf -} - -main diff --git a/rootfs/usr/local/bin/start_privacyidea.sh b/rootfs/usr/local/bin/configure_privacyidea.sh similarity index 74% rename from rootfs/usr/local/bin/start_privacyidea.sh rename to rootfs/usr/local/bin/configure_privacyidea.sh index ef96f9c..244e708 100755 --- a/rootfs/usr/local/bin/start_privacyidea.sh +++ b/rootfs/usr/local/bin/configure_privacyidea.sh @@ -16,61 +16,61 @@ function main { generate_pi_config prestart_privacyidea - exec supervisord -c /etc/supervisor/supervisord.conf + exec /opt/privacyidea/bin/gunicorn -c /opt/privacyidea/gunicorn_conf.py "privacyidea.app:create_app(config_name='production', config_file='$PI_CFG_DIR/$PI_CFG_FILE')" } function generate_pi_config { # Common logic for checking and setting default values check_and_set_defaults() { - [ -z "$DB_HOST" ] && echo "[ERROR] DB_HOST should be defined" && return 1 - [ -z "$DB_USER" ] && echo "[ERROR] DB_USER should be defined" && return 1 - [ -z "$DB_PASSWORD" ] && echo "[ERROR] DB_PASSWORD should be defined" && return 1 - [ -z "$DB_NAME" ] && echo "[ERROR] DB_NAME should be defined" && return 1 + [ -z "$PI_DB_HOST" ] && echo "[ERROR] PI_DB_HOST should be defined" && return 1 + [ -z "$PI_DB_USER" ] && echo "[ERROR] PI_DB_USER should be defined" && return 1 + [ -z "$PI_DB_PASSWORD" ] && echo "[ERROR] PI_DB_PASSWORD should be defined" && return 1 + [ -z "$PI_DB_NAME" ] && echo "[ERROR] PI_DB_NAME should be defined" && return 1 # Set the default port if it is not defined if [ -z "$DB_PORT" ]; then echo "[INFO] DB_PORT is not defined using default port" export DB_PORT=3306 # Default port for mariadb/mysql - [ "$DB_VENDOR" = "postgresql" ] && export DB_PORT=5432 # Default port for postgresql + [ "$PI_DB_VENDOR" = "postgresql" ] && export DB_PORT=5432 # Default port for postgresql fi # URL encode the password - encoded_password=$(printf "%s" "$DB_PASSWORD" | jq -s -R -r @uri) + encoded_password=$(printf "%s" "$PI_DB_PASSWORD" | jq -s -R -r @uri) } # Check the selected database vendor - case $DB_VENDOR in + case $PI_DB_VENDOR in "mariadb" | "mysql") - echo "[INFO] Using $DB_VENDOR ..." + echo "[INFO] Using $PI_DB_VENDOR ..." check_and_set_defaults # Define the SQLAlchemy database URI using the necessary variables - export SQLALCHEMY_DATABASE_URI=${DB_VENDOR}+pymysql://${DB_USER}:${encoded_password}@${DB_HOST}:${DB_PORT}/${DB_NAME} + export SQLALCHEMY_DATABASE_URI=${PI_DB_VENDOR}+pymysql://${PI_DB_USER}:${encoded_password}@${PI_DB_HOST}:${DB_PORT}/${PI_DB_NAME} ;; "postgresql") - echo "Using $DB_VENDOR..." + echo "Using $PI_DB_VENDOR..." check_and_set_defaults # Define the SQLAlchemy database URI using the necessary variables - export SQLALCHEMY_DATABASE_URI=${DB_VENDOR}+psycopg2://${DB_USER}:${encoded_password}@${DB_HOST}:${DB_PORT}/${DB_NAME} + export SQLALCHEMY_DATABASE_URI=${PI_DB_VENDOR}+psycopg2://${PI_DB_USER}:${encoded_password}@${PI_DB_HOST}:${DB_PORT}/${PI_DB_NAME} ;; *) echo "" - echo "[WARNING] DB_VENDOR environment variable is not set. Using default SQLite..." + echo "[WARNING] PI_DB_VENDOR environment variable is not set. Using default SQLite..." echo "" # Define the SQLAlchemy database URI for SQLite - export SQLALCHEMY_DATABASE_URI=sqlite://///data/privacyidea/privacyidea.db + export SQLALCHEMY_DATABASE_URI=sqlite:////${PI_DATA_DIR}/privacyidea.db ;; esac # Check if the configuration file already exists - if [ ! -f /etc/privacyidea/pi.cfg ]; then + if [ ! -f ${PI_CFG_DIR}/pi.cfg ]; then # Check if SQLALCHEMY_DATABASE_URI is defined if [ -z "$SQLALCHEMY_DATABASE_URI" ]; then @@ -79,7 +79,7 @@ function generate_pi_config { echo "" else # Use the pi-config.template file as a template and substitute the necessary variables - envsubst < /opt/templates/pi-config.template > /etc/privacyidea/pi.cfg + envsubst < /opt/templates/pi-config.template > ${PI_CFG_DIR}/pi.cfg fi fi @@ -116,15 +116,15 @@ function prestart_privacyidea { if [ "${PI_SKIP_BOOTSTRAP}" = false ]; then # Create keys directory if not exists - if [ ! -d /data/privacyidea/keys ]; then + if [ ! -d ${PI_DATA_DIR}/keys ]; then echo "" echo "[INFO] Creating keys directory..." echo "" - mkdir /data/privacyidea/keys + mkdir ${PI_DATA_DIR}/keys fi # Create encryption key file if not exists - if [ ! -f /data/privacyidea/keys/encfile ]; then + if [ ! -f ${PI_DATA_DIR}/keys/encfile ]; then echo "" echo "[INFO] Encryption key file not found, creating a new one..." echo "" @@ -132,7 +132,7 @@ function prestart_privacyidea { fi # Create audit keys if not exists - if [ ! -f /data/privacyidea/keys/private.pem ]; then + if [ ! -f ${PI_DATA_DIR}/keys/private.pem ]; then echo "" echo "[INFO] Creating audit keys..." echo "" diff --git a/rootfs/usr/local/bin/privacyidea_entrypoint.sh b/rootfs/usr/local/bin/privacyidea_entrypoint.sh index c0d1cc5..fb9cd14 100755 --- a/rootfs/usr/local/bin/privacyidea_entrypoint.sh +++ b/rootfs/usr/local/bin/privacyidea_entrypoint.sh @@ -11,7 +11,7 @@ function main { echo "" # Set the mount directory for configuration files. - if [ -d /etc/privacyidea/mount ] + if [ -d /etc/privacyidea/mount ]s then PI_MOUNT_DIR=/etc/privacyidea/mount else @@ -22,9 +22,6 @@ function main { # Execute any pre-configuration scripts. execute_scripts /usr/local/privacyidea/scripts/pre-configure - # Configure Nginx. - . configure_nginx.sh - # Execute any pre-startup scripts. execute_scripts /usr/local/privacyidea/scripts/pre-startup @@ -37,7 +34,7 @@ function main { # Define the function to start PrivacyIDEA. function start_privacyidea { - . start_privacyidea.sh + . configure_privacyidea.sh } # Call the main function. diff --git a/structure-tests.yaml b/structure-tests.yaml index ef74a85..856628f 100644 --- a/structure-tests.yaml +++ b/structure-tests.yaml @@ -2,45 +2,17 @@ schemaVersion: 2.0.0 metadataTest: envVars: - - key: UWSGI_INI - value: "/etc/uwsgi/uwsgi.ini" - - key: UWSGI_CHEAPER - value: "2" - - key: UWSGI_PROCESSES - value: "16" - - key: NGINX_MAX_UPLOAD - value: "1m" - - key: NGINX_WORKER_PROCESSES - value: "auto" - - key: NGINX_SERVER_TOKENS - value: "off" - - key: NGINX_WORKER_CONNECTIONS - value: "1024" - - key: NGINX_LISTEN_PORT - value: "80" - - key: NGINX_LISTEN_SSL_PORT - value: "443" - - key: NGINX_SSL_ENABLED - value: "true" - key: PI_SKIP_BOOTSTRAP value: "false" - - key: DB_VENDOR + - key: PI_DB_VENDOR value: "sqlite" - key: PI_HOME value: "/opt/privacyidea" - - key: VIRTUAL_ENV - value: "/opt/privacyidea" workdir: "/opt/privacyidea" entrypoint: ["/usr/bin/tini", "--", "/usr/local/bin/privacyidea_entrypoint.sh"] - exposedPorts: ["80", "443"] + exposedPorts: ["8080"] fileExistenceTests: -- name: 'Supervisor config' - path: '/etc/supervisor/supervisord.conf' - shouldExist: true -- name: 'UWSGI config' - path: '/etc/uwsgi/uwsgi.ini' - shouldExist: true - name: 'Template directory' path: '/opt/templates' shouldExist: true