Update CI

.ci/gen_certs.py

@@ -0,0 +1,59 @@
+import argparse
+import os
+import sys
+import typing as t
+
+import trustme
+
+
+def main(argv: t.Optional[t.List[str]] = None) -> None:
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(prog="gen_certs")
+    parser.add_argument(
+        "-d",
+        "--dir",
+        default=os.getcwd(),
+        help="Directory where certificates and keys are written to. Defaults to cwd.",
+    )
+
+    args = parser.parse_args(argv)
+    cert_dir = args.dir
+
+    if not os.path.isdir(cert_dir):
+        raise ValueError(f"--dir={cert_dir} is not a directory")
+
+    key_type = trustme.KeyType["ECDSA"]
+
+    # Generate the CA certificate
+    ca = trustme.CA(key_type=key_type)
+    # Write the certificate the client should trust
+    ca_cert_path = os.path.join(cert_dir, "ca.pem")
+    ca.cert_pem.write_to_path(path=ca_cert_path)
+
+    # Generate the server certificate
+    server_cert = ca.issue_cert("localhost", "127.0.0.1", "::1", key_type=key_type)
+    # Write the certificate and private key the server should use
+    server_key_path = os.path.join(cert_dir, "server.key")
+    server_cert_path = os.path.join(cert_dir, "server.pem")
+    server_cert.private_key_pem.write_to_path(path=server_key_path)
+    with open(server_cert_path, mode="w") as f:
+        f.truncate()
+    for blob in server_cert.cert_chain_pems:
+        blob.write_to_path(path=server_cert_path, append=True)
+
+    # Generate the client certificate
+    client_cert = ca.issue_cert("admin@example.com", common_name="admin", key_type=key_type)
+    # Write the certificate and private key the client should use
+    client_key_path = os.path.join(cert_dir, "client.key")
+    client_cert_path = os.path.join(cert_dir, "client.pem")
+    client_cert.private_key_pem.write_to_path(path=client_key_path)
+    with open(client_cert_path, mode="w") as f:
+        f.truncate()
+    for blob in client_cert.cert_chain_pems:
+        blob.write_to_path(path=client_cert_path, append=True)
+
+
+if __name__ == "__main__":
+    main()

.ci/nginx.conf.j2

@@ -0,0 +1,147 @@
+# Copy from pulp-oci-images.
+# Ideally we can get it upstream again.
+#
+# TODO: Support IPv6.
+# TODO: Maybe serve multiple `location`s, not just one.
+
+# The "nginx" package on fedora creates this user and group.
+user nginx nginx;
+# Gunicorn docs suggest this value.
+worker_processes 1;
+daemon off;
+events {
+    worker_connections 1024;  # increase if you have lots of clients
+    accept_mutex off;  # set to 'on' if nginx worker_processes > 1
+}
+
+http {
+    include mime.types;
+    # fallback in case we can't determine a type
+    default_type application/octet-stream;
+    sendfile on;
+
+    # If left at the default of 1024, nginx emits a warning about being unable
+    # to build optimal hash types.
+    types_hash_max_size 4096;
+
+    map $ssl_client_s_dn $ssl_client_s_dn_cn {
+        default "";
+        ~CN=(?<CN>[^,]+) $CN;
+    }
+
+    upstream pulp-content {
+         server 127.0.0.1:24816;
+    }
+
+    upstream pulp-api {
+         server 127.0.0.1:24817;
+    }
+
+    server {
+        # Gunicorn docs suggest the use of the "deferred" directive on Linux.
+        {% if https | default(false) -%}
+        listen 443 default_server deferred ssl;
+
+        ssl_certificate /etc/pulp/certs/pulp_webserver.crt;
+        ssl_certificate_key /etc/pulp/certs/pulp_webserver.key;
+        ssl_session_cache shared:SSL:50m;
+        ssl_session_timeout 1d;
+        ssl_session_tickets off;
+
+        # intermediate configuration
+        ssl_protocols TLSv1.2;
+        ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+        ssl_prefer_server_ciphers on;
+
+        # HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)
+        add_header Strict-Transport-Security max-age=15768000;
+
+        # Configure client cert authentication
+        ssl_client_certificate /etc/pulp/certs/ca.pem;
+        ssl_verify_client optional;
+        {%- else -%}
+        listen 80 default_server deferred;
+        {%- endif %}
+        server_name $hostname;
+
+        # The default client_max_body_size is 1m. Clients uploading
+        # files larger than this will need to chunk said files.
+        client_max_body_size 10m;
+
+        # Gunicorn docs suggest this value.
+        keepalive_timeout 5;
+
+        location {{ content_path }} {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-content;
+        }
+
+        location {{ api_root }}api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            proxy_set_header Remoteuser $ssl_client_s_dn_cn;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            client_max_body_size 0;
+        }
+
+        {%- if domain_enabled | default(false) %}
+        location ~ {{ api_root }}.+/api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            client_max_body_size 0;
+        }
+        {%- endif %}
+
+        location /auth/login/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+        }
+
+        include pulp/*.conf;
+
+        location / {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            # most pulp static files are served through whitenoise
+            # http://whitenoise.evans.io/en/stable/
+        }
+
+        {%- if https | default(false) %}
+        # ACME http-01 tokens, i.e, for Let's Encrypt
+        location /.well-known/ {
+            try_files $uri $uri/ =404;
+        }
+        {%- endif %}
+    }
+    {%- if https | default(false) %}
+    server {
+        listen 80 default_server;
+        server_name _;
+        return 301 https://$host$request_uri;
+    }
+    {%- endif %}
+}

.ci/run_container.sh

@@ -16,11 +16,12 @@ then
 fi
 export CONTAINER_RUNTIME
 
-TMPDIR="$(mktemp -d)"
+PULP_CLI_TEST_TMPDIR="$(mktemp -d)"
+export PULP_CLI_TEST_TMPDIR
 
 cleanup () {
   "${CONTAINER_RUNTIME}" stop pulp-ephemeral && true
-  rm -rf "${TMPDIR}"
+  rm -rf "${PULP_CLI_TEST_TMPDIR}"
 }
 
 trap cleanup EXIT
@@ -48,8 +49,8 @@ else
     SELINUX=""
 fi;
 
-mkdir -p "${TMPDIR}/settings/certs"
-cp "${BASEPATH}/settings/settings.py" "${TMPDIR}/settings"
+mkdir -p "${PULP_CLI_TEST_TMPDIR}/settings/certs"
+cp "${BASEPATH}/settings/settings.py" "${PULP_CLI_TEST_TMPDIR}/settings"
 
 if [ -z "${PULP_HTTPS:+x}" ]
 then
@@ -60,10 +61,16 @@ else
   PROTOCOL="https"
   PORT="443"
   PULP_CONTENT_ORIGIN="https://localhost:8080/"
-  python3 -m trustme -d "${TMPDIR}/settings/certs"
-  export PULP_CA_BUNDLE="${TMPDIR}/settings/certs/client.pem"
-  ln -fs server.pem "${TMPDIR}/settings/certs/pulp_webserver.crt"
-  ln -fs server.key "${TMPDIR}/settings/certs/pulp_webserver.key"
+  python3 "${BASEPATH}/gen_certs.py" -d "${PULP_CLI_TEST_TMPDIR}/settings/certs"
+  export PULP_CA_BUNDLE="${PULP_CLI_TEST_TMPDIR}/settings/certs/ca.pem"
+  ln -fs server.pem "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.crt"
+  ln -fs server.key "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.key"
+  {
+    echo "AUTHENTICATION_BACKENDS = '@merge django.contrib.auth.backends.RemoteUserBackend'"
+    echo "MIDDLEWARE  = '@merge django.contrib.auth.middleware.RemoteUserMiddleware'"
+    echo "REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES = '@merge pulpcore.app.authentication.PulpRemoteUserAuthentication'"
+    echo "REMOTE_USER_ENVIRON_NAME = 'HTTP_REMOTEUSER'"
+  } >> "${PULP_CLI_TEST_TMPDIR}/settings/settings.py"
 fi
 export PULP_CONTENT_ORIGIN
 
@@ -75,7 +82,9 @@ export PULP_CONTENT_ORIGIN
   --env PULP_CONTENT_ORIGIN \
   --detach \
   --name "pulp-ephemeral" \
-  --volume "${TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \
+  --volume "${PULP_CLI_TEST_TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \
+  --volume "${BASEPATH}/nginx.conf.j2:/nginx/nginx.conf.j2${SELINUX:+:Z}" \
+  --network bridge \
   --publish "8080:${PORT}" \
   "ghcr.io/pulp/pulp:${IMAGE_TAG}"
 
@@ -104,7 +113,7 @@ done
 "${CONTAINER_RUNTIME}" exec "pulp-ephemeral" pulpcore-manager reset-admin-password --password password
 
 # Create pulp config
-PULP_CLI_CONFIG="${TMPDIR}/settings/certs/cli.toml"
+PULP_CLI_CONFIG="${PULP_CLI_TEST_TMPDIR}/settings/certs/cli.toml"
 export PULP_CLI_CONFIG
 pulp config create --overwrite --location "${PULP_CLI_CONFIG}" --base-url "${PROTOCOL}://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password"
 # show pulpcore/plugin versions we're using

.ci/scripts/create_release_branch.sh

@@ -26,3 +26,8 @@ find CHANGES/ \( -name "*.feature" -o -name "*.bugfix" -o -name "*.removal" -o -
 bump-my-version bump minor --commit --message $'Bump version to {new_version}' --allow-dirty
 
 git push origin "${NEW_BRANCH}"
+
+if [ "${GITHUB_ENV:-}" ]
+then
+  echo "NEW_BRANCH=${NEW_BRANCH}" >> "${GITHUB_ENV}"
+fi

.ci/settings/settings.py

@@ -1,4 +1,3 @@
 ALLOWED_EXPORT_PATHS = ["/tmp"]
-ORPHAN_PROTECTION_TIME = 0
 ANALYTICS = False
 ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha256", "sha512"]

.github/workflows/build.yml

@@ -1,3 +1,4 @@
+---
 name: "Build"
 
 on:
@@ -34,3 +35,4 @@ jobs:
             dist/
           if-no-files-found: "error"
           retention-days: 5
+...

.github/workflows/codeql.yml

@@ -1,3 +1,4 @@
+---
 name: "CodeQL"
 
 on:
@@ -27,3 +28,4 @@ jobs:
         uses: "github/codeql-action/analyze@v3"
         with:
           category: "/language:python"
+...

.github/workflows/collect_changes.yml

@@ -1,3 +1,4 @@
+---
 name: "Collect changes"
 on:
   workflow_call:
@@ -30,3 +31,4 @@ jobs:
           body: ""
           branch: "update_changes"
           delete-branch: true
+...

.github/workflows/lint.yml

@@ -1,3 +1,4 @@
+---
 name: "Lint"
 
 on:
@@ -35,3 +36,4 @@ jobs:
       - name: "Lint code"
         run: |
           make lint
+...

.github/workflows/nightly.yml

@@ -1,3 +1,4 @@
+---
 name: "pulp-cli Nightly"
 
 on:
@@ -17,3 +18,4 @@ jobs:
   collect_changes:
     uses: "./.github/workflows/collect_changes.yml"
     secrets: "inherit"
+...

.github/workflows/pr.yml

@@ -1,3 +1,4 @@
+---
 name: "pulp-cli CI"
 
 on:
@@ -64,3 +65,4 @@ jobs:
           echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result'
           echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0'
           echo "CI says: Looks good!"
+...

.github/workflows/pr_checks.yml

@@ -13,7 +13,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  single_commit:
+  apply_labels:
     runs-on: "ubuntu-latest"
     name: "Label PR"
     permissions:
@@ -57,3 +57,4 @@ jobs:
                 labels: ADD_LABELS.split(","),
               });
             }
+...

.github/workflows/publish.yml

@@ -1,3 +1,4 @@
+---
 name: "pulp-cli Publish"
 
 on:
@@ -35,3 +36,4 @@ jobs:
           twine upload dist/*
           cd ..
           twine upload dist/*
+...

.github/workflows/release.yml

@@ -1,3 +1,4 @@
+---
 name: "pulp-cli Release"
 
 on:
@@ -26,3 +27,4 @@ jobs:
       - name: "Release"
         run: |
           .ci/scripts/release.sh
+...