diff --git a/dev/compose/README.md b/dev/compose/README.md new file mode 100644 index 0000000000..c442574720 --- /dev/null +++ b/dev/compose/README.md @@ -0,0 +1,113 @@ +# Galaxy Simplified Compose Stack + +Profiles: + +- `aap` - Run galaxy_ng for integration with Ansible Automation Platform and Resource Server +- `community` - Run galaxy_ng for galaxy.ansible.com development +- `cloud` - Run galaxy_ng for console.redhat.com development + +## Requirements + +- `docker compose` version `>=2` + +## Usage + +Pick a profile as needed and on the root of `galaxy_ng` repository. + +> Examples assumes `aap` as the profile, change as needed. + +Build images +```bash +docker compose -f dev/compose/aap.yaml build +``` + +Run the stack +```bash +docker compose -f dev/compose/aap.yaml up +# optionally pass `-d` to release the terminal +``` + +Exec commands on the `manager` service + +Bash +```console +$ docker compose -f dev/compose/aap.yaml exec manager /bin/bash +bash-4.4# +``` +Django Admin +```console +$ docker compose -f dev/compose/aap.yaml exec manager pulpcore-manager +Type 'pulpcore-manager help ' for help on a specific subcommand. + +Available subcommands: + +[app] + add-signing-service + analyze-publication +... +``` + +Settings +```console +$ docker compose -f dev/compose/aap.yaml exec manager dynaconf get DATABASES | python -m json.tool +{ + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "postgres", + "NAME": "galaxy_ng", + "PASSWORD": "galaxy_ng", + "PORT": 5432, + "USER": "galaxy_ng" + } +} +``` +```console +$ docker compose -f dev/compose/aap.yaml exec manager dynaconf list +CONTENT_ORIGIN 'https://localhost' +CACHE_ENABLED False +CACHE_SETTINGS {'EXPIRES_TTL': 600} +ALLOWED_CONTENT_CHECKSUMS ['sha224', 'sha256', 'sha384', 'sha512'] +... +``` + +Stopping +```console +$ docker compose -f dev/compose/aap.yaml down +# add -v to stop and remove volumes +``` + + +## API Access + +Galaxy API is available on: + +[http://localhost:5001/api/galaxy/v3/swagger-ui/](http://localhost:5001/api/galaxy/v3/swagger-ui/) + +AAP UI and API will be available only if started separately on: + +[https://localhost](https://localhost) + +Ansible Hub UI can be started separately as a standalone `npm` run. + + +```console + +``` + +## Reload + +Changing `.py` and `.yaml` files on any of the `DEV_SOURCE_PATH` directories will trigger reload of `api`, `worker`, and `content` services. + + +## Tips and Tricks. + +**TBD** + + +### Debugging + +### Connecting to Database + +### Dumping and restoring the database + +### Populating testing data diff --git a/dev/compose/aap.yaml b/dev/compose/aap.yaml new file mode 100644 index 0000000000..5cc9c05954 --- /dev/null +++ b/dev/compose/aap.yaml @@ -0,0 +1,314 @@ +x-common-env: &common-env + + GNUPGHOME: /root/.gnupg/ + + DJANGO_SUPERUSER_USERNAME: admin + DJANGO_SUPERUSER_EMAIL: admin@example.com + DJANGO_SUPERUSER_PASSWORD: admin + + POSTGRES_USER: galaxy_ng + POSTGRES_PASSWORD: galaxy_ng + POSTGRES_DB: galaxy_ng + + # no spying + PULP_ANALYTICS: 'false' + + # normally goes into settings.py ... + PULP_DATABASES__default__ENGINE: django.db.backends.postgresql + PULP_DATABASES__default__NAME: galaxy_ng + PULP_DATABASES__default__USER: galaxy_ng + PULP_DATABASES__default__PASSWORD: galaxy_ng + PULP_DATABASES__default__HOST: postgres + PULP_DATABASES__default__PORT: 5432 + + PULP_DEBUG: 1 + PULP_GALAXY_DEPLOYMENT_MODE: 'standalone' + PULP_DEFAULT_FILE_STORAGE: "pulpcore.app.models.storage.FileSystem" + PULP_REDIRECT_TO_OBJECT_STORAGE: 'false' + + # Hostname and prefix has to be correct + PULP_GALAXY_API_PATH_PREFIX: '/api/galaxy/' + PULP_CONTENT_PATH_PREFIX: '/pulp/content/' + PULP_ANSIBLE_API_HOSTNAME: 'https://localhost' + PULP_ANSIBLE_CONTENT_HOSTNAME: "https://localhost" + PULP_CONTENT_ORIGIN: "https://localhost" + PULP_CSRF_TRUSTED_ORIGINS: "['https://localhost']" + + # signing ... + PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false' + PULP_GALAXY_REQUIRE_CONTENT_APPROVAL: 'true' + PULP_GALAXY_REQUIRE_SIGNATURE_FOR_APPROVAL: 'false' + PULP_GALAXY_COLLECTION_SIGNING_SERVICE: 'ansible-default' + PULP_GALAXY_CONTAINER_SIGNING_SERVICE: 'container-default' + + # pulp container ... + PULP_TOKEN_AUTH_DISABLED: 'false' + PULP_TOKEN_SERVER: 'https://localhost/token/' + PULP_TOKEN_SIGNATURE_ALGORITHM: 'ES256' + PULP_PUBLIC_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_public_key.pem' + PULP_PRIVATE_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_private_key.pem' + + # auth ... + PULP_GALAXY_AUTHENTICATION_CLASSES: "['galaxy_ng.app.auth.session.SessionAuthentication', 'ansible_base.jwt_consumer.hub.auth.HubJWTAuth', 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.BasicAuthentication']" + PULP_ANSIBLE_BASE_JWT_VALIDATE_CERT: 'false' + PULP_ANSIBLE_BASE_JWT_KEY: 'https://localhost' + PULP_GALAXY_FEATURE_FLAGS__external_authentication: 'true' + + # disable user/group modifications + PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'false' + + # role content workaround .. + PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false' + + # DEV EDITABLE STUFF + LOCK_REQUIREMENTS: 0 + DEV_SOURCE_PATH: "galaxy_ng" + # To enable editable installs of local checkouts change the variable above keeping the ordering as follows: + # DEV_SOURCE_PATH: "dynaconf:pulpcore:galaxy_importer:pulp_ansible:pulp_container:galaxy_ng:django-ansible-base" + + +services: + base_img: + build: + context: ../../ + dockerfile: Dockerfile + image: "localhost/galaxy_ng/galaxy_ng:base" + + redis: + image: "redis:5" + + postgres: + image: "postgres:13" + ports: + - '5433:5432' + environment: + <<: *common-env + healthcheck: + test: ["CMD", "pg_isready", "-U", "galaxy_ng"] + interval: 10s + retries: 5 + # Uncomment below to spam out every DB statement to the service stderr + # WARNING: enabling log_statement=all makes database slower + # command: ["postgres", "-c", "log_statement=ddl", "-c", "log_destination=stderr"] + + helper: # should this be moved to a custom Dockerfile based on base_img? + image: quay.io/centos/centos:stream9 + environment: + <<: *common-env + depends_on: + - postgres + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - "../../../:/src" + command: | + bash -c " + echo 'Moving required files to the required places.'; + + # Workarounds - Pulp SigningService can't access envvars; + /src/galaxy_ng/dev/compose/signing/setup_gpg_workarounds.sh; + cat /etc/pulp/certs/GNUPGHOME.workaround.txt; + cat /etc/pulp/certs/HOME.workaround.txt; + + # Keys; + cp /src/galaxy_ng/dev/compose/database/database_fields.symmetric.key /etc/pulp/certs/database_fields.symmetric.key; + cp /src/galaxy_ng/dev/compose/signing/signing-secret.key /etc/pulp/certs/signing-secret.key; + cp /src/galaxy_ng/dev/compose/signing/signing-secret.key.password.txt /etc/pulp/certs/signing-secret.key.password.txt; + cp /src/galaxy_ng/dev/compose/signing/signing-secret.key /etc/pulp/certs/signing-public.key; + find /etc/pulp/certs; + + # Scripts; + cp /src/galaxy_ng/dev/compose/signing/collection_sign.sh /var/lib/pulp/scripts/collection_sign.sh; + cp /src/galaxy_ng/dev/compose/signing/container_sign.sh /var/lib/pulp/scripts/container_sign.sh; + chmod +x /var/lib/pulp/scripts/*_sign.sh; + find /var/lib/pulp/scripts; + + echo 'DONE!'; + " + + migrations: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - "../../../:/src" + environment: + <<: *common-env + user: root + command: | + bash -c " + set -e; + rm -rf /var/lib/pulp/.migrated; + + /src/galaxy_ng/dev/compose/bin/wait /etc/pulp/certs/database_fields.symmetric.key; + /src/galaxy_ng/dev/compose/bin/devinstall; + + pulpcore-manager check --database default; + pulpcore-manager migrate; + pulpcore-manager shell < /src/galaxy_ng/dev/common/setup_test_data.py; + pulpcore-manager createsuperuser --noinput || true; + + touch /var/lib/pulp/.migrated; + " + + api: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - "../../../:/src" + environment: + <<: *common-env + extra_hosts: + localhost: "host-gateway" + networks: + - default + - service-mesh + user: root + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + /src/galaxy_ng/dev/compose/bin/devinstall; + + /src/galaxy_ng/dev/compose/bin/reloader /venv/bin/pulpcore-api + " + + content: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - "../../../:/src" + environment: + <<: *common-env + extra_hosts: + localhost: "host-gateway" + networks: + - default + - service-mesh + user: root + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + /src/galaxy_ng/dev/compose/bin/devinstall; + + /src/galaxy_ng/dev/compose/bin/reloader /venv/bin/pulpcore-content + " + + worker: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - "../../../:/src" + environment: + <<: *common-env + user: root + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + /src/galaxy_ng/dev/compose/bin/wait /etc/pulp/certs/signing-secret.key; + /src/galaxy_ng/dev/compose/bin/devinstall; + + # Worker needs gpg in order to consume signing tasks; + /src/galaxy_ng/dev/compose/signing/setup_gpg_keys.sh; + gpg --list-secret-keys; + + /src/galaxy_ng/dev/compose/bin/reloader /venv/bin/pulpcore-worker + " + + manager: + image: "localhost/galaxy_ng/galaxy_ng:base" + depends_on: + - base_img + - postgres + - helper + - migrations + - worker + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp:/var/lib/pulp" + - "../../../:/src" + environment: + <<: *common-env + user: root + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + /src/galaxy_ng/dev/compose/bin/wait /etc/pulp/certs/signing-secret.key; + /src/galaxy_ng/dev/compose/bin/devinstall; + + # Schedule resource sync; + pulpcore-manager task-scheduler --id dab_sync --interval 15 --path "galaxy_ng.app.tasks.resource_sync.run"; + curl -s -u admin:admin http://api:24817/api/galaxy/pulp/api/v3/task-schedules/?name=dab_sync | python -m json.tool; + + # Keys are needed to register signing services; + /src/galaxy_ng/dev/compose/signing/setup_gpg_keys.sh; + gpg --list-secret-keys; + + # Setup signing services; + /src/galaxy_ng/dev/compose/signing/setup_signing_services.sh; + curl -s -u admin:admin http://api:24817/api/galaxy/pulp/api/v3/signing-services/?fields=name,script,pubkey_fingerprint | python -m json.tool; + + # Setup repository gpgkey for upload verification; + /src/galaxy_ng/dev/compose/signing/setup_repo_keyring.sh; + + # Dev tools; SHOULD THIS MOVE TO A CUSTOM Dockerfile?; + /venv/bin/pip3.11 install ipython ipdb django-extensions; + + echo ' '; + echo '###################### API ROOT ##############################'; + curl -s http://api:24817/api/galaxy/ | python -m json.tool; + + echo '######################## READY ###############################'; + echo ' '; + echo 'API: http://localhost:5001/api/galaxy/v3/swagger-ui/'; + echo 'Django Admin CLI: docker compose -f dev/compose/aap.yaml exec manager pulpcore-manager'; + echo 'Settings list: docker compose -f ... exec manager dynaconf list'; + + # Keep it running indefinitely to enable `docker compose -f ... exec manager /bin/bash`; + tail -f /dev/null + " + + nginx: + image: "nginx:latest" + depends_on: + - base_img + - postgres + - helper + - migrations + - api + - content + ports: + - '5001:5001' + volumes: + - '../nginx/nginx.conf:/etc/nginx/nginx.conf:ro' + +volumes: + var_lib_pulp: + name: var_lib_pulp + etc_pulp_certs: + name: etc_pulp_certs + +networks: + service-mesh: + name: service-mesh diff --git a/dev/compose/bin/README.md b/dev/compose/bin/README.md new file mode 100644 index 0000000000..b447b359d2 --- /dev/null +++ b/dev/compose/bin/README.md @@ -0,0 +1,48 @@ +# Dev Binary Tools + +## Reloader + +A Script that splits `DEV_SOURCE_PATH` variable and list all files on its directories, +then omit some unwanted files watching for `.py|yaml` only for changes. + +Then it calls `entr` to reload the passed command. + +## entr + +A tool to watch files for changes and reload +This file is compiled specifically for the container in use. + +LINK: https://eradman.com/entrproject/ + +In case recompilation is needed, run inside the container. + +```bash +# Install entr so we can have reload on worker processes +# the checkout is to fix a compilation error caused by this commit +# https://github.com/eradman/entr/commit/f9ac92d17e42236fe6b5e8492e087620173c7b24 +RUN git clone https://github.com/eradman/entr && \ + cd entr && \ + git checkout 0d2d92d6052624a1e03a2a654e98e1c49f9955d9 && \ + cp Makefile.linux Makefile && \ + make +``` + +Then copy the generated `entr` binary. + +## devinstall + +Install all paths listed on `DEV_SOURCE_PATH` as editable every time the compose starts. + +## wait + +Wait until a file is present on the shared volumes, + +```console +$ wait /etc/pulp/certs/filename; +Waiting for /etc/pulp/certs/filename ... +# sleep 2 secs +Waiting for /etc/pulp/certs/filename ... +# sleep 2 secs +Waiting for /etc/pulp/certs/filename ... +... +``` diff --git a/dev/compose/bin/devinstall b/dev/compose/bin/devinstall new file mode 100755 index 0000000000..c43638434a --- /dev/null +++ b/dev/compose/bin/devinstall @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +local src_path_list +IFS=':' read -ra src_path_list <<< "$DEV_SOURCE_PATH" + +for item in "${src_path_list[@]}"; do + src_path="/src/${item}" + if [[ -d "$src_path" ]]; then + echo "Installing path ${item} in editable mode." + + if [[ "${LOCK_REQUIREMENTS}" -eq "1" ]]; then + pip3.11 install --no-cache-dir --no-deps --editable "$src_path" >/dev/null + else + pip3.11 install --no-cache-dir --editable "$src_path" >/dev/null + fi + + else + echo "WARNING: Source path ${item} is not a directory." + fi +done + diff --git a/dev/compose/bin/entr b/dev/compose/bin/entr new file mode 100755 index 0000000000..b9b3c55826 Binary files /dev/null and b/dev/compose/bin/entr differ diff --git a/dev/compose/bin/reloader b/dev/compose/bin/reloader new file mode 100755 index 0000000000..0a26a1e379 --- /dev/null +++ b/dev/compose/bin/reloader @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# List all files in the DEV_SOURCE_PATH and watch for changes and use entr to SIGKILL + restart +echo "Watching $DEV_SOURCE_PATH" + +find $(echo $DEV_SOURCE_PATH | tr ':' '\n' | while read item; do echo -n /src/$item\ ; done) \( -path /src/galaxy_ng/.venv -o -path /src/galaxy_ng/build -o -path /src/galaxy_ng/.eggs \) -prune -o -name '*.py' -o -name '*.yaml' | /src/galaxy_ng/dev/compose/bin/entr -n -r timeout -k 5 0 $1 diff --git a/dev/compose/bin/wait b/dev/compose/bin/wait new file mode 100755 index 0000000000..65d08290e0 --- /dev/null +++ b/dev/compose/bin/wait @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +while [[ ! -f $1 ]]; do + echo "Waiting for $1 ..."; + sleep 2; +done; diff --git a/dev/compose/database/README.md b/dev/compose/database/README.md new file mode 100644 index 0000000000..d878677e75 --- /dev/null +++ b/dev/compose/database/README.md @@ -0,0 +1,21 @@ +# Database Field Encryption + +Pulp relies on the file `database_fields.symmetric.key` being on `/etc/pulp/certs` +For development purposes there is a hardcoded key on this folder. + +```bash +mkdir -p /etc/pulp/certs/; +cp database_fields.symmetric.key /etc/pulp/certs/database_fields.symmetric.key +``` + +> NOTE: For development it is better to use a persistent key, so database dumps +> can be easily restored across dev environments. + +## Generating a new key + +```bash +rpm -q openssl || dnf -y install openssl; +mkdir -p /etc/pulp/certs/; +openssl rand -base64 32 > /etc/pulp/certs/database_fields.symmetric.key; +chmod 640 /etc/pulp/certs/database_fields.symmetric.key; +``` diff --git a/dev/compose/database/database_fields.symmetric.key b/dev/compose/database/database_fields.symmetric.key new file mode 100644 index 0000000000..14697a0310 --- /dev/null +++ b/dev/compose/database/database_fields.symmetric.key @@ -0,0 +1 @@ +DNmNdwgyZugTax9S64J0FITTr9IHPxbuoF1F1CGPr68= diff --git a/dev/compose/signing/README.md b/dev/compose/signing/README.md new file mode 100644 index 0000000000..5341520bda --- /dev/null +++ b/dev/compose/signing/README.md @@ -0,0 +1,181 @@ +# Node Setup + +Requirements: + +- system has a running `gpg-agent` +- required files are in place + +```bash +# Keys +/etc/pulp/certs/signing-secret.key +/etc/pulp/certs/signing-secret.key.password.tx +/etc/pulp/certs/signing-public.key + +# Scripts +/var/lib/pulp/scripts/collection_sign.sh +/var/lib/pulp/scripts/container_sign.sh + +# Workarounds for Pulp envvar limitation. +# required only if GPGHOME differs from running user HOME +# ./setup_gpg_workarounds.sh +/etc/pulp/certs/HOME.workaround.txt +/etc/pulp/certs/GNUPGHOME.workaround.txt +``` + +## Workers + +```bash +./setup_gpg_keys.sh +``` + +## Run once, on any node, after workers are alive + +```bash +./setup_gpg_keys.sh +./setup_signing_services.sh +./setup_repo_keyring.sh +``` + +--- + +# How Signing Works + +## 1. GPG and keys + +System needs a GPGHOME and a KEYRING + +Usually: + +```bash +~/.gnupg/pubring.kbx +``` + +> NOTE: If `GNUPGHOME` differs from the current user `HOME`, then it is required to save the path to `/etc/pulp/certs/HOME.workaround.txt` and `/etc/pulp/certs/GNUPGHOME.workaround.txt` as a workaround due to a limitation on Pulp SigningService that cannot access external environment variables. + +The keyring must have the secret-key for signing and public key for verification. +The files are located on this directory and must be copied to `/etc/pulp/certs` + +> These keys are for development purposes only! NEVER USE THOSE IN PRODUCTION! + +- cp `signing-secret.key` -> `/etc/pulp/certs/signing-secret.key` +- cp `signing-public.key` -> `/etc/pulp/certs/signing-public.key` + +Key information: + +- fingerprint is `FB8B3F2D24BCAF7EFDF793A9F37575C52D4F16F3` +- short id `F37575C52D4F16F3` +- admin ID `galaxydev@ansible.com` +- passphrase `Galaxy2024` + +The passphrase must be added to a file named `/etc/pulp/certs/signing-secret.key.password.txt` + +```bash +echo "Galaxy2024" > /etc/pulp/certs/signing-secret.key.password.txt +``` + +To add both keys to the keyring start the agent and run: + +```bash +gpgconf --kill gpg-agent && gpg --batch --no-default-keyring --import /etc/pulp/certs/signing-secret.key; +``` + +it is also require to adjust the key trust level: + +```bash +(echo 5; echo y; echo save) | gpg --command-fd 0 --no-tty --no-greeting -q --edit-key 'FB8B3F2D24BCAF7EFDF793A9F37575C52D4F16F3' +``` + +Ensure key is added and trusted + +```bash +gpg --list-secret-keys +``` + +Must output a `[ultimate]` trust level key. + + +## 2. Signing Scripts and Signing Service + +> For collection the script is `collection_sign.sh` and for containers `container_sign.sh`, both +> located on this directory, the files must be copied to `/var/lib/pulp/scripts/` + +- cp `collection_sign.sh` -> `/var/lib/pulp/scripts/collection_sign.sh` +- cp `container_sign.sh` -> `/var/lib/pulp/scripts/container_sign.sh` + +> NOTE: The path for the signing service can be any path, `/var/lib/pulp/script` is just a convention. + +Use `pulpcore-manager add-signing-service` to register the signing service. + +- name: container-default | ansible-default +- script: path/to/executable that can access the GPG keyring +- key: fingerprint_of_gpg_key that lives on the keyring + +Examples: + +For collections: + +```bash +pulpcore-manager add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh F37575C52D4F16F3 +``` + +For containers: + +```bash +pulpcore-manager add-signing-service container-default /var/lib/pulp/scripts/container_sign.sh F37575C52D4F16F3 --class container:ManifestSigningService; +``` + +> NOTE: The command above will actually try to sign an arbitrary artifact in order to validate the key. + +## 3. Signing Collections and Containers + +Galaxy will call the `sign` endpoint passing the following information: + +- content unit id: +- signing_service: foo +- repository: + +Then: + +Signature content is created based on artifact and added to the same +repo as the content and contains a foreign_key to the content signed. + +## 5. Public keys and Validation + +### Collection + +Galaxy exposes collection signature on UI and API. + +### Container + +For Containers the signature is added to the registry extended API so usually +clients such as `podman` will automatically fetch and validate the signature +if configured with a proper `policies.json` file. + +### Public Key + +When validating a signature the client `ansible-galaxy` will needs to have +the public key on a local keyring, there are 2 ways to get the public key. + +- On Galaxy UI there is a menu on sidebar `Signature Keys` that exposes +the public keys and user can download it. + +- On API there is the `/signing-services` API to expose the same information. + +> NOTE: User needs to create the keyring locally and import the key manually. + +## 6. Signature Upload + +Galaxy can be alternatively configured to accept signature upload instead +of relying on a local GPG keyring and SigningService to generate it. + +In this case the artifact can be externally signed (e.g using a hardware token) +and then signature uploaded during the approval dashboard process. + +1. Repository needs a `gpgkey` field containing a public key. + * `pulpcore-manager set-repo-keyring --repository staging --publickeypath /etc/pulp/certs/signing-public.key -y;` + * OR + * `pulpcore-manager set-repo-keyring --repository staging --keyring ~/.gnupg/pubring.kbx -y;` +2. CollectionVersion is uploaded to the repo and ends in `pending` state on approval dashboard when system has `REQUIRE_SIGNATURE_FOR_APPROVAL` +3. Signature is uploaded to the same repo, file matches the collectionversion (e.g `namespace-collection-1.2.3.asc`) name and repo uses `gpgkey` to verify the signature is valid. + +In this case there is no signing service involved, no need for GPG keyrings. diff --git a/dev/compose/signing/collection_sign.sh b/dev/compose/signing/collection_sign.sh new file mode 100755 index 0000000000..685309ad50 --- /dev/null +++ b/dev/compose/signing/collection_sign.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +GNUPGHOME=$(cat /etc/pulp/certs/GNUPGHOME.workaround.txt) + +gpg --lock-never \ + --quiet \ + --batch \ + --pinentry-mode loopback \ + --yes \ + --passphrase $(cat /etc/pulp/certs/signing-secret.key.password.txt) \ + --homedir "$GNUPGHOME" \ + --detach-sign \ + --default-key $PULP_SIGNING_KEY_FINGERPRINT \ + --armor \ + --output $1.asc \ + $1 + +[ $? -eq 0 ] && echo {\"file\": \"$1\", \"signature\": \"$1.asc\"} || exit $? diff --git a/dev/compose/signing/container_sign.sh b/dev/compose/signing/container_sign.sh new file mode 100755 index 0000000000..160a1d6b2b --- /dev/null +++ b/dev/compose/signing/container_sign.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +MANIFEST_PATH=$1 +HOME=$(cat /etc/pulp/certs/HOME.workaround.txt) + +skopeo standalone-sign \ + --passphrase-file /etc/pulp/certs/signing-secret.key.password.txt \ + --output $SIG_PATH \ + $MANIFEST_PATH $REFERENCE $PULP_SIGNING_KEY_FINGERPRINT + +[ $? -eq 0 ] && echo {\"signature_path\": \"$SIG_PATH\"} || exit $? diff --git a/dev/compose/signing/setup_gpg_keys.sh b/dev/compose/signing/setup_gpg_keys.sh new file mode 100755 index 0000000000..d20d018301 --- /dev/null +++ b/dev/compose/signing/setup_gpg_keys.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# ensure agent is running +gpgconf --kill gpg-agent + +# Import the key +gpg --batch --no-default-keyring --import /etc/pulp/certs/signing-secret.key; + +# Set the key trust level +(echo 5; echo y; echo save) | gpg --command-fd 0 --no-tty --no-greeting -q --edit-key 'FB8B3F2D24BCAF7EFDF793A9F37575C52D4F16F3' trust; diff --git a/dev/compose/signing/setup_gpg_workarounds.sh b/dev/compose/signing/setup_gpg_workarounds.sh new file mode 100755 index 0000000000..59a068d824 --- /dev/null +++ b/dev/compose/signing/setup_gpg_workarounds.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +DEFAULT_GNUPGHOME="$HOME/.gnupg" +CURRENT_GNUPGHOME="${GNUPGHOME:-$DEFAULT_GNUPGHOME}" + +# Remove both `.gnupg` and potential trailing `/` after `.gnupg` +CURRENT_HOME="${CURRENT_GNUPGHOME%/.gnupg/}" +CURRENT_HOME="${CURRENT_HOME%/.gnupg}" + +echo "$CURRENT_GNUPGHOME" > /etc/pulp/certs/GNUPGHOME.workaround.txt +echo "$CURRENT_HOME" > /etc/pulp/certs/HOME.workaround.txt diff --git a/dev/compose/signing/setup_repo_keyring.sh b/dev/compose/signing/setup_repo_keyring.sh new file mode 100755 index 0000000000..f51e048d8a --- /dev/null +++ b/dev/compose/signing/setup_repo_keyring.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +PUBLIC_KEY=/etc/pulp/certs/signing-public.key + +for repo in staging published; do + pulpcore-manager set-repo-keyring --repository $repo --publickeypath $PUBLIC_KEY -y; +done + diff --git a/dev/compose/signing/setup_signing_services.sh b/dev/compose/signing/setup_signing_services.sh new file mode 100755 index 0000000000..9de430badf --- /dev/null +++ b/dev/compose/signing/setup_signing_services.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Collection +HAS_COLLECTION_SIGNING=$(pulpcore-manager shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) +if [[ "$HAS_COLLECTION_SIGNING" -eq "0" ]]; then + pulpcore-manager add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh F37575C52D4F16F3 +else + echo "Collection Signing Service Already exists" +fi + +# Container +HAS_CONTAINER_SIGNING=$(pulpcore-manager shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="container-default").count())' 2>/dev/null || true) +if [[ "$HAS_CONTAINER_SIGNING" -eq "0" ]]; then + pulpcore-manager add-signing-service container-default /var/lib/pulp/scripts/container_sign.sh F37575C52D4F16F3 --class container:ManifestSigningService +else + echo "Container Signing Service Already exists" +fi diff --git a/dev/compose/signing/signing-public.key b/dev/compose/signing/signing-public.key new file mode 100644 index 0000000000..5a27c5c58d --- /dev/null +++ b/dev/compose/signing/signing-public.key @@ -0,0 +1,18 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBGcBkDMBCADcXzLFpIHjlPSNtNsQBvtnRCcqBUKUk7Xz8eZIHVIOt6lF3TPN +ZMfyyPhaihLlKzJQdhzDoc3/hdTKRxfBltrff9nYx5dHWegomVL1i/S0HAB7MOqn +Sj1dErkQEPhYbno1VI5ml71LMeu/anDmEkEhtvkVcqWnXs6DiiudU0Tmc4W/+lVH +Cluk+ISkg1/3pxLAQjSiPpWg/YB/cNDcuuHxueC0KqIHgGIrx0iDI8Tq7KlNbcOV +Cyj0Xpcv653dDlJZAA9O/ljfleNa08NxspRBEoMXhIKuOL/owVY65ZDCBBgg5GGH +uKGnFBITmKd9dWUVRezj7sKu5p4ErATgRi7lABEBAAG0JkdhbGF4eSBEZXYgS2V5 +IDxnYWxheHlkZXZAYW5zaWJsZS5jb20+iQFRBBMBCAA7FiEE+4s/LSS8r37995Op +83V1xS1PFvMFAmcBkDMCGwMFCwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AACgkQ +83V1xS1PFvPDmAgAwJJ/TPDrW9clGbObG6d3RTzM08R1Hvx1xRhpeWGB3uPag+zS +LmmgSHUskWWChRr1mcdIfH60HgdhR1PIk9sjKsTWorzS0Polit7Dd7pS+e1sFdGe +MFGX0ljDYF558yaX5NCVBstIaqxTuHc+SGOYT8/7w5P7/y7Li9OBO4vpPx6gq/RV +J1x6GnDELphGeyzAlPzevRwVr7h27kfIidI8vD2LYs3qV7scnOcW+hFcJ5utLee4 +4ia1BDCYDRgOgz4kMoz/rGQ4rR15aTclihc0+puZJrvsUQMPS1+q9XoJPJs+s+Gr +kY24YID/BNUXy0j/BR5vQQvCSCaWdBDGyFvV7A== +=0hZK +-----END PGP PUBLIC KEY BLOCK----- diff --git a/dev/compose/signing/signing-secret.key b/dev/compose/signing/signing-secret.key new file mode 100644 index 0000000000..3408182cc6 --- /dev/null +++ b/dev/compose/signing/signing-secret.key @@ -0,0 +1,33 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQPGBGcBkDMBCADcXzLFpIHjlPSNtNsQBvtnRCcqBUKUk7Xz8eZIHVIOt6lF3TPN +ZMfyyPhaihLlKzJQdhzDoc3/hdTKRxfBltrff9nYx5dHWegomVL1i/S0HAB7MOqn +Sj1dErkQEPhYbno1VI5ml71LMeu/anDmEkEhtvkVcqWnXs6DiiudU0Tmc4W/+lVH +Cluk+ISkg1/3pxLAQjSiPpWg/YB/cNDcuuHxueC0KqIHgGIrx0iDI8Tq7KlNbcOV +Cyj0Xpcv653dDlJZAA9O/ljfleNa08NxspRBEoMXhIKuOL/owVY65ZDCBBgg5GGH +uKGnFBITmKd9dWUVRezj7sKu5p4ErATgRi7lABEBAAH+BwMCHPHopLXNy/H5x9sv +w+Ca3lAqKjqFX0MTJElj5dOndRgdUh3oUAoYfSB6/F3OHGIl3zlKhh5QrP8UUFjb +PFZjJYVoEU+ymBNucXMLBAymRAb8FxsDD94G75TYZxg+PXjQCnUbBxeeFoYl5vA2 +35KMZpGAIzhhNAjZlyFoX7GiMqdWxaKuUkR+NAi7cEWjhrgTE8gTtPfDl8aWN0if +2b2Jl8YXXlG32P+YN4qn0P91Z/nRAHPnrm0wGbDtcFCPGLGl2wXtoY2eb/2gwruU +r0uh3LEXq/27G6EfgX5ADikmXI4W6VdyCkkIRjbAv7jfou3jEseiy1jI2oGCvWVn +k0y79g04oTSjDGadW0gvZbGA87HndnS7p7S0oVmPVuhzKot894VH3TCCr1V3nO+Z +y7FfjdCDkN9bZ4ijoI0aXdOGZD1lbRHlq2UCaCe9ZKy27xFpqnkqlfuk59wjmcVJ +xv+w11kPh1QW51xY5quv+CXKFd3DF1L9auRvRBJP+UZa6Qq/2w5YZVPKLj9VF5oS +4iQON3l/4casojT/s3F7N1+HfsSZH6y3+WFr9up806GUte8eQUiqHxAnKfKdoUC9 +6izFeGWi6AL57vEiLp939RWh5AERnLHHAp4KpR4fDbwoBREbgqtBe2FHCJCnhqVl +/E/1Ydle1bXV/swpPI5lsKqjf2EhKK2hFUTmBlDkQjfN4tVU1BvC6s7HGD/F0OQu +loZghUisN66t9cA0rZHYzvd0yIknglDXfjgzj+rdXaMoaqOKV8IgEa/SwdNjLw5A +m/dFXCfz3AuezjHrdAw8y1+F/wRlezySFRmd6REStMgHtWJeRgJiv42ZgktBJziL +a671wp6c8bV0Ataa43WIBu7KZbq6NuiJnkZHNI7BILpENA98yV6WNetCb7SsZ6OQ +lPdzW0wp9BILtCZHYWxheHkgRGV2IEtleSA8Z2FsYXh5ZGV2QGFuc2libGUuY29t +PokBUQQTAQgAOxYhBPuLPy0kvK9+/feTqfN1dcUtTxbzBQJnAZAzAhsDBQsJCAcC +AiICBhUKCQgLAgQWAgMBAh4HAheAAAoJEPN1dcUtTxbzw5gIAMCSf0zw61vXJRmz +mxund0U8zNPEdR78dcUYaXlhgd7j2oPs0i5poEh1LJFlgoUa9ZnHSHx+tB4HYUdT +yJPbIyrE1qK80tD6JYrew3e6UvntbBXRnjBRl9JYw2BeefMml+TQlQbLSGqsU7h3 +PkhjmE/P+8OT+/8uy4vTgTuL6T8eoKv0VSdcehpwxC6YRnsswJT83r0cFa+4du5H +yInSPLw9i2LN6le7HJznFvoRXCebrS3nuOImtQQwmA0YDoM+JDKM/6xkOK0deWk3 +JYoXNPqbmSa77FEDD0tfqvV6CTybPrPhq5GNuGCA/wTVF8tI/wUeb0ELwkgmlnQQ +xshb1ew= +=hw1Q +-----END PGP PRIVATE KEY BLOCK----- diff --git a/dev/compose/signing/signing-secret.key.password.txt b/dev/compose/signing/signing-secret.key.password.txt new file mode 100644 index 0000000000..7e0c21b8b5 --- /dev/null +++ b/dev/compose/signing/signing-secret.key.password.txt @@ -0,0 +1 @@ +Galaxy2024 diff --git a/galaxy_ng/app/management/commands/set-repo-keyring.py b/galaxy_ng/app/management/commands/set-repo-keyring.py index fa3f9263b7..80bfd6163f 100644 --- a/galaxy_ng/app/management/commands/set-repo-keyring.py +++ b/galaxy_ng/app/management/commands/set-repo-keyring.py @@ -31,7 +31,10 @@ def echo(self, message, style=None): self.stdout.write(style(message)) def add_arguments(self, parser): - parser.add_argument("--keyring", type=str, help="Keyring", required=True) + parser.add_argument("--keyring", type=str, help="Keyring", required=False, default="") + parser.add_argument( + "--publickeypath", type=str, help="Path to Public Key File", required=False, default="" + ) parser.add_argument("--repository", type=str, help="Repository name", required=True) parser.add_argument( "-y", @@ -46,6 +49,14 @@ def handle(self, *args, **options): repository = options["repository"].strip() keyring = options["keyring"].strip() + publickey = options["publickeypath"].strip() + + if not keyring and not publickey: + self.echo("One of keyring or publickey is required") + exit(1) + if keyring and publickey: + self.echo("keyring or publickey are mutually exclusive") + exit(1) try: repo = AnsibleRepository.objects.get(name=repository) @@ -53,33 +64,36 @@ def handle(self, *args, **options): self.echo(f"Repository {repository} does not exist", self.style.ERROR) sys.exit(1) - certs_dir = settings.get("ANSIBLE_CERTS_DIR", "/etc/pulp/certs") - keyring_path = os.path.join(certs_dir, keyring) - if not os.path.exists(keyring_path): - self.echo(f"Keyring {keyring_path} does not exist", self.style.ERROR) - sys.exit(1) - - if not options["yes"]: - confirm = input( - f"This will set keyring to {keyring_path} for " - "{repository} repository, " "Proceed? (Y/n)" - ).lower() - while True: - if confirm not in ("y", "n", "yes", "no"): - confirm = input('Please enter either "y/yes" or "n/no": ') - continue - if confirm in ("y", "yes"): - break - else: - self.echo("Process canceled.") - return - - tempdir_path = tempfile.mkdtemp() - proc = subprocess.run([ - "gpg", "--homedir", tempdir_path, "--keyring", keyring_path, "--export", "-a" - ], capture_output=True) - - pubkey = proc.stdout.decode().strip() + if publickey: + with open(publickey) as pubkeyfile: + pubkey = pubkeyfile.read() + elif keyring: + certs_dir = settings.get("ANSIBLE_CERTS_DIR", "/etc/pulp/certs") + keyring_path = os.path.join(certs_dir, keyring) + if not os.path.exists(keyring_path): + self.echo(f"Keyring {keyring_path} does not exist", self.style.ERROR) + sys.exit(1) + + if not options["yes"]: + confirm = input( + f"This will set keyring to {keyring_path} for " + f"{repository} repository, " "Proceed? (Y/n)" + ).lower() + while True: + if confirm not in ("y", "n", "yes", "no"): + confirm = input('Please enter either "y/yes" or "n/no": ') + continue + if confirm in ("y", "yes"): + break + else: + self.echo("Process canceled.") + return + + tempdir_path = tempfile.mkdtemp() + proc = subprocess.run([ + "gpg", "--homedir", tempdir_path, "--keyring", keyring_path, "--export", "-a" + ], capture_output=True) + pubkey = proc.stdout.decode().strip() task = dispatch( set_repo_gpgkey,