From 3d216cf01b7e19e8cc108bec7af2ac0bf9960c53 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Tue, 8 Dec 2020 09:45:54 -0800 Subject: [PATCH 1/3] Creates scripts for regenerating rulesets Trying to bottle up the humdrum tasks into a single action, as far as possible. Some more guiding language about the specific steps of the airgap procedure would be welcome, but likely best handled in separate documentation. --- .gitignore | 3 +++ scripts/generate-and-sign | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100755 scripts/generate-and-sign diff --git a/.gitignore b/.gitignore index 0e8c263..bd036f0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ private.pem test-key.jwk public.pem +# Ignore upstream EFF repo +https-everywhere/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/scripts/generate-and-sign b/scripts/generate-and-sign new file mode 100755 index 0000000..75cf1f2 --- /dev/null +++ b/scripts/generate-and-sign @@ -0,0 +1,43 @@ +#!/bin/bash +# Utility script to generate the SecureDrop HTTPS Everywhere rulesets, +# used for managing Onion Names for SecureDrop instances. +# +# Much of the business logic is taken verbatim from the EFF HTTPSE repo: +# +# https://github.com/EFForg/https-everywhere/blob/master/docs/en_US/ruleset-update-channels.md#signing +# +set -e +set -u +set -o pipefail + + +# We need the upstream repo by EFF for a few select scripts. +https_everywhere_repo="https-everywhere" +if [[ ! -d "$https_everywhere_repo" ]]; then + echo "Cloning upstream https-everywhere repo for scripts..." + echo "WARNING: Can take a long time! ~10m even on fast connections." + git clone https://github.com/EFForg/https-everywhere +else + echo "Found https-everywhere repo locally, reusing..." +fi + +# Generate the SD rulesets +echo "Generating SecureDrop Onion Name rulesets..." +python3 sddir.py + +# The EFF scripts require paths to be relative, so copy into subdirs. +echo "Copying SecureDrop Onion Name rulesets ..." +rm -f "${https_everywhere_repo}/rules/"*.xml +cp rulesets/*.xml "${https_everywhere_repo}/rules/" +cp public_release.pem "${https_everywhere_repo}/" + +pushd "$https_everywhere_repo" +sd_rules_dir="securedrop-rules" +rm -rf "$sd_rules_dir" +mkdir "$sd_rules_dir" +docker run -it -v "$(pwd):/opt" --workdir /opt python:3.6 python3 utils/merge-rulesets.py +echo "Preparing rulesets for airgapped signature request..." +docker run -it -v "$(pwd):/opt" --workdir /opt python:3.6 utils/sign-rulesets/async-request.sh public_release.pem "$sd_rules_dir" + +echo "Finished. Review files in ${https_everywhere_repo}/${sd_rules_dir}/" +cp -v "${https_everywhere_repo}/${sd_rules_dir}/"* . From 72ee7df727af2d56e7001a85f8e92e18db288be1 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Fri, 18 Dec 2020 10:36:08 -0800 Subject: [PATCH 2/3] Touch-ups to signing script Based on feedback during review. * Don't use docker, just use system python * Rebuild index.html * Instruct user to commit changes after local review That's it for now. We can automatically 'git add' files once we have the ruleset generation sorted out wrt #20. --- scripts/generate-and-sign | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/generate-and-sign b/scripts/generate-and-sign index 75cf1f2..63409bb 100755 --- a/scripts/generate-and-sign +++ b/scripts/generate-and-sign @@ -31,13 +31,23 @@ rm -f "${https_everywhere_repo}/rules/"*.xml cp rulesets/*.xml "${https_everywhere_repo}/rules/" cp public_release.pem "${https_everywhere_repo}/" +# Switch to upstream subdir, for access to tooling pushd "$https_everywhere_repo" sd_rules_dir="securedrop-rules" rm -rf "$sd_rules_dir" mkdir "$sd_rules_dir" -docker run -it -v "$(pwd):/opt" --workdir /opt python:3.6 python3 utils/merge-rulesets.py +python3 utils/merge-rulesets.py echo "Preparing rulesets for airgapped signature request..." -docker run -it -v "$(pwd):/opt" --workdir /opt python:3.6 utils/sign-rulesets/async-request.sh public_release.pem "$sd_rules_dir" +./utils/sign-rulesets/async-request.sh public_release.pem "$sd_rules_dir" -echo "Finished. Review files in ${https_everywhere_repo}/${sd_rules_dir}/" +# Return to SD ruleset repo root +popd +echo "Copying rules to SecureDrop ruleset repo..." cp -v "${https_everywhere_repo}/${sd_rules_dir}/"* . + +echo "Updating index for SecureDrop rules..." +./update_index.sh + +echo "Finished. Please review local changes, and commit as appropriate." +# TODO: Not automatically running 'git add *' due to +# https://github.com/freedomofpress/securedrop-https-everywhere-ruleset/issues/20 From 1a326198a9f80ce13db70cc94fa7d43de54d71e6 Mon Sep 17 00:00:00 2001 From: mickael e Date: Wed, 23 Dec 2020 13:59:08 -0500 Subject: [PATCH 3/3] Update Readme to reflect changes introduced by the generate-and-sign script --- README.md | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 10e2a76..1128257 100644 --- a/README.md +++ b/README.md @@ -26,32 +26,15 @@ which will create `test-key.jwk` in your current working directory. 2. Add their domain name and the requested URL to the `onboarded.txt` via PR into this repository. We match the domain based on the landing page of the organization, comparing the `netloc` in a URL with structure `scheme://netloc/path;parameters?query#fragment`. -3. Next, perform a ruleset release as described below. - -### Updating the onion URL for an organization (e.g. if they transition to v3 or rotate URLs) - -1. First update their onion URL in the official SecureDrop directory using the existing process. - -2. Next, perform a ruleset release as described below. - -### Release process - -Generate rulesets via the securedrop.org directory using the `sddir.py` script: +3. Next, generate and sign the update ruleset using the following command (requires signing key, please ping `@emkll` for assistance): ``` -source .venv/bin/activate -python sddir.py +./scripts/generate-and-sign ``` -This populates the `rulesets` directory. Inspect them and check all looks sane. - -To sign the rules, see HTTPS Everywhere docs [here](https://github.com/EFForg/https-everywhere/blob/master/docs/en_US/ruleset-update-channels.md#2-signing-rulesets-with-this-key) for the signing process. In the step where you remove all HTTPS Everywhere rules from `rules` in the git checkout of the `https-everywhere` git repo, you should copy all rules from `rulesets` generated from the above Python script. You do not need to create a trivial rule as described in the HTTPS Everywhere docs. - -For the production rules this signing must be done via the official signing ceremony and the existing SD release key (JWK formatted version of the pubkey is in `release-pubkey.jwk`). There is some internal documentation with more detailed instructions on this, ping `@emkll` if you need to do this. - -Once you have the signature, place the files to serve in the root of the git tree in this repository, and then update the directory listing in `index.html` using the `update_index.sh` shell script in this directory. +4. Commit all files generated by the script above and open a Pull Request to this repository. Once the PR is merged, the rulesets will automatically be deployed to production. -# Verifying +## Verifying changes Inspect the diff. If it looks good, commit the resulting `index.html` and all files to be served. To test locally, run @@ -59,6 +42,6 @@ Inspect the diff. If it looks good, commit the resulting `index.html` and all fi And configure your browser to use `http://localhost:4080/https-everywhere/`. -# Deployment +## Deployment Upon merge the container will be published to `quay.io/freedomofpress` and the new tag will be deployed automatically.