diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5c257dd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +local_universe_setup.sh +.git +.gitignore +docker-build.sh +marathon.json diff --git a/CHANGELOG.md b/CHANGELOG.md index e7f4de0..ed50d7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ Changelog =============== +0.0.0-2.0 - 7th December 2017 + +- Added synchronisation of the PKI (users, certificates and keys) between multiple running instances +- Enabled >1 instances to be started at the same time and match their local data +- Cleaned up the output to stdout +- Refactored a number of functions in run.sh to improve robustness +- Increased CPU resource from 0.1 to 1.0 due to DC/OS 1.10 now enforcing CPU usage - required for key generation. +- Fixed https://github.com/dcos-labs/dcos-openvpn/issues/13 +- Improved the function to find the public address +- Fixed the hostports in the marathon.json + 0.0.0-1.0 - 12th September 2017 - Changed znode path from dcos-vpn to openvpn diff --git a/Dockerfile b/Dockerfile index b98ce66..a5b28ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,17 @@ FROM kylemanna/openvpn MAINTAINER Richard Shaw -RUN apk -U add ca-certificates python python-dev py-setuptools alpine-sdk libffi libffi-dev openssl-dev +RUN apk -U add ca-certificates python python-dev \ + py-setuptools alpine-sdk libffi libffi-dev openssl-dev \ + haveged COPY . /dcos WORKDIR /dcos +RUN haveged -n 100g -f - | dd of=/dev/null RUN ["/usr/bin/python", "setup.py", "install"] RUN apk del alpine-sdk && \ apk fix openssl && \ rm -rf /tmp/* /var/tmp/* /var/cache/apk/* /var/cache/distfiles/* -EXPOSE 5000 1194/tcp 1194/udp -CMD ["bin/run.sh", "run_server"] \ No newline at end of file +EXPOSE 5000/tcp 1194/udp +CMD ["bin/run.sh", "run_server"] diff --git a/README.md b/README.md index 6d65de1..7970d6d 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,16 @@ DC/OS OpenVPN [![release](http://github-release-version.herokuapp.com/github/dcos-labs/dcos-openvpn/release.svg?style=flat)](https://github.com/dcos-labs/dcos-openvpn/releases/latest) +Please note: This is a [DC/OS Community package](https://dcos.io/community/), which is not formally tested or supported by Mesosphere. + OpenVPN server and REST management interface package for DC/OS. Please note: This is a [DC/OS Community package](https://dcos.io/community/), which is not formally tested or supported by Mesosphere. Issues and PRs are welcome. +Please review the Changelog for recent changes + Features -------------- @@ -22,21 +26,21 @@ Features 1. The Zookeeper znode `/openvpn` has ACLs enabled, to protect the OpenVPN server and client credentials 1. Synchronisation of assets between the container and Zookeeper in case the container is restarted 1. Clients revoked through the REST interface are correctly revoked from OpenVPN -1. Merged the previously separate openvpn server & openvpn-admin 0.0.0-0.1 packages into one. The openvpn-admin package is no longer required. +1. Merged the previously separate openvpn server & openvpn-admin 0.0.0-0.1 packages into one. The openvpn-admin package is no longer required Installation -------------- **You must configure the OVPN_USERNAME & OVPN_PASSWORD environment variables before installation** These are required for both the REST interface -credentials and for the Zookeeper znode ACL. +credentials and for the Zookeeper znode ACL. Please note, DC/OS 1.10 enforces CPU usage, key generation requires a full 1.0 CPU. This can be reduced back to 0.1 once up and running. ### DC/OS Public Universe Installation -1. From the **DC/OS Dashboard > Universe > Packages > enter openvpn in the search box** -1. Select **Install Package > Advanced Installation** and scroll down -1. Configure both the OVPN_USERNAME & OVPN_PASSWORD -1. Select **Review and Install > Install** +1. From the `DC/OS Dashboard > Universe > Packages > enter openvpn in the search box` +1. Select `Install Package > Advanced Installation` and scroll down +1. Configure both the `OVPN_USERNAME` & `OVPN_PASSWORD` +1. Select `Review and Install > Install` 1. The service is installed and initialises, when complete, it'll be marked as Running and Healthy 1. See Troubleshooting for any issues, otherwise go to Usage @@ -51,7 +55,6 @@ The task can be also be added as a package to a local Universe repository 1. Clone https://github.com/mesosphere/universe 1. Read https://docs.mesosphere.com/1.9/administering-clusters/deploying-a-local-dcos-universe/ -1. Read and amend the source of local_universe_setup.sh to facilitate building and publishing Usage @@ -59,22 +62,21 @@ Usage ### Endpoints -The exact endpoints can be confirmed from **DC/OS Dashboard > Services > OpenVPN > > Details** +The exact endpoints can be confirmed from `DC/OS Dashboard > Services > OpenVPN > > Details` -1. OpenVPN is presented on 1194/UDP and any OpenVPN client will default to this port -1. The REST management interface is available on 5000/TCP and will be accessed at https://:5000 +1. OpenVPN is presented on `1194/UDP` and any OpenVPN client will default to this port +1. The REST management interface is available on `5000/TCP` and will be accessed at `https://:5000` 1. /status /test /client are all valid REST endpoints. /status does not require authentication as it is used for health checks ### Add a User 1. Authenticate and POST to the REST endpoint, the new user's credentials will be output to the POST body ``` -curl -k -u username:password -X POST -d "name=richard" https://:5000/client +curl -k -u username:password -X POST -d "name=richard" https://:5000/client > richard.ovpn ``` -2. Copy the entire ouput and save to a single file - you may need to amend the target server IP if on an internal network -3. Save the file as dcos.ovpn and add to any suitable OpenVPN client, like [Tunnelblick](https://tunnelblick.net/) for macOS for example -4. Test connecting with the OpenVPN client. See Troubleshooting for help. -5. The new client credentials will be backed up to Zookeeper for persistence in case the task is killed, and will be copied back as required +2. Import the .ovpn file into any suitable OpenVPN client, Tunnelblick for macOS, for example +3. Test connecting with the OpenVPN client. See Troubleshooting for help +4. The new client credentials will be backed up to Zookeeper for persistence in case the task is killed, and will be synchronised with any other instances ### Revoke a User @@ -82,22 +84,22 @@ curl -k -u username:password -X POST -d "name=richard" https://:5000/client ``` curl -k -u username:password -X DELETE https://:5000/client/richard ``` -2. The client is correctly revoked from OpenVPN and the assets are removed from the container and Zookeeper +2. The client is correctly revoked from OpenVPN and the change is synchronised with all running instances ### Remove Zookeeper data -During installation, an ACL is set on the Zookeeper openvpn znode, restricting access based on the OVPN_USERNAME & OVPN_PASSWWORD credentials. +During installation, an ACL is set on the Zookeeper OpenVPN znode, restricting access based on the `OVPN_USERNAME` & `OVPN_PASSWWORD` credentials. In order to remove the znode data you must either authenticate with those same credentials or as the Zookeeper super user. Some examples of how to achieve this using zk-shell which is shipped in the Docker image: ``` zk-shell connect master.mesos:2181 (CONNECTED) / add_auth digest : -(CONNECTED) / rmr openvpn/ +(CONNECTED) / rmr /openvpn/ (CONNECTED) / exit ``` -If you intend to change the OVPN_USERNAME & OVPN_PASSWORD, you will need to change the ACL on the existing znode, then reinstall the package +If you intend to change the `OVPN_USERNAME` & `OVPN_PASSWORD`, you will need to change the ACL on the existing znode, then reinstall the package with new credentials ``` zk-shell connect master.mesos:2181 @@ -132,10 +134,19 @@ zkshrun.sh is a little standalone helper script that provides run_command to the A modified version of easyrsa is shipped which removes user prompts. +Synchronisation between multiple running instances is handled via a cron job, which runs every 2 minutes. It checks to see +if the `openvpn/pki/issue.txt` differs between localhost and in Zookeeper. If there's a diff, it signifies that a user has been created +or revoked by another instance which has been uploaded to Zookeeper. The full pki directory is copied down to update the local instance +and the ovpn daemon is restarted. + +This functionality is rudimentary and it's recommended not to add or revoke more than one user at a time and then leave >3 minutes between +each change to allow the synchronisation to work. + ### Startup order 1. run.sh checks for existing assets in Zookeeper and copies them to the container if they exist, otherwise initpki and genconfig are run 1. Launchs the OpenVPN daemon in daemon mode 1. Starts the Python REST interface +1. Synchronisation cron job every 2 minutes Troubleshooting @@ -143,7 +154,7 @@ Troubleshooting ### Service -1. Review stdout and stderr from the task's logs under the **DC/OS Dashboard > Service > openvpn > running task > logs** +1. Review stdout and stderr from the task's logs under the `DC/OS Dashboard > Service > openvpn > running task > logs` 2. If the task is running on DC/OS, find out which agent is running the service using the DC/OS cli `dcos task | grep openvpn` 4. SSH to that agent and get a shell on the running container ``` @@ -176,10 +187,8 @@ DC/OS to allow you to delete the root openvpn znode. Setting ZK credentials is r Todo -------------- -1. Get defined host ports working in the marathon.json - works in the Universe marathon template 1. The patch for zk-shell https://github.com/rgs1/zk_shell/pull/82 as managed in run.bash around line 100 needs removing when zk-shell is fixed 1. Update the /status endpoint for ovpn_status output and tie into a healthcheck -1. run.sh usage and tidying 1. Update for DC/OS 1.10 and file based secrets 1. Either extend zk-shell to add auth to its params or replace with Kazoo code 1. Replace the location function which calls out to ifconfig.me as it's of no use for internal networks diff --git a/bin/envs.sh b/bin/envs.sh index 6451d7f..1ebb5cb 100755 --- a/bin/envs.sh +++ b/bin/envs.sh @@ -6,3 +6,4 @@ export ZKURL=${ZKURL:="master.mesos:2181"} export CONFIG_LOCATION=${CONFIG_LOCATION:="/etc/openvpn"} export HOST=${HOST:=127.0.0.1} export PORT0=${PORT0:=6000} +export OVPN_PORT=${OVPN_PORT:=1194} diff --git a/bin/run.sh b/bin/run.sh index 2782271..7a10d03 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -4,25 +4,8 @@ # Vars, checks and admin ############################## -container_files=0 -zookeeper_path=0 source /dcos/bin/envs.sh -# Check to see if the container already has the openvpn files locally -# And the second checks for an existing znode in Zookeeper which suggests -# This is being re-run after a previous setup - -function check_status { - if [ -f $CONFIG_LOCATION/openvpn.conf ]; then - container_files=1 - fi - if [[ -z $(run_command "ls $ZKPATH/openvpn.conf") ]]; then - zookeeper_path=1 - fi - echo "container_files = " $container_files - echo "zookeeper_path = " $zookeeper_path -} - # Workaround to pass add_auth on the one liner as zk-shell doens't provide this as a param function run_command { @@ -30,97 +13,167 @@ function run_command { return $? } +function fix_scripts { + # Fix a bug in zk-shell copy that's pending a pull request + sed -i 's/return PathValue("".os.path.join(fph.readlines()))/return PathValue("".join(os.path.join(fph.readlines())))/g' /usr/lib/python2.7/site-packages/zk_shell-1.1.3-py2.7.egg/zk_shell/copy.py + + # Replace the shipped easyrsa with our easyrsa to remove the revoke confirmation + sed -i 's/easyrsa/\/dcos\/bin\/easyrsa/g' /usr/local/bin/ovpn_revokeclient +} + ############################## # File download and upload ############################## function download_files { - ZKPATH_STRIPPED=$(echo $ZKPATH | sed -e 's/^\///') - for fname in $(run_command "find / $ZKPATH_STRIPPED"); do - local sub_path=$(echo $fname | cut -d/ -f3-) + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then + ZKPATH_STRIPPED=$(echo $ZKPATH | sed -e 's/^\///') + for fname in $(run_command "find / $ZKPATH_STRIPPED"); do + local sub_path=$(echo $fname | cut -d/ -f3-) - # If the sub_path is empty, there's no reason to copy - [[ -z $sub_path ]] && continue + # If the sub_path is empty, there's no reason to copy + [[ -z $sub_path ]] && continue - if [ "$sub_path" == "Failed" ]; then - err "Unable to get data from $ZKURL$ZKPATH. Check your zookeeper." - fi + if [ "$sub_path" == "Failed" ]; then + err "Unable to get data from $ZKURL$ZKPATH. Check your zookeeper." + fi - local fs_path=$CONFIG_LOCATION/$sub_path - run_command "cp $fname file://$fs_path" > /dev/null 2>&1 - # Directories are copied as empty files, remove them so that the - # subsequent copies actually work. - [ -s $fs_path ] || rm $fs_path - done + local fs_path=$CONFIG_LOCATION/$sub_path + run_command "cp $fname file://$fs_path false true false false" > /dev/null 2>&1 + # Directories are copied as empty files, remove them so that the + # subsequent copies actually work. + [ -s $fs_path ] || rm $fs_path + done + else + echo "INFO: Upload marker found, leaving until next cron run" + fi } function upload_files { - if [ $zookeeper_path = 0 ]; then - run_command "create $ZKPATH '' false false true" - run_command "set_acls /$ZKPATH username_password:$OVPN_USERNAME:$OVPN_PASSWORD:cdrwa" - fi + + # Adding a marker so we know when all the files have been uploaded + run_command "create $ZKPATH/upload_marker ''" + for fname in $(find $CONFIG_LOCATION -not -type d); do local zk_location=$(echo $fname | sed 's|'$CONFIG_LOCATION'/|/|') - run_command "cp file://$fname $ZKPATH$zk_location" + run_command "cp file://$fname $ZKPATH$zk_location false true false true" > /dev/null 2>&1 done + + # Removing upload marker + run_command "rm $ZKPATH/upload_marker" } + +############################## +# Synchronise +############################## + +function synchronise { + index_on_zk="/openvpn/pki/index.txt" + index_on_local="/etc/openvpn/pki/index.txt" + index_tmp="/tmp/index.txt" + + rm -f $index_tmp + run_command "cp $index_on_zk file://$index_tmp false true false true" > /dev/null 2>&1 + if [[ $(diff -q $index_on_local $index_tmp) != "" ]]; then + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then + echo "INFO: Zookeeper has a new dataset, downloading and restarting OpenVPN to apply" + download_files + pkill openvpn + ovpn_run --daemon + set_public_location + else + echo "INFO: Upload marker found, will attempt on next cron run" + fi + else + echo "INFO: No changes found on Zookeeper" + fi +} + + ############################## # Location set and get ############################## function get_location { - echo $(run_command "get $ZKPATH/location.conf") + cat /etc/openvpn/location.conf } function set_public_location { - local loc=$ZKPATH/location.conf source $OPENVPN/ovpn_env.sh - if run_command "ls $loc" ; then - run_command "set $loc \"remote $(wget -O - -U curl ifconfig.me) $PORT0 $OVPN_PROTO\"" - else - run_command "create $loc ''" - set_public_location - fi + echo "INFO: Setting public location" + echo "remote $(wget -q -O - -U curl ipinfo.io/ip) $PORT1 $OVPN_PROTO" > /etc/openvpn/location.conf } + ############################## # Main setup ############################## +function create_zkpath { + if [[ $(run_command "find /openvpn") = "" ]]; then + echo "INFO: Creating the zkpath if it doesn't already exist" + run_command "create $ZKPATH '' false false true" + run_command "set_acls /$ZKPATH username_password:$OVPN_USERNAME:$OVPN_PASSWORD:cdrwa" + fi +} + function build_configuration { - ovpn_genconfig -u udp://$CA_CN - rm -rf $CONFIG_LOCATION/pki + # Adding a lock to stop any other instances trying to upload at the same time + echo "INFO: Creating lock file" + run_command "create $ZKPATH/upload_marker ''" > /dev/null 2>&1 + echo "INFO: Resetting container" + reset_container + echo "INFO: Building configuration" + ovpn_genconfig -u udp://$CA_CN > /dev/null 2>&1 + echo "INFO: Building PKI" (echo $CA_CN) | PATH=/dcos/bin:$PATH ovpn_initpki nopass + touch /etc/openvpn/complete } function setup { - - # Fix a bug in zk-shell copy that's pending a pull request - sed -i 's/return PathValue("".os.path.join(fph.readlines()))/return PathValue("".join(os.path.join(fph.readlines())))/g' /usr/lib/python2.7/site-packages/zk_shell-1.1.3-py2.7.egg/zk_shell/copy.py - - # Replace the shipped easyrsa with our easyrsa to remove the revoke confirmation - sed -i 's/easyrsa/\/dcos\/bin\/easyrsa/g' /usr/local/bin/ovpn_revokeclient - - if [ $zookeeper_path = 1 ] && [ $container_files = 0 ]; then - echo "Files found in Zookeeper - copying to container" - reset_container - download_files + # Introduce a random delay between 1-21 seconds in case of multiple instances starting at the same time + sleep $[ ( $RANDOM % 20 ) + 1 ]s + + create_zkpath + + if [[ $(run_command "find /openvpn/ complete") = "" ]]; then + echo "INFO: I didn't find a marker signifying a full dataset on Zookeeper" + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then + echo "INFO: I didn't find a lock" + build_configuration + echo "INFO: Uploading files to Zookeeper" + upload_files + set_public_location + else + echo "INFO: Lock found, will random sleep then try again" + setup + fi else - echo "Files not found in Zookeeper - generating and uploading" - reset - build_configuration - upload_files - set_public_location - fi + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then + reset_container + echo "INFO: Files found in Zookeeper, no lock found, downloading to container" + download_files + set_public_location + else + echo "INFO: Lock found, will random sleep then try again" + setup + fi + fi } function run_server { source /dcos/bin/envs.sh - check_status + fix_scripts setup + echo "INFO: Starting crond" + crond + echo "INFO: Adding cron job for synchronisation" + echo "*/2 * * * * /dcos/bin/run.sh synchronise >> /mnt/mesos/sandbox/stdout 2>&1" >> /etc/crontabs/root + echo "INFO: Starting OpenVPN daemon" ovpn_run --daemon - /usr/bin/python -m dcos_openvpn.main + echo "INFO: Starting Python REST interface" + /usr/bin/python -m dcos_openvpn.main } function reset { @@ -138,11 +191,11 @@ case "$@" in setup) setup ;; download_files) download_files ;; upload_files) upload_files ;; - check_status) check_status ;; build_configuration) build_configuration ;; set_public_location) set_public_location ;; get_location) get_location ;; reset) reset ;; reset_container) reset_container ;; + synchronise) synchronise ;; *) exit 1 ;; esac diff --git a/dcos_openvpn/cert.py b/dcos_openvpn/cert.py index 37cb8f8..7fa5058 100644 --- a/dcos_openvpn/cert.py +++ b/dcos_openvpn/cert.py @@ -7,6 +7,7 @@ OVPN_USERNAME = os.environ.get('OVPN_USERNAME') OVPN_PASSWORD = os.environ.get('OVPN_PASSWORD') +OVPN_PORT = os.environ.get('OVPN_PORT') CA_PASS = "nopass" @@ -33,8 +34,8 @@ def output(name): def remove(name): subprocess.check_call("ovpn_revokeclient {0} remove ".format(name), shell=True) - subprocess.check_call('/dcos/bin/zkshrun.sh "rmr /openvpn/pki"'.format(name), shell=True) - subprocess.check_call('/dcos/bin/zkshrun.sh "cp file:///etc/openvpn/pki /openvpn/pki true true"'.format(name), shell=True) + subprocess.check_call('/dcos/bin/run.sh upload_files'.format(name), shell=True) + def test(): diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index febf97f..0000000 --- a/docker-build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -docker build -t dcos-openvpn . -docker tag dcos-openvpn aggress/dcos-openvpn:$1 -docker push aggress/dcos-openvpn:$1 diff --git a/local_universe_setup.sh b/local_universe_setup.sh deleted file mode 100755 index 9a248e3..0000000 --- a/local_universe_setup.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Prerequisites -# Docker registry running and available by DC/OS -# docker run -d -p 5000:5000 --restart=always --name registry registry:2 -# Docker daemon on each DC/OS node configured to work with insecure registry -# https://docs.docker.com/registry/insecure/ or secure your registry - -function rebuild_universe { - dcos config set core.dcos_url https://192.168.33.11 - dcos auth login --username=admin --password=password - rm -rf /Users/richars/code/universe - cd /Users/richard/code - git clone https://github.com/mesosphere/universe --branch=version-3.x - cd universe - #cp -R repo/packages/O/openvpn/1 - #sed -i -e 's/mesosphere\/dcos-openvpn/aggress\/dcos-openvpn/g' repo/packages/O/openvpn/1/resource.json - #sed -i -e 's/0.0.0-0.1/0.0.0-0.2/g' repo/packages/O/openvpn/1/package.json -} - -function build { - cd /Users/richard/code/universe - scripts/build.sh - DOCKER_IMAGE="192.168.33.10:5000/universe-server" DOCKER_TAG="universe-server" docker/server/build.bash - DOCKER_IMAGE="192.168.33.10:5000/universe-server" DOCKER_TAG="universe-server" docker/server/build.bash publish - dcos marathon app add /Users/richard/code/universe/docker/server/target/marathon.json - dcos package repo add --index=0 universe-server http://universe.marathon.mesos:8085/repo -} - -function remove { - dcos package uninstall openvpn - dcos marathon app remove /universe - dcos package repo remove universe-server - docker image rm -f 192.168.33.10:5000/universe-server:universe-server - docker rmi -f $(docker images -aq) -} - -case "$@" in - remove) remove ;; - build) build ;; - rebuild_universe) rebuild_universe ;; - *) echo "build or remove"; exit 1 ;; -esac diff --git a/marathon.json b/marathon.json index 32a79c9..7ec8568 100644 --- a/marathon.json +++ b/marathon.json @@ -2,32 +2,35 @@ "id": "openvpn", "instances": 1, "portDefinitions": [], + "acceptedResourceRoles":[ + "slave_public" + ], "container": { "type": "DOCKER", "docker": { "portMappings": [ { - "hostport": 5000, + "hostPort": 5000, "containerPort": 5000, "protocol": "tcp", "name": "openvpn-admin" }, { - "hostport": 1194, + "hostPort": 1194, "containerPort": 1194, "protocol": "udp", "name": "openvpnudp" } ], "network": "BRIDGE", - "image": "aggress/dcos-openvpn:0.0.0-0.2", + "image": "aggress/dcos-openvpn:0.0.0-2.0", "forcePullImage": true, "privileged": true } }, "healthChecks": [ { - "gracePeriodSeconds": 120, + "gracePeriodSeconds": 360, "intervalSeconds": 30, "timeoutSeconds": 5, "maxConsecutiveFailures": 3, @@ -37,7 +40,7 @@ "ignoreHttp1xx": false } ], - "cpus": 0.1, + "cpus": 1, "mem": 128, "requirePorts": false, "env": {