From 5295d516bc29e09d18602b7d5d02d7a796410c9f Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Wed, 13 Sep 2017 14:16:00 +0100 Subject: [PATCH 01/20] Removed generic helper scripts Added .dockerignore --- .dockerignore | 5 +++++ docker-build.sh | 5 ----- local_universe_setup.sh | 43 ----------------------------------------- 3 files changed, 5 insertions(+), 48 deletions(-) create mode 100644 .dockerignore delete mode 100755 docker-build.sh delete mode 100755 local_universe_setup.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5c257dd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +local_universe_setup.sh +.git +.gitignore +docker-build.sh +marathon.json diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index febf97f..0000000 --- a/docker-build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -docker build -t dcos-openvpn . -docker tag dcos-openvpn aggress/dcos-openvpn:$1 -docker push aggress/dcos-openvpn:$1 diff --git a/local_universe_setup.sh b/local_universe_setup.sh deleted file mode 100755 index 9a248e3..0000000 --- a/local_universe_setup.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Prerequisites -# Docker registry running and available by DC/OS -# docker run -d -p 5000:5000 --restart=always --name registry registry:2 -# Docker daemon on each DC/OS node configured to work with insecure registry -# https://docs.docker.com/registry/insecure/ or secure your registry - -function rebuild_universe { - dcos config set core.dcos_url https://192.168.33.11 - dcos auth login --username=admin --password=password - rm -rf /Users/richars/code/universe - cd /Users/richard/code - git clone https://github.com/mesosphere/universe --branch=version-3.x - cd universe - #cp -R repo/packages/O/openvpn/1 - #sed -i -e 's/mesosphere\/dcos-openvpn/aggress\/dcos-openvpn/g' repo/packages/O/openvpn/1/resource.json - #sed -i -e 's/0.0.0-0.1/0.0.0-0.2/g' repo/packages/O/openvpn/1/package.json -} - -function build { - cd /Users/richard/code/universe - scripts/build.sh - DOCKER_IMAGE="192.168.33.10:5000/universe-server" DOCKER_TAG="universe-server" docker/server/build.bash - DOCKER_IMAGE="192.168.33.10:5000/universe-server" DOCKER_TAG="universe-server" docker/server/build.bash publish - dcos marathon app add /Users/richard/code/universe/docker/server/target/marathon.json - dcos package repo add --index=0 universe-server http://universe.marathon.mesos:8085/repo -} - -function remove { - dcos package uninstall openvpn - dcos marathon app remove /universe - dcos package repo remove universe-server - docker image rm -f 192.168.33.10:5000/universe-server:universe-server - docker rmi -f $(docker images -aq) -} - -case "$@" in - remove) remove ;; - build) build ;; - rebuild_universe) rebuild_universe ;; - *) echo "build or remove"; exit 1 ;; -esac From 2c19a00d7c7d7de1e85f74e3a74fb5e3e01ecab0 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Thu, 14 Sep 2017 08:44:51 +0100 Subject: [PATCH 02/20] Increased grace period in marathon.json --- marathon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon.json b/marathon.json index 32a79c9..e8b21df 100644 --- a/marathon.json +++ b/marathon.json @@ -27,7 +27,7 @@ }, "healthChecks": [ { - "gracePeriodSeconds": 120, + "gracePeriodSeconds": 900, "intervalSeconds": 30, "timeoutSeconds": 5, "maxConsecutiveFailures": 3, From b9cd98505f7639cdb1ae4f68737d9e6195b7d437 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Thu, 12 Oct 2017 20:31:51 +0100 Subject: [PATCH 03/20] Added haveged in an attempt to speed up cert generation as DC/OS 1.10 fixed CPU resource limits which means 0.1 will take >30 mins for the initial. Work in Progress Added marker to ZK to make file sync more robust Added env var for OVPN_PORT to fix client export from REST API defaulting to 5000 rather than 1194 Increased CPU quota in marathon.json from 0.1 to 1 because life is short --- Dockerfile | 9 ++++++--- bin/run.sh | 2 ++ dcos_openvpn/cert.py | 1 + marathon.json | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index b98ce66..a5b28ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,17 @@ FROM kylemanna/openvpn MAINTAINER Richard Shaw -RUN apk -U add ca-certificates python python-dev py-setuptools alpine-sdk libffi libffi-dev openssl-dev +RUN apk -U add ca-certificates python python-dev \ + py-setuptools alpine-sdk libffi libffi-dev openssl-dev \ + haveged COPY . /dcos WORKDIR /dcos +RUN haveged -n 100g -f - | dd of=/dev/null RUN ["/usr/bin/python", "setup.py", "install"] RUN apk del alpine-sdk && \ apk fix openssl && \ rm -rf /tmp/* /var/tmp/* /var/cache/apk/* /var/cache/distfiles/* -EXPOSE 5000 1194/tcp 1194/udp -CMD ["bin/run.sh", "run_server"] \ No newline at end of file +EXPOSE 5000/tcp 1194/udp +CMD ["bin/run.sh", "run_server"] diff --git a/bin/run.sh b/bin/run.sh index 2782271..6b87d0f 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -111,6 +111,8 @@ function setup { reset build_configuration upload_files + # Adding a marker so we know when all the files have been uploaded + run_command "create $ZKPATH/complete ''" set_public_location fi } diff --git a/dcos_openvpn/cert.py b/dcos_openvpn/cert.py index 37cb8f8..ab27846 100644 --- a/dcos_openvpn/cert.py +++ b/dcos_openvpn/cert.py @@ -7,6 +7,7 @@ OVPN_USERNAME = os.environ.get('OVPN_USERNAME') OVPN_PASSWORD = os.environ.get('OVPN_PASSWORD') +OVPN_PORT = os.environ.get('OVPN_PORT') CA_PASS = "nopass" diff --git a/marathon.json b/marathon.json index e8b21df..2d25db0 100644 --- a/marathon.json +++ b/marathon.json @@ -20,7 +20,7 @@ } ], "network": "BRIDGE", - "image": "aggress/dcos-openvpn:0.0.0-0.2", + "image": "aggress/dcos-openvpn:0.0.0-1.0", "forcePullImage": true, "privileged": true } @@ -37,7 +37,7 @@ "ignoreHttp1xx": false } ], - "cpus": 0.1, + "cpus": 1, "mem": 128, "requirePorts": false, "env": { From 6bdb2002413b267792bef5dac7f06075c450dbfb Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Thu, 12 Oct 2017 20:47:53 +0100 Subject: [PATCH 04/20] Force the local ovpn port --- bin/envs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/envs.sh b/bin/envs.sh index 6451d7f..1ebb5cb 100755 --- a/bin/envs.sh +++ b/bin/envs.sh @@ -6,3 +6,4 @@ export ZKURL=${ZKURL:="master.mesos:2181"} export CONFIG_LOCATION=${CONFIG_LOCATION:="/etc/openvpn"} export HOST=${HOST:=127.0.0.1} export PORT0=${PORT0:=6000} +export OVPN_PORT=${OVPN_PORT:=1194} From f0ce0d8a288053a22b7db82aa65fa9075d8bf32a Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Mon, 4 Dec 2017 15:34:51 +0000 Subject: [PATCH 05/20] Synchronisation --- bin/run.sh | 69 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 6b87d0f..53cb983 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -35,23 +35,24 @@ function run_command { ############################## function download_files { - ZKPATH_STRIPPED=$(echo $ZKPATH | sed -e 's/^\///') - for fname in $(run_command "find / $ZKPATH_STRIPPED"); do - local sub_path=$(echo $fname | cut -d/ -f3-) + if [[ $(run_command "ifind /openvpn/upload_marker") = "" ]]; then + ZKPATH_STRIPPED=$(echo $ZKPATH | sed -e 's/^\///') + for fname in $(run_command "find / $ZKPATH_STRIPPED"); do + local sub_path=$(echo $fname | cut -d/ -f3-) - # If the sub_path is empty, there's no reason to copy - [[ -z $sub_path ]] && continue + # If the sub_path is empty, there's no reason to copy + [[ -z $sub_path ]] && continue - if [ "$sub_path" == "Failed" ]; then - err "Unable to get data from $ZKURL$ZKPATH. Check your zookeeper." - fi + if [ "$sub_path" == "Failed" ]; then + err "Unable to get data from $ZKURL$ZKPATH. Check your zookeeper." + fi - local fs_path=$CONFIG_LOCATION/$sub_path - run_command "cp $fname file://$fs_path" > /dev/null 2>&1 - # Directories are copied as empty files, remove them so that the - # subsequent copies actually work. - [ -s $fs_path ] || rm $fs_path - done + local fs_path=$CONFIG_LOCATION/$sub_path + run_command "cp $fname file://$fs_path false true false true" + done + else + echo "Upload marker found, leaving until next cron run" + fi } function upload_files { @@ -59,12 +60,45 @@ function upload_files { run_command "create $ZKPATH '' false false true" run_command "set_acls /$ZKPATH username_password:$OVPN_USERNAME:$OVPN_PASSWORD:cdrwa" fi + + # Adding a marker so we know when all the files have been uploaded + run_command "create $ZKPATH/upload_marker ''" + for fname in $(find $CONFIG_LOCATION -not -type d); do local zk_location=$(echo $fname | sed 's|'$CONFIG_LOCATION'/|/|') - run_command "cp file://$fname $ZKPATH$zk_location" + run_command "cp file://$fname $ZKPATH$zk_location false true false true" done + + # Removing upload marker + run_command "rm $ZKPATH/upload_marker" } + +############################## +# Synchronise +############################## + +function synchronise { + index_on_zk="/openvpn/pki/index.txt" + index_on_local="/etc/openvpn/pki/index.txt" + index_tmp="/tmp/index.txt" + + rm -f $index_tmp + run_command "cp $index_on_zk file://$index_tmp false true false true" + if [[ $(diff -q $index_on_local $index_tmp) != "" ]]; then + if [[ $(run_command "ifind /openvpn/upload_marker") = "" ]]; then + download_files + pkill openvpn + ovpn_run --daemon + else + echo "Upload marker found will attempt on next cron run" + fi + else + echo "index.txt matches so nothing to do" + fi +} + + ############################## # Location set and get ############################## @@ -111,8 +145,6 @@ function setup { reset build_configuration upload_files - # Adding a marker so we know when all the files have been uploaded - run_command "create $ZKPATH/complete ''" set_public_location fi } @@ -121,6 +153,8 @@ function run_server { source /dcos/bin/envs.sh check_status setup + crond + echo "*/2 * * * * /dcos/bin/run.sh synchronise" >> /etc/crontabs/root ovpn_run --daemon /usr/bin/python -m dcos_openvpn.main } @@ -146,5 +180,6 @@ case "$@" in get_location) get_location ;; reset) reset ;; reset_container) reset_container ;; + synchronise) synchronise ;; *) exit 1 ;; esac From ffbf8e8a63ca2bb8035b17a5d24f58689a0ed3fa Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Mon, 4 Dec 2017 18:08:03 +0000 Subject: [PATCH 06/20] debuggig --- bin/run.sh | 3 +-- marathon.json | 7 +++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 53cb983..dc96fa2 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -48,7 +48,7 @@ function download_files { fi local fs_path=$CONFIG_LOCATION/$sub_path - run_command "cp $fname file://$fs_path false true false true" + run_command "cp $fname file://$fs_path true true false true" done else echo "Upload marker found, leaving until next cron run" @@ -145,7 +145,6 @@ function setup { reset build_configuration upload_files - set_public_location fi } diff --git a/marathon.json b/marathon.json index 2d25db0..21f9985 100644 --- a/marathon.json +++ b/marathon.json @@ -1,7 +1,10 @@ { - "id": "openvpn", + "id": "openvpn2", "instances": 1, "portDefinitions": [], + "acceptedResourceRoles":[ + "slave_public" + ], "container": { "type": "DOCKER", "docker": { @@ -27,7 +30,7 @@ }, "healthChecks": [ { - "gracePeriodSeconds": 900, + "gracePeriodSeconds": 360, "intervalSeconds": 30, "timeoutSeconds": 5, "maxConsecutiveFailures": 3, From 51e6914aac9d9fd591f47253e6747f47f2be36eb Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Mon, 4 Dec 2017 19:39:42 +0000 Subject: [PATCH 07/20] Debugging --- bin/run.sh | 5 ++++- marathon.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index dc96fa2..84ea13e 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -48,7 +48,10 @@ function download_files { fi local fs_path=$CONFIG_LOCATION/$sub_path - run_command "cp $fname file://$fs_path true true false true" + run_command "cp $fname file://$fs_path false true false true" + # Directories are copied as empty files, remove them so that the + # subsequent copies actually work. + [ -s $fs_path ] || rm $fs_path done else echo "Upload marker found, leaving until next cron run" diff --git a/marathon.json b/marathon.json index 21f9985..d2b9fa7 100644 --- a/marathon.json +++ b/marathon.json @@ -1,5 +1,5 @@ { - "id": "openvpn2", + "id": "openvpn", "instances": 1, "portDefinitions": [], "acceptedResourceRoles":[ From fe202588cd56d1844057e4c048e55eea10edec98 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Mon, 4 Dec 2017 20:38:18 +0000 Subject: [PATCH 08/20] More debugging --- bin/run.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 84ea13e..9a2933a 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -113,14 +113,17 @@ function get_location { function set_public_location { local loc=$ZKPATH/location.conf source $OPENVPN/ovpn_env.sh + public_address="remote $(wget -q -O - -U curl ipinfo.io/ip)" if run_command "ls $loc" ; then - run_command "set $loc \"remote $(wget -O - -U curl ifconfig.me) $PORT0 $OVPN_PROTO\"" + #run_command "set $loc \"remote $(wget -O - -U curl ifconfig.me) $PORT0 $OVPN_PROTO\"" + run_command "set $loc \"$public_address $PORT1 $OVPN_PROTO\"" else run_command "create $loc ''" set_public_location - fi + fi } + ############################## # Main setup ############################## @@ -148,6 +151,7 @@ function setup { reset build_configuration upload_files + set_public_location fi } From 56898c362937fe1ca90ad98627ccd4ac509dde27 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Tue, 5 Dec 2017 12:49:31 +0000 Subject: [PATCH 09/20] Updated remove to use synchronise --- dcos_openvpn/cert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dcos_openvpn/cert.py b/dcos_openvpn/cert.py index ab27846..032865f 100644 --- a/dcos_openvpn/cert.py +++ b/dcos_openvpn/cert.py @@ -34,8 +34,8 @@ def output(name): def remove(name): subprocess.check_call("ovpn_revokeclient {0} remove ".format(name), shell=True) - subprocess.check_call('/dcos/bin/zkshrun.sh "rmr /openvpn/pki"'.format(name), shell=True) - subprocess.check_call('/dcos/bin/zkshrun.sh "cp file:///etc/openvpn/pki /openvpn/pki true true"'.format(name), shell=True) + subprocess.check_call('/dcos/bin/run.sh synchronise'.format(name), shell=True) + def test(): From a0e35f9d3347ccfe5e150f6d77808e144c6e754d Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Tue, 5 Dec 2017 15:06:58 +0000 Subject: [PATCH 10/20] Test to fix hostport and set public location after sync --- bin/run.sh | 1 + marathon.json | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 9a2933a..10dccf9 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -93,6 +93,7 @@ function synchronise { download_files pkill openvpn ovpn_run --daemon + set_public_location else echo "Upload marker found will attempt on next cron run" fi diff --git a/marathon.json b/marathon.json index d2b9fa7..18913c5 100644 --- a/marathon.json +++ b/marathon.json @@ -10,13 +10,13 @@ "docker": { "portMappings": [ { - "hostport": 5000, + "hostPort": 5000, "containerPort": 5000, "protocol": "tcp", "name": "openvpn-admin" }, { - "hostport": 1194, + "hostPort": 1194, "containerPort": 1194, "protocol": "udp", "name": "openvpnudp" From b7b5d9c7a94e6545f9f3a8c6c42a5eb3870fc8aa Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Tue, 5 Dec 2017 15:51:13 +0000 Subject: [PATCH 11/20] Corrected set location --- bin/run.sh | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 10dccf9..dd454ff 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -108,20 +108,11 @@ function synchronise { ############################## function get_location { - echo $(run_command "get $ZKPATH/location.conf") + cat /etc/openvpn/location.conf } function set_public_location { - local loc=$ZKPATH/location.conf - source $OPENVPN/ovpn_env.sh - public_address="remote $(wget -q -O - -U curl ipinfo.io/ip)" - if run_command "ls $loc" ; then - #run_command "set $loc \"remote $(wget -O - -U curl ifconfig.me) $PORT0 $OVPN_PROTO\"" - run_command "set $loc \"$public_address $PORT1 $OVPN_PROTO\"" - else - run_command "create $loc ''" - set_public_location - fi + echo "remote $(wget -q -O - -U curl ipinfo.io/ip) $PORT0 $OVPN_PROTO" > /etc/openvpn/location.conf } From c43aa8a54106609abaed30495a8d6b811c419593 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Tue, 5 Dec 2017 16:12:00 +0000 Subject: [PATCH 12/20] More location.conf changes --- bin/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index dd454ff..6757af3 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -112,7 +112,7 @@ function get_location { } function set_public_location { - echo "remote $(wget -q -O - -U curl ipinfo.io/ip) $PORT0 $OVPN_PROTO" > /etc/openvpn/location.conf + echo "remote $(wget -q -O - -U curl ipinfo.io/ip) $PORT1 $OVPN_PROTO" > /etc/openvpn/location.conf } @@ -143,8 +143,8 @@ function setup { reset build_configuration upload_files - set_public_location fi + set_public_location } function run_server { From cf721d626866044ae25e234a6c2a1016bad7095e Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Tue, 5 Dec 2017 16:23:29 +0000 Subject: [PATCH 13/20] Dave, my mind is going --- bin/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/run.sh b/bin/run.sh index 6757af3..a5a8292 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -112,6 +112,7 @@ function get_location { } function set_public_location { + source $OPENVPN/ovpn_env.sh echo "remote $(wget -q -O - -U curl ipinfo.io/ip) $PORT1 $OVPN_PROTO" > /etc/openvpn/location.conf } From 25f1954d91ef8fa3eb33729d40da9086ebe47e7e Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Tue, 5 Dec 2017 16:56:58 +0000 Subject: [PATCH 14/20] Upload changes on revoke rather than synchronise --- dcos_openvpn/cert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcos_openvpn/cert.py b/dcos_openvpn/cert.py index 032865f..7fa5058 100644 --- a/dcos_openvpn/cert.py +++ b/dcos_openvpn/cert.py @@ -34,7 +34,7 @@ def output(name): def remove(name): subprocess.check_call("ovpn_revokeclient {0} remove ".format(name), shell=True) - subprocess.check_call('/dcos/bin/run.sh synchronise'.format(name), shell=True) + subprocess.check_call('/dcos/bin/run.sh upload_files'.format(name), shell=True) From 1518c1feaf3cc6d6005119911a362621b1f00140 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Tue, 5 Dec 2017 17:43:47 +0000 Subject: [PATCH 15/20] Updated readme --- README.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a643d1e..d99418e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ DC/OS OpenVPN OpenVPN server and REST management interface package for DC/OS -Please note: This is a [DC/OS Community package](https://dcos.io/community/), which is not formally tested or supported by Mesosphere +Please note: This is a [DC/OS Community package](https://dcos.io/community/), which is not formally tested or supported by Mesosphere. Issues and PRs are welcome @@ -19,7 +19,7 @@ Features 1. The Zookeeper znode `/openvpn` has ACLs enabled, to protect the OpenVPN server and client credentials 1. Synchronisation of assets between the container and Zookeeper in case the container is restarted 1. Clients revoked through the REST interface are correctly revoked from OpenVPN -1. Merged the previously separate openvpn server & openvpn-admin 0.0.0-0.1 packages into one. The openvpn-admin package is no longer required. +1. Merged the previously separate openvpn server & openvpn-admin 0.0.0-0.1 packages into one. The openvpn-admin package is no longer required Installation -------------- @@ -57,19 +57,18 @@ Usage The exact endpoints can be confirmed from **DC/OS Dashboard > Services > OpenVPN > > Details** 1. OpenVPN is presented on 1194/UDP and any OpenVPN client will default to this port -1. The REST management interface is available on 5000/TCP and will be accessed at https://:5000 +1. The REST management interface is available on `5000/TCP` and will be accessed at `https://:5000` 1. /status /test /client are all valid REST endpoints. /status does not require authentication as it is used for health checks ### Add a User 1. Authenticate and POST to the REST endpoint, the new user's credentials will be output to the POST body ``` -curl -k -u username:password -X POST -d "name=richard" https://:5000/client +curl -k -u username:password -X POST -d "name=richard" https://:5000/client > richard.ovpn ``` -2. Copy the entire ouput and save to a single file - you may need to amend the target server IP if on an internal network -3. Save the file as dcos.ovpn and add to any suitable OpenVPN client, like (Tunnelblick)[https://tunnelblick.net/] for macOS for example -4. Test connecting with the OpenVPN client. See Troubleshooting for help. -5. The new client credentials will be backed up to Zookeeper for persistence in case the task is killed, and will be copied back as required +2. Import the .ovpn file into any suitable OpenVPN client, like (Tunnelblick)[https://tunnelblick.net/] for macOS for example +3. Test connecting with the OpenVPN client. See Troubleshooting for help. +4. The new client credentials will be backed up to Zookeeper for persistence in case the task is killed, and will be synchronised with any other instances ### Revoke a User @@ -77,18 +76,18 @@ curl -k -u username:password -X POST -d "name=richard" https://:5000/client ``` curl -k -u username:password -X DELETE https://:5000/client/richard ``` -2. The client is correctly revoked from OpenVPN and the assets are removed from the container and Zookeeper +2. The client is correctly revoked from OpenVPN and the change is synchronised with all running instances ### Remove Zookeeper data -During installation, an ACL is set on the Zookeeper openvpn znode, restricting access based on the OVPN_USERNAME & OVPN_PASSWWORD credentials. +During installation, an ACL is set on the Zookeeper OpenVPN znode, restricting access based on the OVPN_USERNAME & OVPN_PASSWWORD credentials. In order to remove the znode data you must either authenticate with those same credentials or as the Zookeeper super user. Some examples of how to achieve this using zk-shell which is shipped in the Docker image: ``` zk-shell connect master.mesos:2181 (CONNECTED) / add_auth digest : -(CONNECTED) / rmr openvpn/ +(CONNECTED) / rmr /openvpn/ (CONNECTED) / exit ``` @@ -126,10 +125,19 @@ zkshrun.sh is a little standalone helper script that provides run_command to the A modified version of easyrsa is shipped which removes user prompts. +Synchronisation between multiple running instances is handled via a cron job, which runs every 2 minutes. It checks to see +if the `openvpn/pki/issue.txt` differs between localhost and in Zookeeper. If there's a diff, it signifies that a user has been created +or revoked by another instance which has been uploaded to Zookeeper. The full pki directory is copied down to update the local instance +and the ovpn daemon is restarted. + +This functionality is rudimentary and it's recommended not to add or revoke more than one user at a time and then leave >3 minutes between +each change to allow the synchronisation to work. + ### Startup order 1. run.sh checks for existing assets in Zookeeper and copies them to the container if they exist, otherwise initpki and genconfig are run 1. Launchs the OpenVPN daemon in daemon mode 1. Starts the Python REST interface +1. Synchronisation cron job every 2 minutes Troubleshooting From f527bb3c52df667a29b74b4fbf16145c78814fa4 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Wed, 6 Dec 2017 09:52:44 +0000 Subject: [PATCH 16/20] Added logic to handle multiple instances started at same time --- bin/run.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index a5a8292..f4185b5 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -122,6 +122,8 @@ function set_public_location { ############################## function build_configuration { + # Adding a lock to stop any other instances trying to upload at the same time + run_command "create $ZKPATH/upload_marker ''" ovpn_genconfig -u udp://$CA_CN rm -rf $CONFIG_LOCATION/pki (echo $CA_CN) | PATH=/dcos/bin:$PATH ovpn_initpki nopass @@ -140,10 +142,17 @@ function setup { reset_container download_files else - echo "Files not found in Zookeeper - generating and uploading" - reset - build_configuration - upload_files + echo "Files not found in Zookeeper" + if [[ $(run_command "ifind /openvpn/upload_marker") = "" ]]; then + echo "Initialising OpenVPN config, pki and uploading to Zookeeper" + reset + build_configuration + upload_files + else + echo "Upload marker found, another instance may be initialising, sleeping for 2m then will attempt download" + sleep 2000 + setup + fi fi set_public_location } From b6111eaccde5f6a128b7b86734386915d8febcab Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Wed, 6 Dec 2017 09:53:46 +0000 Subject: [PATCH 17/20] Updated marathon.json to reflect new Docker images verion --- marathon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marathon.json b/marathon.json index 18913c5..7ec8568 100644 --- a/marathon.json +++ b/marathon.json @@ -23,7 +23,7 @@ } ], "network": "BRIDGE", - "image": "aggress/dcos-openvpn:0.0.0-1.0", + "image": "aggress/dcos-openvpn:0.0.0-2.0", "forcePullImage": true, "privileged": true } From dea065c50ec932d916c12307b58cc6ecf7f3f0e3 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Thu, 7 Dec 2017 16:51:06 +0000 Subject: [PATCH 18/20] Removed check_status function and updated Changelog --- CHANGELOG.md | 10 +++++ bin/run.sh | 119 +++++++++++++++++++++++++++------------------------ 2 files changed, 73 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 810bf40..35aecad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ Changelog =============== +0.0.0-2.0 - 7th December 2017 + +- Added synchronisation of the PKI (users, certificates and keys) between multiple running instances +- Enabled >1 instances to be started at the same time and match their local data +- Cleaned up the output to stdout +- Refactored a number of functions in run.sh to improve robustness +- Increased CPU resource from 0.1 to 1.0 due to DC/OS 1.10 now enforcing CPU usage - required for key generation. +- Fixed https://github.com/dcos-labs/dcos-openvpn/issues/13 +- Improved the function to find the public address + 0.0.0-1.0 - 12th September 2017 - Changed znode path from dcos-vpn to openvpn diff --git a/bin/run.sh b/bin/run.sh index f4185b5..9ce0617 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -4,25 +4,8 @@ # Vars, checks and admin ############################## -container_files=0 -zookeeper_path=0 source /dcos/bin/envs.sh -# Check to see if the container already has the openvpn files locally -# And the second checks for an existing znode in Zookeeper which suggests -# This is being re-run after a previous setup - -function check_status { - if [ -f $CONFIG_LOCATION/openvpn.conf ]; then - container_files=1 - fi - if [[ -z $(run_command "ls $ZKPATH/openvpn.conf") ]]; then - zookeeper_path=1 - fi - echo "container_files = " $container_files - echo "zookeeper_path = " $zookeeper_path -} - # Workaround to pass add_auth on the one liner as zk-shell doens't provide this as a param function run_command { @@ -30,12 +13,20 @@ function run_command { return $? } +function fix_scripts { + # Fix a bug in zk-shell copy that's pending a pull request + sed -i 's/return PathValue("".os.path.join(fph.readlines()))/return PathValue("".join(os.path.join(fph.readlines())))/g' /usr/lib/python2.7/site-packages/zk_shell-1.1.3-py2.7.egg/zk_shell/copy.py + + # Replace the shipped easyrsa with our easyrsa to remove the revoke confirmation + sed -i 's/easyrsa/\/dcos\/bin\/easyrsa/g' /usr/local/bin/ovpn_revokeclient +} + ############################## # File download and upload ############################## function download_files { - if [[ $(run_command "ifind /openvpn/upload_marker") = "" ]]; then + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then ZKPATH_STRIPPED=$(echo $ZKPATH | sed -e 's/^\///') for fname in $(run_command "find / $ZKPATH_STRIPPED"); do local sub_path=$(echo $fname | cut -d/ -f3-) @@ -48,28 +39,24 @@ function download_files { fi local fs_path=$CONFIG_LOCATION/$sub_path - run_command "cp $fname file://$fs_path false true false true" + run_command "cp $fname file://$fs_path false true false false" > /dev/null 2>&1 # Directories are copied as empty files, remove them so that the # subsequent copies actually work. [ -s $fs_path ] || rm $fs_path done else - echo "Upload marker found, leaving until next cron run" + echo "INFO: Upload marker found, leaving until next cron run" fi } function upload_files { - if [ $zookeeper_path = 0 ]; then - run_command "create $ZKPATH '' false false true" - run_command "set_acls /$ZKPATH username_password:$OVPN_USERNAME:$OVPN_PASSWORD:cdrwa" - fi # Adding a marker so we know when all the files have been uploaded run_command "create $ZKPATH/upload_marker ''" for fname in $(find $CONFIG_LOCATION -not -type d); do local zk_location=$(echo $fname | sed 's|'$CONFIG_LOCATION'/|/|') - run_command "cp file://$fname $ZKPATH$zk_location false true false true" + run_command "cp file://$fname $ZKPATH$zk_location false true false true" > /dev/null 2>&1 done # Removing upload marker @@ -87,18 +74,19 @@ function synchronise { index_tmp="/tmp/index.txt" rm -f $index_tmp - run_command "cp $index_on_zk file://$index_tmp false true false true" + run_command "cp $index_on_zk file://$index_tmp false true false true" > /dev/null 2>&1 if [[ $(diff -q $index_on_local $index_tmp) != "" ]]; then - if [[ $(run_command "ifind /openvpn/upload_marker") = "" ]]; then + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then + echo "INFO: Zookeeper has a new dataset, downloading and restarting OpenVPN to apply" download_files pkill openvpn ovpn_run --daemon set_public_location else - echo "Upload marker found will attempt on next cron run" + echo "INFO: Upload marker found, will attempt on next cron run" fi else - echo "index.txt matches so nothing to do" + echo "INFO: No changes found on Zookeeper" fi } @@ -113,6 +101,7 @@ function get_location { function set_public_location { source $OPENVPN/ovpn_env.sh + echo "INFO: Setting public location" echo "remote $(wget -q -O - -U curl ipinfo.io/ip) $PORT1 $OVPN_PROTO" > /etc/openvpn/location.conf } @@ -121,50 +110,69 @@ function set_public_location { # Main setup ############################## +function create_zkpath { + if [[ $(run_command "find /openvpn") = "" ]]; then + run_command "create $ZKPATH '' false false true" + run_command "set_acls /$ZKPATH username_password:$OVPN_USERNAME:$OVPN_PASSWORD:cdrwa" + fi +} + function build_configuration { # Adding a lock to stop any other instances trying to upload at the same time - run_command "create $ZKPATH/upload_marker ''" - ovpn_genconfig -u udp://$CA_CN - rm -rf $CONFIG_LOCATION/pki + echo "INFO: Creating lock file" + run_command "create $ZKPATH/upload_marker ''" > /dev/null 2>&1 + echo "INFO: Resetting container" + reset_container + echo "INFO: Building configuration" + ovpn_genconfig -u udp://$CA_CN > /dev/null 2>&1 + echo "INFO: Building PKI" (echo $CA_CN) | PATH=/dcos/bin:$PATH ovpn_initpki nopass + touch /etc/openvpn/complete } function setup { - - # Fix a bug in zk-shell copy that's pending a pull request - sed -i 's/return PathValue("".os.path.join(fph.readlines()))/return PathValue("".join(os.path.join(fph.readlines())))/g' /usr/lib/python2.7/site-packages/zk_shell-1.1.3-py2.7.egg/zk_shell/copy.py - - # Replace the shipped easyrsa with our easyrsa to remove the revoke confirmation - sed -i 's/easyrsa/\/dcos\/bin\/easyrsa/g' /usr/local/bin/ovpn_revokeclient - - if [ $zookeeper_path = 1 ] && [ $container_files = 0 ]; then - echo "Files found in Zookeeper - copying to container" - reset_container - download_files - else - echo "Files not found in Zookeeper" - if [[ $(run_command "ifind /openvpn/upload_marker") = "" ]]; then - echo "Initialising OpenVPN config, pki and uploading to Zookeeper" - reset + # Introduce a random delay between 1-21 seconds in case of multiple instances starting at the same time + sleep $[ ( $RANDOM % 20 ) + 1 ]s + + if [[ $(run_command "find /openvpn/ complete") = "" ]]; then + echo "INFO: I didn't find a marker signifying a full dataset on Zookeeper" + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then + echo "INFO: I didn't find a lock" + echo "INFO: Creating the zkpath if it doesn't already exist" + create_zkpath build_configuration + echo "INFO: Uploading files to Zookeeper" upload_files + set_public_location + else + echo "INFO: Lock found, will random sleep then try again" + setup + fi + else + if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then + reset_container + echo "INFO: Files found in Zookeeper, no lock found, downloading to container" + download_files + set_public_location else - echo "Upload marker found, another instance may be initialising, sleeping for 2m then will attempt download" - sleep 2000 + echo "INFO: Lock found, will random sleep then try again" setup fi - fi - set_public_location + fi } function run_server { source /dcos/bin/envs.sh - check_status + fix_scripts setup + echo "INFO: Starting crond" crond - echo "*/2 * * * * /dcos/bin/run.sh synchronise" >> /etc/crontabs/root + echo "INFO: Adding cron job for synchronisation" + echo "*/2 * * * * /dcos/bin/run.sh synchronise >> /mnt/mesos/sandbox/stdout 2>&1" >> /etc/crontabs/root + echo "INFO: Starting OpenVPN daemon" ovpn_run --daemon - /usr/bin/python -m dcos_openvpn.main + echo "INFO: Starting Python REST interface" + /usr/bin/python -m dcos_openvpn.main } function reset { @@ -182,7 +190,6 @@ case "$@" in setup) setup ;; download_files) download_files ;; upload_files) upload_files ;; - check_status) check_status ;; build_configuration) build_configuration ;; set_public_location) set_public_location ;; get_location) get_location ;; From 0c64c832ddd43e80ab984b3a495acddbbbc48e28 Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Thu, 7 Dec 2017 17:23:00 +0000 Subject: [PATCH 19/20] Corrected marathon running instances, updated changelog and readme --- CHANGELOG.md | 1 + README.md | 27 ++++++++++++--------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35aecad..78dced9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Changelog - Increased CPU resource from 0.1 to 1.0 due to DC/OS 1.10 now enforcing CPU usage - required for key generation. - Fixed https://github.com/dcos-labs/dcos-openvpn/issues/13 - Improved the function to find the public address +- Fixed the hostports in the marathon.json 0.0.0-1.0 - 12th September 2017 diff --git a/README.md b/README.md index d99418e..7185ec7 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,14 @@ Installation -------------- **You must configure the OVPN_USERNAME & OVPN_PASSWORD environment variables before installation** These are required for both the REST interface -credentials and for the Zookeeper znode ACL. +credentials and for the Zookeeper znode ACL. Please note, DC/OS 1.10 enforces CPU usage, key generation requires a full 1.0 CPU. This can be reduced back to 0.1 once up and running. ### DC/OS Public Universe Installation -1. From the **DC/OS Dashboard > Universe > Packages > enter openvpn in the search box** -1. Select **Install Package > Advanced Installation** and scroll down -1. Configure both the OVPN_USERNAME & OVPN_PASSWORD -1. Select **Review and Install > Install** +1. From the `DC/OS Dashboard > Universe > Packages > enter openvpn in the search box` +1. Select `Install Package > Advanced Installation` and scroll down +1. Configure both the `OVPN_USERNAME` & `OVPN_PASSWORD` +1. Select `Review and Install > Install` 1. The service is installed and initialises, when complete, it'll be marked as Running and Healthy 1. See Troubleshooting for any issues, otherwise go to Usage @@ -47,16 +47,15 @@ The task can be also be added as a package to a local Universe repository 1. Clone https://github.com/mesosphere/universe 1. Read https://docs.mesosphere.com/1.9/administering-clusters/deploying-a-local-dcos-universe/ -1. Read and amend the source of local_universe_setup.sh to facilitate building and publishing Usage -------------- ### Endpoints -The exact endpoints can be confirmed from **DC/OS Dashboard > Services > OpenVPN > > Details** +The exact endpoints can be confirmed from `DC/OS Dashboard > Services > OpenVPN > > Details` -1. OpenVPN is presented on 1194/UDP and any OpenVPN client will default to this port +1. OpenVPN is presented on `1194/UDP` and any OpenVPN client will default to this port 1. The REST management interface is available on `5000/TCP` and will be accessed at `https://:5000` 1. /status /test /client are all valid REST endpoints. /status does not require authentication as it is used for health checks @@ -66,8 +65,8 @@ The exact endpoints can be confirmed from **DC/OS Dashboard > Services > OpenVPN ``` curl -k -u username:password -X POST -d "name=richard" https://:5000/client > richard.ovpn ``` -2. Import the .ovpn file into any suitable OpenVPN client, like (Tunnelblick)[https://tunnelblick.net/] for macOS for example -3. Test connecting with the OpenVPN client. See Troubleshooting for help. +2. Import the .ovpn file into any suitable OpenVPN client, Tunnelblick for macOS, for example +3. Test connecting with the OpenVPN client. See Troubleshooting for help 4. The new client credentials will be backed up to Zookeeper for persistence in case the task is killed, and will be synchronised with any other instances ### Revoke a User @@ -80,7 +79,7 @@ curl -k -u username:password -X DELETE https://:5000/client/richard ### Remove Zookeeper data -During installation, an ACL is set on the Zookeeper OpenVPN znode, restricting access based on the OVPN_USERNAME & OVPN_PASSWWORD credentials. +During installation, an ACL is set on the Zookeeper OpenVPN znode, restricting access based on the `OVPN_USERNAME` & `OVPN_PASSWWORD` credentials. In order to remove the znode data you must either authenticate with those same credentials or as the Zookeeper super user. Some examples of how to achieve this using zk-shell which is shipped in the Docker image: @@ -91,7 +90,7 @@ zk-shell connect master.mesos:2181 (CONNECTED) / exit ``` -If you intend to change the OVPN_USERNAME & OVPN_PASSWORD, you will need to change the ACL on the existing znode, then reinstall the package +If you intend to change the `OVPN_USERNAME` & `OVPN_PASSWORD`, you will need to change the ACL on the existing znode, then reinstall the package with new credentials ``` zk-shell connect master.mesos:2181 @@ -145,7 +144,7 @@ Troubleshooting ### Service -1. Review stdout and stderr from the task's logs under the **DC/OS Dashboard > Service > openvpn > running task > logs** +1. Review stdout and stderr from the task's logs under the `DC/OS Dashboard > Service > openvpn > running task > logs` 2. If the task is running on DC/OS, find out which agent is running the service using the DC/OS cli `dcos task | grep openvpn` 4. SSH to that agent and get a shell on the running container ``` @@ -177,10 +176,8 @@ DC/OS to allow you to delete the root openvpn znode. Setting ZK credentials is r Todo -------------- -1. Get defined host ports working in the marathon.json - works in the Universe marathon template 1. The patch for zk-shell https://github.com/rgs1/zk_shell/pull/82 as managed in run.bash around line 100 needs removing when zk-shell is fixed 1. Update the /status endpoint for ovpn_status output and tie into a healthcheck -1. run.sh usage and tidying 1. Update for DC/OS 1.10 and file based secrets 1. Either extend zk-shell to add auth to its params or replace with Kazoo code 1. Replace the location function which calls out to ifconfig.me as it's of no use for internal networks From 97150ff4ed09f9136be902b0a148c1a00428ebed Mon Sep 17 00:00:00 2001 From: Richard Shaw Date: Thu, 7 Dec 2017 18:19:11 +0000 Subject: [PATCH 20/20] Corrected startup issue where zkpath didn't exist so initil checks failed --- bin/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 9ce0617..7a10d03 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -112,6 +112,7 @@ function set_public_location { function create_zkpath { if [[ $(run_command "find /openvpn") = "" ]]; then + echo "INFO: Creating the zkpath if it doesn't already exist" run_command "create $ZKPATH '' false false true" run_command "set_acls /$ZKPATH username_password:$OVPN_USERNAME:$OVPN_PASSWORD:cdrwa" fi @@ -134,12 +135,12 @@ function setup { # Introduce a random delay between 1-21 seconds in case of multiple instances starting at the same time sleep $[ ( $RANDOM % 20 ) + 1 ]s + create_zkpath + if [[ $(run_command "find /openvpn/ complete") = "" ]]; then echo "INFO: I didn't find a marker signifying a full dataset on Zookeeper" if [[ $(run_command "find /openvpn/ upload_marker") = "" ]]; then echo "INFO: I didn't find a lock" - echo "INFO: Creating the zkpath if it doesn't already exist" - create_zkpath build_configuration echo "INFO: Uploading files to Zookeeper" upload_files