diff --git a/.config/.gitignore b/.config/.gitignore new file mode 100644 index 00000000..a1538949 --- /dev/null +++ b/.config/.gitignore @@ -0,0 +1 @@ +/env.private.sh diff --git a/.env b/.env deleted file mode 100644 index 6aab0dcd..00000000 --- a/.env +++ /dev/null @@ -1,4 +0,0 @@ -PYTHONPATH=./app/scheduler:./lib/common:./app/controller:./lib/runners/rexfw:./lib/schedule_estimation:./lib/grpc - -# Prefix for docker namespace. If non-empty must end with a slash, e.g. eu.gcr.io/resaas-simeon-dev/ -HUB_NAMESPACE= diff --git a/.envrc b/.envrc index adb8e83c..37a09376 100644 --- a/.envrc +++ b/.envrc @@ -1 +1,2 @@ use flake .#controller +watch_file .config/env.private.sh diff --git a/.gitignore b/.gitignore index ad8c87a0..f8453f9a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ app/client/app.yaml /result /.direnv/ +/.cache/ diff --git a/Makefile b/Makefile index b8e6bf20..e03df86b 100644 --- a/Makefile +++ b/Makefile @@ -5,23 +5,24 @@ .PHONY: images images: @echo "Building docker images..." - docker build -t "${HUB_NAMESPACE}chainsail-celery-worker:latest" -f ./docker/celery/Dockerfile . - docker build -t "${HUB_NAMESPACE}chainsail-scheduler:latest" -f ./docker/scheduler/Dockerfile . - docker build -t "${HUB_NAMESPACE}chainsail-nginx:latest" -f ./docker/nginx/Dockerfile . - docker build -t "${HUB_NAMESPACE}chainsail-user-code:latest" -f docker/user-code/Dockerfile . - docker build -t "${HUB_NAMESPACE}chainsail-httpstan-server:latest" -f docker/httpstan-server/Dockerfile . - docker build -t "${HUB_NAMESPACE}chainsail-mpi-node-k8s:latest" -f docker/node/Dockerfile . - docker build -t "${HUB_NAMESPACE}chainsail-mcmc-stats-server:latest" -f docker/mcmc-stats-server/Dockerfile . + docker build -t "${IMAGE_PREFIX}chainsail-celery-worker:latest" -f ./docker/celery/Dockerfile . + docker build -t "${IMAGE_PREFIX}chainsail-scheduler:latest" -f ./docker/scheduler/Dockerfile . + docker build -t "${IMAGE_PREFIX}chainsail-nginx:latest" -f ./docker/nginx/Dockerfile . + docker build -t "${IMAGE_PREFIX}chainsail-user-code:latest" -f docker/user-code/Dockerfile . + docker build -t "${IMAGE_PREFIX}chainsail-httpstan-server:latest" -f docker/httpstan-server/Dockerfile . + docker build -t "${IMAGE_PREFIX}chainsail-sshd:latest" ./docker/sshd + docker build -t "${IMAGE_PREFIX}chainsail-mpi-node-k8s:latest" -f docker/node/Dockerfile . + docker build -t "${IMAGE_PREFIX}chainsail-mcmc-stats-server:latest" -f docker/mcmc-stats-server/Dockerfile . @echo "Done." .PHONY: push-images push-images: images @echo "Pushing docker images..." - docker push "${HUB_NAMESPACE}chainsail-celery-worker:latest" - docker push "${HUB_NAMESPACE}chainsail-scheduler:latest" - docker push "${HUB_NAMESPACE}chainsail-nginx:latest" - docker push "${HUB_NAMESPACE}chainsail-user-code:latest" - docker push "${HUB_NAMESPACE}chainsail-httpstan-server:latest" - docker push "${HUB_NAMESPACE}chainsail-mpi-node-k8s:latest" - docker push "${HUB_NAMESPACE}chainsail-mcmc-stats-server:latest" + docker push "${IMAGE_PREFIX}chainsail-celery-worker:latest" + docker push "${IMAGE_PREFIX}chainsail-scheduler:latest" + docker push "${IMAGE_PREFIX}chainsail-nginx:latest" + docker push "${IMAGE_PREFIX}chainsail-user-code:latest" + docker push "${IMAGE_PREFIX}chainsail-httpstan-server:latest" + docker push "${IMAGE_PREFIX}chainsail-mpi-node-k8s:latest" + docker push "${IMAGE_PREFIX}chainsail-mcmc-stats-server:latest" @echo "Done." diff --git a/app/client/run_dev_client.sh b/app/client/run_dev_client.sh index 9f1160fd..7d8f70c4 100755 --- a/app/client/run_dev_client.sh +++ b/app/client/run_dev_client.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash SCHEDULER_URL=$(minikube service scheduler --url) -MCMC_STATS_SERVER_URL=$(minikube service mcmc-stats-server --url) +MCMC_STATS_URL=$(minikube service mcmc-stats-server --url) GRAPHITE_URL=$(minikube service graphite --url | head -n1) -export SCHEDULER_URL=$SCHEDULER_URL -export GRAPHITE_URL=$GRAPHITE_URL -export MCMC_STATS_URL=$MCMC_STATS_SERVER_URL +export SCHEDULER_URL +export GRAPHITE_URL +export MCMC_STATS_URL yarn run dev diff --git a/docker/sshd.nix/.gitignore b/docker/sshd.nix/.gitignore new file mode 100644 index 00000000..c4a847d9 --- /dev/null +++ b/docker/sshd.nix/.gitignore @@ -0,0 +1 @@ +/result diff --git a/docker/sshd.nix/default.nix b/docker/sshd.nix/default.nix new file mode 100644 index 00000000..2526e4d7 --- /dev/null +++ b/docker/sshd.nix/default.nix @@ -0,0 +1,62 @@ +{ pkgs ? import {} }: + +let + config = '' + Port 2222 + PermitRootLogin no + PasswordAuthentication no + + AuthorizedKeysFile .ssh/authorized_keys + + KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com + + # modes are broken since the rewrite.. fix it!! + StrictModes no + ''; +in +with pkgs; dockerTools.buildImage { + name = "chainsail-sshd"; + + runAsRoot = '' + #!${stdenv.shell} + echo ${dockerTools.shadowSetup} + ${dockerTools.shadowSetup} + + useradd sshd + mkdir -p /var/empty /run + + mkdir -p /etc/ssh + cfg=/etc/ssh/sshd_config + cp ${pkgs.openssh}/etc/ssh/sshd_config $cfg + echo 'PermitRootLogin yes' >>$cfg + echo 'PasswordAuthentication no' >>$cfg + + # XXX: Is this okay? + ${pkgs.openssh}/bin/ssh-keygen -A + + # setup ssh user +# groupadd -g 1337 -r git +# useradd -r -u 1337 -g git -d /data -M git +# usermod -p '*' git # set (invalid) password to only allow pubkey auth +# chsh -s ${pkgs.git}/bin/git-shell git +# +# # create home dir for repo storage +# mkdir /data +# +# # setup ssh user auth +# mkdir -p /data/.ssh +# touch /data/.ssh/authorized_keys +# chmod go-w /data/ +# chmod 700 /data/.ssh +# chmod 600 /data/.ssh/authorized_keys + ''; + + config = { + Cmd = [ "${pkgs.openssh}/bin/sshd" "-e" "-D" "-p" "26" ]; + ExposedPorts = { + "26/tcp" = {}; + }; + }; +} diff --git a/docker/sshd/Dockerfile b/docker/sshd/Dockerfile new file mode 100644 index 00000000..35df236b --- /dev/null +++ b/docker/sshd/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.8.7-slim + +RUN mkdir -p /run/sshd + +RUN apt-get update && \ + apt-get install -y curl wget unzip openssh-client openssh-server && \ + rm -rf /var/lib/apt/lists/* + +# Global SSH configurations +RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# Disable password authentication +RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config +# Disable StrictMode (required for k8s secret volumes to work properly since their symlinks have open permissions) +RUN sed -i 's/#StrictModes yes/StrictModes no/' /etc/ssh/sshd_config +# Listen on a non-default port to avoid clashing with the host's ssh +RUN sed -i 's/#Port 22/Port 26/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +ADD ssh/ssh_config /etc/ssh/ssh_config +ADD entrypoint.sh /app/entrypoint.sh + +# For SSH. Using non-default ssh port. +EXPOSE 26 + +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/docker/sshd/entrypoint.sh b/docker/sshd/entrypoint.sh new file mode 100755 index 00000000..6e43f8ce --- /dev/null +++ b/docker/sshd/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -ex + +mkdir -p /root/.ssh + +exec "$@" diff --git a/docker/sshd/ssh/ssh_config b/docker/sshd/ssh/ssh_config new file mode 100644 index 00000000..de3ca8e7 --- /dev/null +++ b/docker/sshd/ssh/ssh_config @@ -0,0 +1,5 @@ +Host * + IdentityFile /root/.ssh/id.pem + StrictHostKeyChecking accept-new + UserKnownHostsFile /root/known_hosts + Port 26 diff --git a/docs/deployment.md b/docs/deployment.md index 36a6a831..b861b35d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -26,15 +26,6 @@ This guide describes how to set up a Chainsail environment from scratch, either - [Allowing / disallowing application users](#allowing--disallowing-application-users) # -## Prerequisites for both Minikube and Google Cloud deployment -Make sure that you correctly edit the Terraform and Helm files. -Specifically, you'll want to make sure that you have matching container registry in -the following files / environment variables: -- `/terraform/cluster/local/main.tf` (`container_registry` in the `locals` block), -- `/helm/values.yaml` (`imageHubNamespace`), -- `/helm/values-local.yaml` (`imageHubNamespace`), -- `/helm/values-dev.yaml` (`imageHubNamespace`),if you're considering a Google Cloud deployment, -- and the `HUB_NAMESPACE` environment variable later. ## Local deployment @@ -67,9 +58,8 @@ and add them to Minikube's Docker registry. One way to do this is: # This makes docker commands use Minikube's Docker daemon eval $(minikube docker-env) -HUB_NAMESPACE="/" make images +make images ``` -The hub namespace environment variable has to match the value of the `imageHubNamespace` property in `helm/values.yaml` and the `container_registry` value in the `locals` block of `terraform/cluster/local/main.tf`. Then, you can install Chainsail with Helm: @@ -151,24 +141,34 @@ In order to be able to push the images to the container registry, the Google Clo It is called something like `.artifacts.` bucket. The name of the bucket might vary depending on the `zone` and `node_location` entries in the `chainsail_gcp` Terraform module in `terraform/base/dev/main.tf`. +Configure the image prefix for container images by creating a shell file call +`.config/env.private.sh` containing something like: + +``` +export IMAGE_PREFIX='' +export TF_VAR_image_prefix="$IMAGE_PREFIX" +``` + +Setting `TF_VAR_image_prefix` ensures that Terraform will use the same image +prefix. + To build and push images, run ```bash -HUB_NAMESPACE="/" make push-images +make push-images ``` -The hub namespace environment variable has to match the value of the `imageHubNamespace` property in `helm/values.yaml`. - The first time you deploy Chainsail, you will need to fetch the cluster's Kubernetes access credentials using `gcloud`: ```bash gcloud container clusters get-credentials --region $GCP_REGION chainsail ``` + The GCP region can be found in `terraform/base/dev/main.tf` in the `node_location` entry of the `chainsail_gcp` module. Once all of the desired images are published, you can install Chainsail with: ```bash -helm install -f helm/values-dev.yaml chainsail ./helm +helm install -f helm/values-dev.yaml --set imagePrefix=$IMAGE_PREFIX chainsail ./helm ``` ### 4. Deploy Chainsail front-end @@ -198,7 +198,7 @@ There are a couple of additional steps which need to be configured manually: To upgrade an already running Chainsail cluster to a newer version of the chart. Use: ```bash -helm upgrade -f helm/values-dev.yaml chainsail ./helm +helm upgrade -f helm/values-dev.yaml --set imagePrefix=$IMAGE_PREFIX chainsail ./helm ``` If using the `latest` tag for images in the helm chart, you will also need to restart the services so that diff --git a/flake.nix b/flake.nix index 4786312a..4d04014d 100644 --- a/flake.nix +++ b/flake.nix @@ -47,6 +47,26 @@ }; controller = poetry2nixPkg.mkPoetryApplication controllerOpts; controllerEnv = poetry2nixPkg.mkPoetryEnv controllerOpts; + sshdUser = pkgs.runCommand "user-setup" { } '' + mkdir -p $out/etc/ + + echo 'sshd:x:105:65534::/run/sshd:/usr/sbin/nologin' >>$out/etc/passwd + echo "root:x:0:0:root user:/root:${pkgs.bash}/bin/bash" >> $out/etc/passwd + ''; + controller-image = pkgs.dockerTools.streamLayeredImage { + name = "chainsail-mpi-node-k8s"; + contents = [ + controller + pkgs.openssh + sshdUser + + # debug aids + pkgs.bashInteractive + pkgs.coreutils + pkgs.ps + ]; + config.Cmd = [ "chainsail-controller" ]; + }; basePackages = [ pkgs.docker-compose @@ -63,21 +83,33 @@ ourYarn ]; - pythonDevShell = ps: pkgs.mkShell { - packages = ps ++ basePackages; + mkDevShell = {extraPkgs ? []}: pkgs.mkShell { + packages = extraPkgs ++ basePackages; + shellHook = '' + private_env_sh=.config/env.private.sh + if [[ -r $private_env_sh ]]; then + echo loading $private_env_sh + source $private_env_sh + fi + unset private_env_sh + PATH=$PWD/script:$PATH + ''; + }; + controllerDevShell = mkDevShell { + extraPkgs = [ + controllerEnv + controller + ]; }; - controllerDevShell = pythonDevShell [ - controllerEnv - controller - ]; in { devShells = { controller = controllerDevShell; - default = pkgs.mkShell { packages = basePackages; }; + default = mkDevShell {}; }; packages = { inherit controller; + inherit controller-image; scheduler = poetry2nixPkg.mkPoetryApplication { projectDir = ./app/scheduler; overrides = poetry2nixPkg.overrides.withDefaults ( diff --git a/helm/templates/scheduler-workers.yaml b/helm/templates/scheduler-workers.yaml index 328332a0..d4b30f3e 100644 --- a/helm/templates/scheduler-workers.yaml +++ b/helm/templates/scheduler-workers.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: chainsail-scheduler automountServiceAccountToken: true containers: - - image: {{ .Values.imageHubNamespace }}chainsail-celery-worker:{{ .Values.imageTag }} + - image: {{ .Values.imagePrefix }}chainsail-celery-worker:{{ .Values.imageTag }} imagePullPolicy: {{ .Values.image.pullPolicy }} name: scheduler-worker resources: {} diff --git a/helm/templates/scheduler.yaml b/helm/templates/scheduler.yaml index 61e8cecb..127834f3 100644 --- a/helm/templates/scheduler.yaml +++ b/helm/templates/scheduler.yaml @@ -17,7 +17,7 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 8}} containers: - - image: {{ .Values.imageHubNamespace }}chainsail-scheduler:{{ .Values.imageTag }} + - image: {{ .Values.imagePrefix }}chainsail-scheduler:{{ .Values.imageTag }} imagePullPolicy: {{ .Values.image.pullPolicy }} name: scheduler ports: diff --git a/helm/templates/stats.yaml b/helm/templates/stats.yaml index c0047120..4c39b6c4 100644 --- a/helm/templates/stats.yaml +++ b/helm/templates/stats.yaml @@ -17,7 +17,7 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 8}} containers: - - image: {{ .Values.imageHubNamespace }}chainsail-mcmc-stats-server:{{ .Values.imageTag }} + - image: {{ .Values.imagePrefix }}chainsail-mcmc-stats-server:{{ .Values.imageTag }} imagePullPolicy: {{ .Values.image.pullPolicy }} name: mcmc-stats-server ports: @@ -37,4 +37,4 @@ spec: volumes: - name: storage-config secret: - secretName: storage-yaml \ No newline at end of file + secretName: storage-yaml diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index b4596881..abe428da 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -17,13 +17,6 @@ tolerations: value: 'true' effect: 'NoSchedule' -# This has to match the `imageHubNamespace` value in `values-dev.yaml` -# Furthermore, this value with an additional forward slash is the -# `HUB_NAMESPACE` environment variable you have to set when building the -# Docker images using the Makefile. -imageHubNamespace: 'eu.gcr.io/project-name' -imageTag: 'latest' - ############################################################################## # Scheduler Settings ############################################################################## diff --git a/helm/values-local.yaml b/helm/values-local.yaml index 45d66aea..201b9e69 100644 --- a/helm/values-local.yaml +++ b/helm/values-local.yaml @@ -17,13 +17,6 @@ tolerations: value: 'true' effect: 'NoSchedule' -# This has to match the `container_registry` value in the `locals` block of -# `terraform/cluster/local/main.tf`. -# Furthermore, this value is the `HUB_NAMESPACE` environment variable you -# have to set when building the Docker images using the Makefile. -imageHubNamespace: 'eu.gcr.io/project-name/' -imageTag: 'latest' - ############################################################################## # Scheduler Settings ############################################################################## diff --git a/helm/values.yaml b/helm/values.yaml index 025d08bd..e709f84a 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -17,14 +17,12 @@ tolerations: value: 'true' effect: 'NoSchedule' -# For Minikube deployment, `imageHubNamespace` has to match the -# `imageHubNamespace` value in `values-local.yaml` and the `container_registry` -# value in the `locals` block of `terraform/cluster/local/main.tf`. -# For GCP deployment, it has to match the `imageHubNamespace` value in -# `values-dev.yaml`. -# Furthermore, this value is the `HUB_NAMESPACE` environment variable you have -# to set when building the Docker images using the Makefile. -imageHubNamespace: 'eu.gcr.io/project-name/' +# Ensure the imagePrefix matches your IMAGE_PREFIX environment variable with +# this command: +# +# helm install -f helm/values-dev.yaml --set imagePrefix=$IMAGE_PREFIX chainsail ./helm +# +imagePrefix: '' imageTag: 'latest' ############################################################################## diff --git a/script/chainsail-build-controller-image b/script/chainsail-build-controller-image new file mode 100755 index 00000000..647d6b3b --- /dev/null +++ b/script/chainsail-build-controller-image @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +name=${1:-controller-image} +image="${IMAGE_PREFIX:-}chainsail-mpi-node-k8s" + +mkdir -p .cache +out_file=$(mktemp --tmpdir=.cache --suffix=.out docker-load.XXXXXXXXXX) + +image_gen_script=".cache/build/$name" +nix build ".#${name}" -o "$image_gen_script" +$image_gen_script | docker load | tee "$out_file" +built_image=$(sed 's/Loaded image: //' <"$out_file") +echo "Built image $image" +set -x +docker tag "$built_image" "$image:latest" +echo docker run -it --rm "${image}" diff --git a/script/docker-test-node-sshd b/script/docker-test-node-sshd new file mode 100755 index 00000000..b10983aa --- /dev/null +++ b/script/docker-test-node-sshd @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +docker run -it --rm "${IMAGE_PREFIX}chainsail-mpi-node-k8s:latest" \ + sh -c 'ssh-keygen -A && mkdir -p /var/empty && exec /bin/sshd -p26 -D' diff --git a/terraform/cluster/local/.gitignore b/terraform/cluster/local/.gitignore index 5dd3ac89..3f531503 100644 --- a/terraform/cluster/local/.gitignore +++ b/terraform/cluster/local/.gitignore @@ -1 +1 @@ -/terraform.tfplan +/tfplan diff --git a/terraform/cluster/local/main.tf b/terraform/cluster/local/main.tf index be9bd9c8..7670cb36 100644 --- a/terraform/cluster/local/main.tf +++ b/terraform/cluster/local/main.tf @@ -9,7 +9,7 @@ terraform { required_providers { kubernetes = { version = ">= 2.16.1" - source = "hashicorp/kubernetes" + source = "hashicorp/kubernetes" } } } @@ -41,14 +41,12 @@ locals { } } ) - # When using both Minikube and cloud deployments, you probably - # want to replace `project-name` with your actual Google Cloud - # project name in order to avoid having to retag Docker images. - # Either way, make sure this matches the value of `imageHubNamespace` - # in `helm/values.yaml`. - container_registry = "eu.gcr.io/project-name" } +variable "image_prefix" { + type = string + default = "" +} module "chainsail-k8s" { source = "../../modules/chainsail-k8s" @@ -59,12 +57,10 @@ module "chainsail-k8s" { storage_access_key = "chainsail" storage_secret_key = "chainsail" storage_bucket = "chainsail-samples" - # TODO: Make these images match whatever local build script we use - # for rebuilding images - image_controller = "${local.container_registry}/chainsail-mpi-node-k8s:latest" - image_worker = "${local.container_registry}/chainsail-mpi-node-k8s:latest" - image_httpstan = "${local.container_registry}/chainsail-httpstan-server:latest" - image_user_code = "${local.container_registry}/chainsail-user-code:latest" + image_controller = "${var.image_prefix}chainsail-mpi-node-k8s:latest" + image_worker = "${var.image_prefix}chainsail-mpi-node-k8s:latest" + image_httpstan = "${var.image_prefix}chainsail-httpstan-server:latest" + image_user_code = "${var.image_prefix}chainsail-user-code:latest" } ###############################################################################