Controller image with Nix Docker Tools

.config/.gitignore

@@ -0,0 +1 @@
+/env.private.sh

.envrc

@@ -1 +1,2 @@
 use flake .#controller
+watch_file .config/env.private.sh

.gitignore

@@ -27,3 +27,4 @@ app/client/app.yaml
 
 /result
 /.direnv/
+/.cache/

Makefile

@@ -5,23 +5,24 @@
 .PHONY: images
 images:
 	@echo "Building docker images..."
-	docker build -t "${HUB_NAMESPACE}chainsail-celery-worker:latest" -f ./docker/celery/Dockerfile .
-	docker build -t "${HUB_NAMESPACE}chainsail-scheduler:latest" -f ./docker/scheduler/Dockerfile .
-	docker build -t "${HUB_NAMESPACE}chainsail-nginx:latest" -f ./docker/nginx/Dockerfile .
-	docker build -t "${HUB_NAMESPACE}chainsail-user-code:latest" -f docker/user-code/Dockerfile .
-	docker build -t "${HUB_NAMESPACE}chainsail-httpstan-server:latest" -f docker/httpstan-server/Dockerfile .
-	docker build -t "${HUB_NAMESPACE}chainsail-mpi-node-k8s:latest" -f docker/node/Dockerfile .
-	docker build -t "${HUB_NAMESPACE}chainsail-mcmc-stats-server:latest" -f docker/mcmc-stats-server/Dockerfile .
+	docker build -t "${IMAGE_PREFIX}chainsail-celery-worker:latest" -f ./docker/celery/Dockerfile .
+	docker build -t "${IMAGE_PREFIX}chainsail-scheduler:latest" -f ./docker/scheduler/Dockerfile .
+	docker build -t "${IMAGE_PREFIX}chainsail-nginx:latest" -f ./docker/nginx/Dockerfile .
+	docker build -t "${IMAGE_PREFIX}chainsail-user-code:latest" -f docker/user-code/Dockerfile .
+	docker build -t "${IMAGE_PREFIX}chainsail-httpstan-server:latest" -f docker/httpstan-server/Dockerfile .
+	docker build -t "${IMAGE_PREFIX}chainsail-sshd:latest" ./docker/sshd
+	docker build -t "${IMAGE_PREFIX}chainsail-mpi-node-k8s:latest" -f docker/node/Dockerfile .
+	docker build -t "${IMAGE_PREFIX}chainsail-mcmc-stats-server:latest" -f docker/mcmc-stats-server/Dockerfile .
 	@echo "Done."
 
 .PHONY: push-images
 push-images: images
 	@echo "Pushing docker images..."
-	docker push "${HUB_NAMESPACE}chainsail-celery-worker:latest"
-	docker push "${HUB_NAMESPACE}chainsail-scheduler:latest"
-	docker push "${HUB_NAMESPACE}chainsail-nginx:latest"
-	docker push "${HUB_NAMESPACE}chainsail-user-code:latest"
-	docker push "${HUB_NAMESPACE}chainsail-httpstan-server:latest"
-	docker push "${HUB_NAMESPACE}chainsail-mpi-node-k8s:latest"
-	docker push "${HUB_NAMESPACE}chainsail-mcmc-stats-server:latest"
+	docker push "${IMAGE_PREFIX}chainsail-celery-worker:latest"
+	docker push "${IMAGE_PREFIX}chainsail-scheduler:latest"
+	docker push "${IMAGE_PREFIX}chainsail-nginx:latest"
+	docker push "${IMAGE_PREFIX}chainsail-user-code:latest"
+	docker push "${IMAGE_PREFIX}chainsail-httpstan-server:latest"
+	docker push "${IMAGE_PREFIX}chainsail-mpi-node-k8s:latest"
+	docker push "${IMAGE_PREFIX}chainsail-mcmc-stats-server:latest"
 	@echo "Done."

app/client/run_dev_client.sh

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 
 SCHEDULER_URL=$(minikube service scheduler --url)
-MCMC_STATS_SERVER_URL=$(minikube service mcmc-stats-server --url)
+MCMC_STATS_URL=$(minikube service mcmc-stats-server --url)
 GRAPHITE_URL=$(minikube service graphite --url | head -n1)
 
-export SCHEDULER_URL=$SCHEDULER_URL
-export GRAPHITE_URL=$GRAPHITE_URL
-export MCMC_STATS_URL=$MCMC_STATS_SERVER_URL
+export SCHEDULER_URL
+export GRAPHITE_URL
+export MCMC_STATS_URL
 yarn run dev

docker/sshd.nix/.gitignore

@@ -0,0 +1 @@
+/result

docker/sshd.nix/default.nix

@@ -0,0 +1,62 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+let
+  config = ''
+    Port 2222
+    PermitRootLogin no
+    PasswordAuthentication no
+
+    AuthorizedKeysFile .ssh/authorized_keys
+
+    KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256
+    Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr
+    MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com
+
+    # modes are broken since the rewrite.. fix it!!
+    StrictModes no
+  '';
+in
+with pkgs; dockerTools.buildImage {
+  name = "chainsail-sshd";
+
+  runAsRoot = ''
+    #!${stdenv.shell}
+    echo ${dockerTools.shadowSetup}
+    ${dockerTools.shadowSetup}
+
+    useradd sshd
+    mkdir -p /var/empty /run
+
+    mkdir -p /etc/ssh
+    cfg=/etc/ssh/sshd_config
+    cp ${pkgs.openssh}/etc/ssh/sshd_config $cfg
+    echo 'PermitRootLogin yes' >>$cfg
+    echo 'PasswordAuthentication no' >>$cfg
+
+    # XXX: Is this okay?
+    ${pkgs.openssh}/bin/ssh-keygen -A
+
+    # setup ssh user
+#    groupadd -g 1337 -r git
+#    useradd -r -u 1337 -g git -d /data -M git
+#    usermod -p '*' git # set (invalid) password to only allow pubkey auth
+#    chsh -s ${pkgs.git}/bin/git-shell git
+#
+#    # create home dir for repo storage
+#    mkdir /data
+#
+#    # setup ssh user auth
+#    mkdir -p /data/.ssh
+#    touch /data/.ssh/authorized_keys
+#    chmod go-w /data/
+#    chmod 700 /data/.ssh
+#    chmod 600 /data/.ssh/authorized_keys
+  '';
+
+  config = {
+    Cmd = [ "${pkgs.openssh}/bin/sshd" "-e" "-D" "-p" "26" ];
+    ExposedPorts = {
+      "26/tcp" = {};
+    };
+  };
+}

docker/sshd/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.8.7-slim
+
+RUN mkdir -p /run/sshd
+
+RUN apt-get update && \
+    apt-get install -y curl wget unzip openssh-client openssh-server && \
+    rm -rf /var/lib/apt/lists/*
+
+# Global SSH configurations
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# Disable password authentication
+RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config
+# Disable StrictMode (required for k8s secret volumes to work properly since their symlinks have open permissions)
+RUN sed -i 's/#StrictModes yes/StrictModes no/' /etc/ssh/sshd_config
+# Listen on a non-default port to avoid clashing with the host's ssh
+RUN sed -i 's/#Port 22/Port 26/' /etc/ssh/sshd_config
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+ADD ssh/ssh_config /etc/ssh/ssh_config
+ADD entrypoint.sh /app/entrypoint.sh
+
+# For SSH. Using non-default ssh port.
+EXPOSE 26
+
+ENTRYPOINT ["/app/entrypoint.sh"]

docker/sshd/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+mkdir -p /root/.ssh
+
+exec "$@"

docker/sshd/ssh/ssh_config

@@ -0,0 +1,5 @@
+Host *
+  IdentityFile /root/.ssh/id.pem
+  StrictHostKeyChecking accept-new
+  UserKnownHostsFile /root/known_hosts
+  Port 26

docs/deployment.md

@@ -26,15 +26,6 @@ This guide describes how to set up a Chainsail environment from scratch, either
     - [Allowing / disallowing application users](#allowing--disallowing-application-users)
 #
 
-## Prerequisites for both Minikube and Google Cloud deployment
-Make sure that you correctly edit the Terraform and Helm files.
-Specifically, you'll want to make sure that you have matching container registry in
-the following files / environment variables:
-- `/terraform/cluster/local/main.tf` (`container_registry` in the `locals` block),
-- `/helm/values.yaml` (`imageHubNamespace`),
-- `/helm/values-local.yaml` (`imageHubNamespace`),
-- `/helm/values-dev.yaml` (`imageHubNamespace`),if you're considering a Google Cloud deployment,
-- and the `HUB_NAMESPACE` environment variable later.
 
 ## Local deployment
 
@@ -67,9 +58,8 @@ and add them to Minikube's Docker registry. One way to do this is:
 # This makes docker commands use Minikube's Docker daemon
 eval $(minikube docker-env)
 
-HUB_NAMESPACE="<container registry>/" make images
+make images
 ```
-The hub namespace environment variable has to match the value of the `imageHubNamespace` property in `helm/values.yaml` and the `container_registry` value in the `locals` block of `terraform/cluster/local/main.tf`.
 
 Then, you can install Chainsail with Helm:
 
@@ -151,24 +141,34 @@ In order to be able to push the images to the container registry, the Google Clo
 It is called something like `<eu, ...>.artifacts.<project name>` bucket.
 The name of the bucket might vary depending on the `zone` and `node_location` entries in the `chainsail_gcp` Terraform module in `terraform/base/dev/main.tf`.
 
+Configure the image prefix for container images by creating a shell file call
+`.config/env.private.sh` containing something like:
+
+```
+export IMAGE_PREFIX='<your-container-registry>'
+export TF_VAR_image_prefix="$IMAGE_PREFIX"
+```
+
+Setting `TF_VAR_image_prefix` ensures that Terraform will use the same image
+prefix.
+
 To build and push images, run
 ```bash
-HUB_NAMESPACE="<container registry>/" make push-images
+make push-images
 ```
 
-The hub namespace environment variable has to match the value of the `imageHubNamespace` property in `helm/values.yaml`.
-
 The first time you deploy Chainsail, you will need to fetch the cluster's Kubernetes access credentials using `gcloud`:
 
 ```bash
 gcloud container clusters get-credentials --region $GCP_REGION chainsail
 ```
+
 The GCP region can be found in `terraform/base/dev/main.tf` in the `node_location` entry of the `chainsail_gcp` module.
 
 Once all of the desired images are published, you can install Chainsail with:
 
 ```bash
-helm install -f helm/values-dev.yaml chainsail ./helm
+helm install -f helm/values-dev.yaml --set imagePrefix=$IMAGE_PREFIX chainsail ./helm
 ```
 
 ### 4. Deploy Chainsail front-end
@@ -198,7 +198,7 @@ There are a couple of additional steps which need to be configured manually:
 To upgrade an already running Chainsail cluster to a newer version of the chart. Use:
 
 ```bash
-helm upgrade -f helm/values-dev.yaml chainsail ./helm
+helm upgrade -f helm/values-dev.yaml --set imagePrefix=$IMAGE_PREFIX chainsail ./helm
 ```
 
 If using the `latest` tag for images in the helm chart, you will also need to restart the services so that

flake.nix

@@ -47,6 +47,26 @@
         };
         controller = poetry2nixPkg.mkPoetryApplication controllerOpts;
         controllerEnv = poetry2nixPkg.mkPoetryEnv controllerOpts;
+        sshdUser = pkgs.runCommand "user-setup" { } ''
+          mkdir -p $out/etc/
+
+          echo 'sshd:x:105:65534::/run/sshd:/usr/sbin/nologin' >>$out/etc/passwd
+          echo "root:x:0:0:root user:/root:${pkgs.bash}/bin/bash" >> $out/etc/passwd
+        '';
+        controller-image = pkgs.dockerTools.streamLayeredImage {
+          name = "chainsail-mpi-node-k8s";
+          contents = [
+            controller
+            pkgs.openssh
+            sshdUser
+
+            # debug aids
+            pkgs.bashInteractive
+            pkgs.coreutils
+            pkgs.ps
+          ];
+          config.Cmd = [ "chainsail-controller" ];
+        };
 
         basePackages = [
           pkgs.docker-compose
@@ -63,21 +83,33 @@
           ourYarn
         ];
 
-        pythonDevShell = ps: pkgs.mkShell {
-          packages = ps ++ basePackages;
+        mkDevShell = {extraPkgs ? []}: pkgs.mkShell {
+          packages = extraPkgs ++ basePackages;
+          shellHook = ''
+            private_env_sh=.config/env.private.sh
+            if [[ -r $private_env_sh ]]; then
+              echo loading $private_env_sh
+              source $private_env_sh
+            fi
+            unset private_env_sh
+            PATH=$PWD/script:$PATH
+          '';
+        };
+        controllerDevShell = mkDevShell {
+          extraPkgs = [
+            controllerEnv
+            controller
+          ];
         };
-        controllerDevShell = pythonDevShell [
-          controllerEnv
-          controller
-        ];
       in
       {
         devShells = {
           controller = controllerDevShell;
-          default = pkgs.mkShell { packages = basePackages; };
+          default = mkDevShell {};
         };
         packages = {
           inherit controller;
+          inherit controller-image;
           scheduler = poetry2nixPkg.mkPoetryApplication {
             projectDir = ./app/scheduler;
             overrides = poetry2nixPkg.overrides.withDefaults (

helm/templates/scheduler-workers.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: chainsail-scheduler
       automountServiceAccountToken: true
       containers:
-      - image: {{ .Values.imageHubNamespace }}chainsail-celery-worker:{{ .Values.imageTag }}
+      - image: {{ .Values.imagePrefix }}chainsail-celery-worker:{{ .Values.imageTag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         name: scheduler-worker
         resources: {}

helm/templates/scheduler.yaml

@@ -17,7 +17,7 @@ spec:
       tolerations:
 {{ toYaml .Values.tolerations | indent 8}}
       containers:
-      - image: {{ .Values.imageHubNamespace }}chainsail-scheduler:{{ .Values.imageTag }}
+      - image: {{ .Values.imagePrefix }}chainsail-scheduler:{{ .Values.imageTag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         name: scheduler
         ports:

helm/templates/stats.yaml

@@ -17,7 +17,7 @@ spec:
       tolerations:
 {{ toYaml .Values.tolerations | indent 8}}
       containers:
-      - image: {{ .Values.imageHubNamespace }}chainsail-mcmc-stats-server:{{ .Values.imageTag }}
+      - image: {{ .Values.imagePrefix }}chainsail-mcmc-stats-server:{{ .Values.imageTag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         name: mcmc-stats-server
         ports:
@@ -37,4 +37,4 @@ spec:
       volumes:
       - name: storage-config
         secret:
-          secretName: storage-yaml
+          secretName: storage-yaml

helm/values-dev.yaml

@@ -17,13 +17,6 @@ tolerations:
     value: 'true'
     effect: 'NoSchedule'
 
-# This has to match the `imageHubNamespace` value in `values-dev.yaml`
-# Furthermore, this value with an additional forward slash is the
-# `HUB_NAMESPACE` environment variable you have to set when building the
-# Docker images using the Makefile.
-imageHubNamespace: 'eu.gcr.io/project-name'
-imageTag: 'latest'
-
 ##############################################################################
 # Scheduler Settings
 ##############################################################################

helm/values-local.yaml

@@ -17,13 +17,6 @@ tolerations:
     value: 'true'
     effect: 'NoSchedule'
 
-# This has to match the `container_registry` value in the `locals` block of
-# `terraform/cluster/local/main.tf`.
-# Furthermore, this value is the `HUB_NAMESPACE` environment variable you
-# have to set when building the Docker images using the Makefile.
-imageHubNamespace: 'eu.gcr.io/project-name/'
-imageTag: 'latest'
-
 ##############################################################################
 # Scheduler Settings
 ##############################################################################