From 22b3d910effb85364b38a32e18759d8c024bece1 Mon Sep 17 00:00:00 2001 From: Josh Mandel Date: Mon, 11 Sep 2023 16:42:08 -0700 Subject: [PATCH] Isolate zulip and ssh keys from build container (#34) --- images/ig-build/Dockerfile | 7 +- images/ig-build/builder/builder.py | 40 ++++--- images/ig-build/builder/util.py | 23 ---- images/ig-build/watch-and-publish | 37 +++++++ k8s/create.sh | 6 + k8s/dev-setup.md | 27 +++++ k8s/example-job-for-minikube.yaml | 83 ++++++++++++++ triggers/ig-commit-trigger/job.json | 165 +++++++++++++++++----------- 8 files changed, 279 insertions(+), 109 deletions(-) create mode 100755 images/ig-build/watch-and-publish create mode 100644 k8s/example-job-for-minikube.yaml diff --git a/images/ig-build/Dockerfile b/images/ig-build/Dockerfile index dcfc986a..6e3c3bb6 100644 --- a/images/ig-build/Dockerfile +++ b/images/ig-build/Dockerfile @@ -1,14 +1,14 @@ FROM openjdk:21-jdk-bullseye MAINTAINER Josh Mandel -RUN apt-get update && apt-get -y install python3 python3-pip gosu openssl wget graphviz ruby2.7 ruby2.7-dev +RUN apt-get update && apt-get -y install python3 python3-pip gosu openssl wget graphviz ruby2.7 ruby2.7-dev inotify-tools RUN pip3 install --upgrade requests zulip RUN gem install jekyll jekyll-asciidoc RUN cd /tmp && \ - wget --quiet https://nodejs.org/dist/latest-v18.x/node-v18.16.0-linux-x64.tar.xz && \ + wget --quiet https://nodejs.org/dist/v18.17.1/node-v18.17.1-linux-x64.tar.xz && \ cd /usr/local && \ - tar --strip-components 1 -xf /tmp/node-v18.16.0-linux-x64.tar.xz + tar --strip-components 1 -xf /tmp/node-v18.17.1-linux-x64.tar.xz # Install required packages @@ -27,6 +27,7 @@ RUN mkdir -p /root/.ssh && \ IdentitiesOnly yes" > /root/.ssh/config && \ chmod go-wrx /root/.ssh/config +ADD watch-and-publish /usr/local/bin/watch-and-publish ADD publish /usr/local/bin/publish ENTRYPOINT python3 -m builder.builder || true diff --git a/images/ig-build/builder/builder.py b/images/ig-build/builder/builder.py index 4fc2b703..a103ec49 100644 --- a/images/ig-build/builder/builder.py +++ b/images/ig-build/builder/builder.py @@ -1,13 +1,10 @@ import logging import os -import random -import requests import shutil import string import subprocess -import sys -from .util import make_temp_dir, do, send_zulip +from .util import do, SCRATCH_SPACE from os.path import normpath GITHUB = 'https://github.com/%(org)s/%(repo)s' @@ -26,19 +23,30 @@ def get_qa_score(build_dir): except: return "No QA File" - def build(config): - - if config['branch'] == 'gh-pages': - sys.exit(0) - - temp_dir = make_temp_dir() + temp_dir = SCRATCH_SPACE clone_dir = os.path.join(temp_dir, 'repo') build_dir = os.path.join(clone_dir, 'output') logfile = os.path.join(temp_dir, 'build.log') + upload_dir = os.path.join(SCRATCH_SPACE, 'upload') logging.basicConfig(filename=logfile, level=logging.DEBUG) logging.info('about to clone!') + def finalize(result_dir, message, pubargs): + os.rename(result_dir, upload_dir) + message_path = os.path.join(SCRATCH_SPACE, 'message') + with open(message_path, 'w') as f: + f.write(message) + + # Write each argument to a new line in a temporary file + done_path = os.path.join(SCRATCH_SPACE, 'done') + done_temp_path = done_path + '.temp' + with open(done_temp_path, 'w') as f: + for arg in pubargs: + f.write(arg + '\n') + # Atomically rename the temporary file to the desired file name + os.rename(done_temp_path, done_path) + def run_git_cmd(cmds): return subprocess.check_output(cmds, cwd=clone_dir, universal_newlines=True).strip() @@ -78,14 +86,14 @@ def is_default_branch(): message = ["**[%(org)s/%(repo)s: %(branch)s](https://github.com/%(org)s/%(repo)s/tree/%(branch)s)** rebuilt\n", "Commit: %(commit)s :%(emoji)s:\n", "Details: [build logs](%(root)s/%(org)s/%(repo)s/branches/%(branch)s/%(buildlog)s)"] - + print("finalizing") if not built: print("Build error occurred") details['emoji'] = 'thumbs_down' details['buildlog'] = 'failure/build.log' message += [" | [debug](%(root)s/%(org)s/%(repo)s/branches/%(branch)s/failure)"] shutil.copy(logfile, clone_dir) - do(['publish', details['org'], details['repo'], details['branch'], 'failure', details['default']], clone_dir, pipe=True) + finalize(clone_dir, "".join(message)%details, [details['org'], details['repo'], details['branch'], 'failure', details['default']]) else: print("Build succeeded") details['emoji'] = 'thumbs_up' @@ -94,12 +102,8 @@ def is_default_branch(): message += [" | [qa: %s]"%get_qa_score(build_dir), "(%(root)s/%(org)s/%(repo)s/branches/%(branch)s/qa.html)"] print("Copying logfile") shutil.copy(logfile, build_dir) - print("publishing") - do(['publish', details['org'], details['repo'], details['branch'], 'success', details['default']], build_dir, pipe=True) - print("published") - - send_zulip('committers/notification', 'ig-build', "".join(message)%details) - # sys.exit(0 if built else 1) + finalize(build_dir, "".join(message)%details, [details['org'], details['repo'], details['branch'], 'success', details['default']]) + print("finalized") if __name__ == '__main__': build({ diff --git a/images/ig-build/builder/util.py b/images/ig-build/builder/util.py index 0e8a5f9f..34e583ad 100644 --- a/images/ig-build/builder/util.py +++ b/images/ig-build/builder/util.py @@ -1,24 +1,14 @@ import datetime import logging import os -import random import string import subprocess -import zulip - -ZULIP_API = os.environ.get('ZULIP_API', 'https://chat.fhir.org') SCRATCH_SPACE = os.environ.get('SCRATCH', '/scratch') DEADLINE_SECONDS = int(os.environ.get('DEADLINE_SECONDS', '1800')) DEADLINE_BUFFER = int(os.environ.get('DEADLINE_BUFFER', '120')) deadline_time = datetime.datetime.now() + datetime.timedelta(seconds=(DEADLINE_SECONDS-DEADLINE_BUFFER)) -def make_temp_dir(prefix='ig-build-temp-', N=6): - dirname = prefix + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(N)) - dirpath = os.path.abspath(os.path.join(SCRATCH_SPACE, dirname)) - os.makedirs(dirpath) - return dirpath - def do(args, cwd=SCRATCH_SPACE, pipe=False, deadline=False): logging.debug('running: %s'%" ".join(args)) logfile = logging.getLoggerClass().root.handlers[0].baseFilename @@ -36,16 +26,3 @@ def do(args, cwd=SCRATCH_SPACE, pipe=False, deadline=False): pr.kill() logging.debug("\n\n*** Timeout -- deadline reached") return 1 - -def send_zulip(stream, topic, content): - logging.debug('zulip messaging: %s %s %s'%(stream, topic, content)) - zulip.Client( - site=ZULIP_API, - api_key=os.environ.get('ZULIP_API_KEY'), - email=os.environ.get('ZULIP_EMAIL') - ).send_message({ - 'type': 'stream', - 'content': content, - 'to': stream, - 'subject': topic - }) diff --git a/images/ig-build/watch-and-publish b/images/ig-build/watch-and-publish new file mode 100755 index 00000000..6e1b724f --- /dev/null +++ b/images/ig-build/watch-and-publish @@ -0,0 +1,37 @@ +#!/bin/bash + +# Wait for 'done' file to appear +echo "Waiting for the builder to complete" + +cd /scratch + +while true; do + inotifywait -e create -q -q . && [ -f done ] && break +done +echo "Builder completed; uploading" + +# At this point, both 'message' and 'done' files are present +# Read each line from 'done' and store as positional arguments +pubargs=() +while IFS= read -r line; do + pubargs+=("$line") +done < /scratch/done + +echo "Message" +cat /scratch/message + +echo "publish" +cat /scratch/done + +# Then, Run publish.sh with the arguments from the 'done' file +cd /scratch/upload +publish "${pubargs[@]}" + +# Send Zulip message +echo "Uploaded; notifying zulip" +zulip-send --stream committers/notification --subject ig-build \ + --message "$(< /scratch/message)" \ + --user "$ZULIP_EMAIL" \ + --api-key "$ZULIP_API_KEY" \ + --site https://chat.fhir.org +echo "Notified" diff --git a/k8s/create.sh b/k8s/create.sh index d5e2ccbe..25f64599 100644 --- a/k8s/create.sh +++ b/k8s/create.sh @@ -2,6 +2,12 @@ gcloud compute disks create fhir-ci-build-disk --size 100GB kubectl -n fhir create secret generic zulip-secrets --from-literal=email=$ZULIP_EMAIL --from-literal=api_key=$ZULIP_API_KEY +ssh-keygen -t rsa -f id +kubectl -n fhir create secret generic ci-build-keys --from-file=id --from-file=id.pub + +# get these from Grahame ;-) +kubectl -n fhir create secret generic fhir-settings --from-file=keyfile.ini --from-file=fhir-settings.json + gcloud compute disks create caddy-cert-disk --size=10GB --zone=us-east1-d kubectl -n fhir create configmap caddy-conf-volume --from-file Caddyfile diff --git a/k8s/dev-setup.md b/k8s/dev-setup.md index 4c758db5..6c0d2761 100644 --- a/k8s/dev-setup.md +++ b/k8s/dev-setup.md @@ -41,3 +41,30 @@ gcloud docker -- push gcr.io/fhir-org-starter-project/ci-build cd triggers/ig-commit-trigger gcloud functions deploy ig-commit-trigger --runtime nodejs18 --trigger-http ``` + + +--- + +## Testing locally with minikube + +```sh +minikube config set memory 20000 +minikube start +eval $(minikube -p minikube docker-env) + +kubectl create ns fhir + +ssh-keygen -t rsa -f id +kubectl -n fhir create secret generic ci-build-keys --from-file=id --from-file=id.pub + +echo "" > keyfile.ini +echo "{}" > fhir-settings.json +kubectl -n fhir create secret generic fhir-settings --from-file=keyfile.ini --from-file=fhir-settings.json + +kubectl -n fhir create secret generic zulip-secrets --from-literal=email=bot@hsot --from-literal=api_key=zapi +``` + +Build the igbuild image as `igbuild` in minikube docker + + kubectl apply -f example-job-for-minikube.yaml + diff --git a/k8s/example-job-for-minikube.yaml b/k8s/example-job-for-minikube.yaml new file mode 100644 index 00000000..8adffb61 --- /dev/null +++ b/k8s/example-job-for-minikube.yaml @@ -0,0 +1,83 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generation: 1 + name: buildk + namespace: fhir +spec: + activeDeadlineSeconds: 36000 + backoffLimit: 6 + completions: 1 + parallelism: 1 + suspend: false + template: + spec: + containers: + - name: ig-upload + image: igbuild + imagePullPolicy: Never + command: ["/usr/local/bin/watch-and-publish"] + args: [] + env: + - name: ZULIP_EMAIL + valueFrom: + secretKeyRef: + key: email + name: zulip-secrets + - name: ZULIP_API_KEY + valueFrom: + secretKeyRef: + key: api_key + name: zulip-secrets + resources: + requests: + memory: 512Mi + volumeMounts: + - name: scratch + mountPath: /scratch + - mountPath: /etc/ci_build_keys + name: keys + - name: ig-build + image: igbuild + imagePullPolicy: Never + env: + - name: PUBLISHER_JAR_URL + value: https://github.com/HL7/fhir-ig-publisher/releases/latest/download/publisher.jar + - name: TX_SERVER_URL + value: http://tx.fhir.org + - name: DEADLINE_SECONDS + value: "36000" + - name: JAVA_MEMORY + value: 19000m + - name: IG_ORG + value: "HL7" + - name: IG_REPO + value: "smart-app-launch" + - name: IG_BRANCH + value: "master" + resources: + limits: + memory: 30Gi + requests: + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /scratch + - mountPath: /etc/ig.builder.keyfile.ini + name: fhir-settings + subPath: ig.builder.keyfile.ini + - mountPath: /etc/fhir-settings.json + name: fhir-settings + subPath: fhir-settings.json + restartPolicy: Never + volumes: + - name: scratch + emptyDir: {} + - name: keys + secret: + defaultMode: 256 + secretName: ci-build-keys + - name: fhir-settings + secret: + defaultMode: 256 + secretName: fhir-settings diff --git a/triggers/ig-commit-trigger/job.json b/triggers/ig-commit-trigger/job.json index f495eda9..f813f06a 100644 --- a/triggers/ig-commit-trigger/job.json +++ b/triggers/ig-commit-trigger/job.json @@ -9,85 +9,120 @@ "activeDeadlineSeconds": 36000, "ttlSecondsAfterFinished": 30, "template": { - "metadata": { - "labels": { }, - "name": "ig-build" - }, "spec": { - "volumes": [ - { - "name": "keys", - "secret": { - "secretName": "ci-build-keys", - "defaultMode": 256 - } - }, - { - "name": "fhir-settings", - "secret": { - "secretName": "fhir-settings", - "defaultMode": 256 - } - } - ], "containers": [ { - "name": "ig-build", + "env": [ + { + "name": "PUBLISHER_JAR_URL", + "value": "https://github.com/HL7/fhir-ig-publisher/releases/latest/download/publisher.jar" + }, + { + "name": "TX_SERVER_URL", + "value": "http://tx.fhir.org" + }, + { + "name": "DEADLINE_SECONDS", + "value": "36000" + }, + { + "name": "JAVA_MEMORY", + "value": "19000m" + } + ], "image": "gcr.io/fhir-org-starter-project/ig-build", "imagePullPolicy": "Always", - "env": [{ - "name": "PUBLISHER_JAR_URL", - "value": "https://github.com/HL7/fhir-ig-publisher/releases/latest/download/publisher.jar" - }, { - "name": "TX_SERVER_URL", - "value": "http://tx.fhir.org" - }, { - "name": "DEADLINE_SECONDS", - "value": "36000" - }, { - "name": "JAVA_MEMORY", - "value": "19000m" - }, { - "name": "ZULIP_EMAIL", - "valueFrom": { - "secretKeyRef": { - "name": "zulip-secrets", - "key": "email" - } + "name": "ig-build", + "resources": { + "limits": { + "memory": "30Gi" + }, + "requests": { + "memory": "22Gi" } - }, { - "name": "ZULIP_API_KEY", - "valueFrom": { - "secretKeyRef": { - "name": "zulip-secrets", - "key": "api_key" + }, + "volumeMounts": [ + { + "mountPath": "/scratch", + "name": "scratch" + }, + { + "mountPath": "/etc/ig.builder.keyfile.ini", + "name": "fhir-settings", + "subPath": "ig.builder.keyfile.ini" + }, + { + "mountPath": "/etc/fhir-settings.json", + "name": "fhir-settings", + "subPath": "fhir-settings.json" + } + ] + }, { + "args": [], + "command": [ + "/usr/local/bin/watch-and-publish" + ], + "env": [ + { + "name": "ZULIP_EMAIL", + "valueFrom": { + "secretKeyRef": { + "key": "email", + "name": "zulip-secrets" + } + } + }, + { + "name": "ZULIP_API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "api_key", + "name": "zulip-secrets" + } } } - }], - "volumeMounts": [{ - "name": "keys", - "mountPath": "/etc/ci_build_keys" - },{ - "name": "keys", - "subPath": "ig.builder.keyfile.ini", - "mountPath": "/etc/ig.builder.keyfile.ini" - }, - { - "name": "fhir-settings", - "subPath": "fhir-settings.json", - "mountPath": "/etc/fhir-settings.json" - }], + ], + "image": "gcr.io/fhir-org-starter-project/ig-build", + "imagePullPolicy": "Always", + "name": "ig-upload", "resources": { "requests": { - "memory": "22Gi" + "memory": "512Mi" + } + }, + "volumeMounts": [ + { + "mountPath": "/scratch", + "name": "scratch" }, - "limits": { - "memory": "30Gi" + { + "mountPath": "/etc/ci_build_keys", + "name": "keys" } - } + ] } ], - "restartPolicy": "Never" + "restartPolicy": "Never", + "volumes": [ + { + "emptyDir": {}, + "name": "scratch" + }, + { + "name": "keys", + "secret": { + "defaultMode": 256, + "secretName": "ci-build-keys" + } + }, + { + "name": "fhir-settings", + "secret": { + "defaultMode": 256, + "secretName": "fhir-settings" + } + } + ] } } }