cm-service: configure htcondor

applications/cm-service/README.md

@@ -10,6 +10,9 @@ Campaign Management for Rubin Data Release Production
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| config.butler.storage | string | `"1Gi"` | Minimum storage requested in the butler remote area PVC |
+| config.butler.storageClassName | string | `nil` | If specified, name of storage class requested in butler remote area PVC |
+| config.butler.subPath | string | `nil` | If specified, sub-path within bound PV to be mounted as butler remote area |
 | config.db.echo | bool | `false` | Whether to echo SQLAlchemy generated SQL to the log |
 | config.db.hostname | string | `""` | Name of the database host |
 | config.db.name | string | `"cmservice"` | Name of the database to use for the application |
@@ -39,9 +42,6 @@ Campaign Management for Rubin Data Release Production
 | ingress.annotations | object | `{}` | Additional annotations for the frontend ingress rule |
 | internalDB | bool | `false` | Whether to use the internal (phalanx) database |
 | worker.affinity | object | `{}` | Affinity rules for the worker pods |
-| worker.butler.dbUrl | string | `nil` | URL of a Butler Registry Database |
-| worker.butler.dbUsername | string | `nil` | Name of a user to use with the Butler Registry Database |
-| worker.butler.mountPath | string | `nil` | If specified, location for htcondor config file to be injected into worker containers |
 | worker.htcondor.config.contents | string | `nil` | If specified, contents of htcondor config file to be injected into worker containers |
 | worker.htcondor.config.mountPath | string | `nil` | If specified, location for htcondor config file to be injected into worker containers |
 | worker.htcondor.fsRemoteDir.storage | string | `"1Gi"` | Minimum storage requested in the condor remote area PVC |

applications/cm-service/templates/api-deployment.yaml

@@ -31,6 +31,8 @@ spec:
             - configMapRef:
                 name: {{ .Chart.Name }}-env
           env:
+            - name: BUTLER__MOCK
+              value: "1"
             - name: DB__PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -54,12 +56,13 @@ spec:
               drop:
                 - "all"
             readOnlyRootFilesystem: true
+          # TODO: we hopefully do not actually need this mounted on the API
           volumeMounts:
-            - mountPath: "/output"
+            {{- with .Values.config.outputVolume }}
+            - mountPath: {{ .mountPath }}
               name: "output-volume"
-              {{- with .Values.config.outputVolume.subPath }}
-              subPath: {{ . }}
-              {{- end }}
+              subPath: {{ .subPath }}
+            {{- end }}
       volumes:
         - name: output-volume
           persistentVolumeClaim:

applications/cm-service/templates/configmap.yaml

@@ -6,7 +6,6 @@ metadata:
     {{- include "application.labels" . | nindent 4 }}
 data:
   ASGI__PREFIX: {{ .Values.config.pathPrefix | quote }}
-  BUTLER__MOCK: "1"
   {{- with .Values.config.db }}
   DB__URL: "postgresql://{{ .username }}@{{ .hostname }}:{{ .port }}/{{ .name }}"
   DB__ECHO: {{ .echo | quote }}
@@ -16,13 +15,28 @@ data:
 ---
 apiVersion: v1
 kind: ConfigMap
+metadata:
+  name: "{{ .Chart.Name }}-butler-env"
+  labels:
+    {{- include "application.labels" . | nindent 4 }}
+data:
+  BUTLER__REPO__0__NAME: "/repo/main"
+  BUTLER__REPO__0__URI: "/sdf/group/rubin/repo/main/butler.yaml"
+  BUTLER__REPO__1__NAME: "/repo/main+sasquatch_dev"
+  BUTLER__REPO__1__URI: "/sdf/group/rubin/repo/main/butler+sasquatch_dev.yaml"
+---
+apiVersion: v1
+kind: ConfigMap
 metadata:
   name: "{{ .Chart.Name }}-htcondor-env"
   labels:
     {{- include "application.labels" . | nindent 4 }}
 data:
-  CONDOR_CONFIG: /home/lsstsvc1/.config/condor/condor-config
-  DAF_BUTLER_PASSWORD: placeholder_for_secret
+  DAEMON__allocate_resources: "1"
+  HTCONDOR__schedd_host: sdfiana012.sdf.slac.stanford.edu
+  HTCONDOR__collector_host: sdfiana012.sdf.slac.stanford.edu
+  HTCONDOR__condor_home: /opt/htcondor
   HTCONDOR__condor_submit_bin: /opt/htcondor/bin/condor_submit
   HTCONDOR__condor_q_bin: /opt/htcondor/bin/condor_q
-  HTCONDOR__alias_path: /sdf/group/rubin/{{ .Values.config.outputVolume.subPath }}
+  HTCONDOR__working_directory: /sdf/group/rubin/{{ .Values.config.outputVolume.subPath }}
+  HTCONDOR__fs_remote_dir: /sdf/group/rubin/services/htcondor/shared

applications/cm-service/templates/pvc.yaml

@@ -17,6 +17,22 @@ spec:
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
+metadata:
+  name: {{ $.Chart.Name }}-butler
+  labels:
+    {{- include "application.labels" $ | nindent 4 }}
+spec:
+  {{- with .Values.config.butler }}
+  storageClassName: {{ .storageClassName }}
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: {{ .storage }}
+  {{- end }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
 metadata:
   name: {{ $.Chart.Name }}-htcondor
   labels:

applications/cm-service/templates/worker-configmap.yaml

@@ -9,9 +9,3 @@ data:
   config: |
     {{- .contents | nindent 4 }}
   {{- end }}
-  {{- with .Values.worker.butler }}
-  butlerAuth: |
-    - url: {{ .dbUrl }}
-      username: {{ .dbUsername }}
-      password: ${env:DAF_BUTLER_PASSWORD}
-  {{- end }}

applications/cm-service/templates/worker-deployment.yaml

@@ -20,6 +20,11 @@ spec:
         {{- include "application.selectorLabels" . | nindent 8 }}
         app.kubernetes.io/component: "worker"
     spec:
+      securityContext:
+        runAsUser: 17951
+        runAsGroup: 1126
+        fsGroup: 1126
+        runAsNonRoot: true
       {{- with .Values.worker.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
@@ -37,12 +42,20 @@ spec:
             httpGet:
               path: "/healthz"
               port: "http"
+          livenessProbe:
+            httpGet:
+              path: "/healthz"
+              port: "http"
           envFrom:
             - configMapRef:
                 name: {{ .Chart.Name }}-env
+            - configMapRef:
+                name: {{ .Chart.Name }}-butler-env
             - configMapRef:
                 name: {{ .Chart.Name }}-htcondor-env
           env:
+            - name: BUTLER__AUTHENTICATION_FILE
+              value: "/var/run/secrets/db-auth.yaml"
             - name: DB__PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -55,54 +68,48 @@ spec:
             capabilities:
               drop:
                 - "all"
-            readOnlyRootFilesystem: true
+            # readOnlyRootFilesystem: true
           volumeMounts:
-            - mountPath: "/output"
-              name: "output-volume"
-              {{- with .Values.config.outputVolume.subPath }}
-              subPath: {{ . }}
-              {{- end }}
-            - mountPath: "/tmp"
-              name: "tmp-volume"
-            {{- with .Values.worker.htcondor.config }}
+            {{- with .Values.config.outputVolume }}
             - mountPath: {{ .mountPath }}
-              name: "condor-config-volume"
-              readOnly: true
+              name: output-volume
+              subPath: {{ .subPath }}
             {{- end }}
-            {{- with .Values.worker.butler }}
+            {{- with .Values.config.butler }}
             - mountPath: {{ .mountPath }}
-              name: "butler-auth-volume"
-              readOnly: true
+              name: butler-repo-volume
+              subPath: {{ .subPath }}
             {{- end }}
+            - mountPath: "/tmp"
+              name: "tmp-volume"
             {{- with .Values.worker.htcondor.fsRemoteDir }}
             - mountPath: {{ .mountPath }}
-              name: "condor-remote-volume"
-              {{- with .subPath }}
-              subPath: {{ . }}
-              {{- end }}
+              name: condor-remote-volume
+              subPath: {{ .subPath }}
             {{- end }}
+            - name: butler-registry-volume
+              mountPath: "/var/run/secrets"
+              readonly: true
       volumes:
         - name: output-volume
           persistentVolumeClaim:
             claimName: {{ .Chart.Name }}-output
+        - name: butler-repo-volume
+          persistentVolumeClaim:
+            claimName: {{ .Chart.Name }}-butler
         - name: tmp-volume
           emptyDir:
             sizeLimit: 500Mi
-        - name: condor-config-volume
-          configMap:
-            name: {{ $.Chart.Name }}-config-worker
-            items:
-              - key: config
-                path: condor-config
         - name: condor-remote-volume
           persistentVolumeClaim:
             claimName: {{ $.Chart.Name }}-htcondor
-        - name: butler-auth-volume
-          configMap:
-            name: {{ $.Chart.Name }}-config-worker
+        - name: butler-registry-volume
+          secret:
+            secretName: cm-service-butler
             items:
-              - key: butlerAuth
+              - key: butler-registry
                 path: db-auth.yaml
+                mode: 0600
       {{- with .Values.worker.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

applications/cm-service/values-usdf-cm-dev.yaml

@@ -1,27 +1,34 @@
+# service writes artifacts that eventually resolve to the path
+#   /sdf/group/rubin/shared/campaigns/users/usdf-cm-dev/<production>/<campaign>/
+# outputVolume: mounted at the container's path /output
+#  sdf-data-rubin == /sdf/group/rubin
+#  sdf-group-rubin == /sdf/group/rubin
+---
 internalDB: true
 config:
-  logLevel: "INFO"
+  logLevel: "DEBUG"
   logProfile: "development"
   db:
     hostname: "postgres.postgres"
     echo: true
   outputVolume:
     storageClassName: "sdf-data-rubin"
     subPath: "shared/campaigns/users/usdf-cm-dev"
+    mountPath: "/sdf/group/rubin/shared/campaigns/users/usdf-cm-dev"
+  butler:
+    storageClassName: "sdf-data-rubin"
+    subPath: "repo/main"
+    mountPath: "/sdf/group/rubin/repo/main"
 image:
-  tag: "tickets-DM-48099-release"
+  tag: "tickets-DM-48560-resources"
   pullPolicy: Always
 worker:
   image:
-    tag: "tickets-DM-48099-release"
+    tag: "tickets-DM-48560-resources"
     pullPolicy: Always
-  butler:
-    mountPath: "/home/lsstsvc1/.config/butler"
-    dbUrl: "postgresql://usdf-butler.slac.stanford.edu:5432/lsstdb1"
-    dbUsername: rubin
   htcondor:
     config:
-      mountPath: "/home/lsstsvc1/.config/condor"
+      mountPath: "/usr/local/etc"
       contents: |
         CONDOR_HOST = sdfiana012.sdf.slac.stanford.edu
         COLLECTOR_HOST = sdfiana012.sdf.slac.stanford.edu

applications/cm-service/values.yaml

@@ -42,6 +42,17 @@ config:
   # -- URL path prefix
   pathPrefix: "/cm-service"
 
+  butler:
+    # -- If specified, name of storage class requested in butler remote area PVC
+    storageClassName: null
+
+    # -- Minimum storage requested in the butler remote area PVC
+    storage: 1Gi
+
+    # -- If specified, sub-path within bound PV to be mounted as butler remote area
+    subPath: null
+
+
 image:
   # -- Image to use for frontend containers
   repository: "ghcr.io/lsst-dm/cm-service"
@@ -131,16 +142,6 @@ worker:
   # -- Tolerations for the worker pods
   tolerations: []
 
-  butler:
-    # -- If specified, location for htcondor config file to be injected into worker containers
-    mountPath: null
-
-    # -- URL of a Butler Registry Database
-    dbUrl: null
-
-    # -- Name of a user to use with the Butler Registry Database
-    dbUsername: null
-
   htcondor:
 
     host: