diff --git a/applications/cm-service/README.md b/applications/cm-service/README.md index c3ed233f19..49aeacd49a 100644 --- a/applications/cm-service/README.md +++ b/applications/cm-service/README.md @@ -10,6 +10,9 @@ Campaign Management for Rubin Data Release Production | Key | Type | Default | Description | |-----|------|---------|-------------| +| config.butler.storage | string | `"1Gi"` | Minimum storage requested in the butler remote area PVC | +| config.butler.storageClassName | string | `nil` | If specified, name of storage class requested in butler remote area PVC | +| config.butler.subPath | string | `nil` | If specified, sub-path within bound PV to be mounted as butler remote area | | config.db.echo | bool | `false` | Whether to echo SQLAlchemy generated SQL to the log | | config.db.hostname | string | `""` | Name of the database host | | config.db.name | string | `"cmservice"` | Name of the database to use for the application | @@ -39,9 +42,6 @@ Campaign Management for Rubin Data Release Production | ingress.annotations | object | `{}` | Additional annotations for the frontend ingress rule | | internalDB | bool | `false` | Whether to use the internal (phalanx) database | | worker.affinity | object | `{}` | Affinity rules for the worker pods | -| worker.butler.dbUrl | string | `nil` | URL of a Butler Registry Database | -| worker.butler.dbUsername | string | `nil` | Name of a user to use with the Butler Registry Database | -| worker.butler.mountPath | string | `nil` | If specified, location for htcondor config file to be injected into worker containers | | worker.htcondor.config.contents | string | `nil` | If specified, contents of htcondor config file to be injected into worker containers | | worker.htcondor.config.mountPath | string | `nil` | If specified, location for htcondor config file to be injected into worker containers | | worker.htcondor.fsRemoteDir.storage | string | `"1Gi"` | Minimum storage requested in the condor remote area PVC | diff --git a/applications/cm-service/templates/api-deployment.yaml b/applications/cm-service/templates/api-deployment.yaml index 9bb7cb2c32..149bbcef59 100644 --- a/applications/cm-service/templates/api-deployment.yaml +++ b/applications/cm-service/templates/api-deployment.yaml @@ -31,6 +31,8 @@ spec: - configMapRef: name: {{ .Chart.Name }}-env env: + - name: BUTLER__MOCK + value: "1" - name: DB__PASSWORD valueFrom: secretKeyRef: @@ -54,12 +56,13 @@ spec: drop: - "all" readOnlyRootFilesystem: true + # TODO: we hopefully do not actually need this mounted on the API volumeMounts: - - mountPath: "/output" + {{- with .Values.config.outputVolume }} + - mountPath: {{ .mountPath }} name: "output-volume" - {{- with .Values.config.outputVolume.subPath }} - subPath: {{ . }} - {{- end }} + subPath: {{ .subPath }} + {{- end }} volumes: - name: output-volume persistentVolumeClaim: diff --git a/applications/cm-service/templates/configmap.yaml b/applications/cm-service/templates/configmap.yaml index e345349928..3efbdbfc87 100644 --- a/applications/cm-service/templates/configmap.yaml +++ b/applications/cm-service/templates/configmap.yaml @@ -6,7 +6,6 @@ metadata: {{- include "application.labels" . | nindent 4 }} data: ASGI__PREFIX: {{ .Values.config.pathPrefix | quote }} - BUTLER__MOCK: "1" {{- with .Values.config.db }} DB__URL: "postgresql://{{ .username }}@{{ .hostname }}:{{ .port }}/{{ .name }}" DB__ECHO: {{ .echo | quote }} @@ -16,13 +15,28 @@ data: --- apiVersion: v1 kind: ConfigMap +metadata: + name: "{{ .Chart.Name }}-butler-env" + labels: + {{- include "application.labels" . | nindent 4 }} +data: + BUTLER__REPO__0__NAME: "/repo/main" + BUTLER__REPO__0__URI: "/sdf/group/rubin/repo/main/butler.yaml" + BUTLER__REPO__1__NAME: "/repo/main+sasquatch_dev" + BUTLER__REPO__1__URI: "/sdf/group/rubin/repo/main/butler+sasquatch_dev.yaml" +--- +apiVersion: v1 +kind: ConfigMap metadata: name: "{{ .Chart.Name }}-htcondor-env" labels: {{- include "application.labels" . | nindent 4 }} data: - CONDOR_CONFIG: /home/lsstsvc1/.config/condor/condor-config - DAF_BUTLER_PASSWORD: placeholder_for_secret + DAEMON__allocate_resources: "1" + HTCONDOR__schedd_host: sdfiana012.sdf.slac.stanford.edu + HTCONDOR__collector_host: sdfiana012.sdf.slac.stanford.edu + HTCONDOR__condor_home: /opt/htcondor HTCONDOR__condor_submit_bin: /opt/htcondor/bin/condor_submit HTCONDOR__condor_q_bin: /opt/htcondor/bin/condor_q - HTCONDOR__alias_path: /sdf/group/rubin/{{ .Values.config.outputVolume.subPath }} + HTCONDOR__working_directory: /sdf/group/rubin/{{ .Values.config.outputVolume.subPath }} + HTCONDOR__fs_remote_dir: /sdf/group/rubin/services/htcondor/shared diff --git a/applications/cm-service/templates/pvc.yaml b/applications/cm-service/templates/pvc.yaml index b4a54110bb..0300a7cb10 100644 --- a/applications/cm-service/templates/pvc.yaml +++ b/applications/cm-service/templates/pvc.yaml @@ -17,6 +17,22 @@ spec: --- apiVersion: v1 kind: PersistentVolumeClaim +metadata: + name: {{ $.Chart.Name }}-butler + labels: + {{- include "application.labels" $ | nindent 4 }} +spec: + {{- with .Values.config.butler }} + storageClassName: {{ .storageClassName }} + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{ .storage }} + {{- end }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim metadata: name: {{ $.Chart.Name }}-htcondor labels: diff --git a/applications/cm-service/templates/worker-configmap.yaml b/applications/cm-service/templates/worker-configmap.yaml index 21667b7574..634ef307d8 100644 --- a/applications/cm-service/templates/worker-configmap.yaml +++ b/applications/cm-service/templates/worker-configmap.yaml @@ -9,9 +9,3 @@ data: config: | {{- .contents | nindent 4 }} {{- end }} - {{- with .Values.worker.butler }} - butlerAuth: | - - url: {{ .dbUrl }} - username: {{ .dbUsername }} - password: ${env:DAF_BUTLER_PASSWORD} - {{- end }} diff --git a/applications/cm-service/templates/worker-deployment.yaml b/applications/cm-service/templates/worker-deployment.yaml index edaedee1df..9c79b7135f 100644 --- a/applications/cm-service/templates/worker-deployment.yaml +++ b/applications/cm-service/templates/worker-deployment.yaml @@ -20,6 +20,11 @@ spec: {{- include "application.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: "worker" spec: + securityContext: + runAsUser: 17951 + runAsGroup: 1126 + fsGroup: 1126 + runAsNonRoot: true {{- with .Values.worker.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -37,12 +42,20 @@ spec: httpGet: path: "/healthz" port: "http" + livenessProbe: + httpGet: + path: "/healthz" + port: "http" envFrom: - configMapRef: name: {{ .Chart.Name }}-env + - configMapRef: + name: {{ .Chart.Name }}-butler-env - configMapRef: name: {{ .Chart.Name }}-htcondor-env env: + - name: BUTLER__AUTHENTICATION_FILE + value: "/var/run/secrets/db-auth.yaml" - name: DB__PASSWORD valueFrom: secretKeyRef: @@ -55,54 +68,48 @@ spec: capabilities: drop: - "all" - readOnlyRootFilesystem: true + # readOnlyRootFilesystem: true volumeMounts: - - mountPath: "/output" - name: "output-volume" - {{- with .Values.config.outputVolume.subPath }} - subPath: {{ . }} - {{- end }} - - mountPath: "/tmp" - name: "tmp-volume" - {{- with .Values.worker.htcondor.config }} + {{- with .Values.config.outputVolume }} - mountPath: {{ .mountPath }} - name: "condor-config-volume" - readOnly: true + name: output-volume + subPath: {{ .subPath }} {{- end }} - {{- with .Values.worker.butler }} + {{- with .Values.config.butler }} - mountPath: {{ .mountPath }} - name: "butler-auth-volume" - readOnly: true + name: butler-repo-volume + subPath: {{ .subPath }} {{- end }} + - mountPath: "/tmp" + name: "tmp-volume" {{- with .Values.worker.htcondor.fsRemoteDir }} - mountPath: {{ .mountPath }} - name: "condor-remote-volume" - {{- with .subPath }} - subPath: {{ . }} - {{- end }} + name: condor-remote-volume + subPath: {{ .subPath }} {{- end }} + - name: butler-registry-volume + mountPath: "/var/run/secrets" + readonly: true volumes: - name: output-volume persistentVolumeClaim: claimName: {{ .Chart.Name }}-output + - name: butler-repo-volume + persistentVolumeClaim: + claimName: {{ .Chart.Name }}-butler - name: tmp-volume emptyDir: sizeLimit: 500Mi - - name: condor-config-volume - configMap: - name: {{ $.Chart.Name }}-config-worker - items: - - key: config - path: condor-config - name: condor-remote-volume persistentVolumeClaim: claimName: {{ $.Chart.Name }}-htcondor - - name: butler-auth-volume - configMap: - name: {{ $.Chart.Name }}-config-worker + - name: butler-registry-volume + secret: + secretName: cm-service-butler items: - - key: butlerAuth + - key: butler-registry path: db-auth.yaml + mode: 0600 {{- with .Values.worker.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/applications/cm-service/values-usdf-cm-dev.yaml b/applications/cm-service/values-usdf-cm-dev.yaml index 7d961ae67e..2266b8dabb 100644 --- a/applications/cm-service/values-usdf-cm-dev.yaml +++ b/applications/cm-service/values-usdf-cm-dev.yaml @@ -1,6 +1,12 @@ +# service writes artifacts that eventually resolve to the path +# /sdf/group/rubin/shared/campaigns/users/usdf-cm-dev/// +# outputVolume: mounted at the container's path /output +# sdf-data-rubin == /sdf/group/rubin +# sdf-group-rubin == /sdf/group/rubin +--- internalDB: true config: - logLevel: "INFO" + logLevel: "DEBUG" logProfile: "development" db: hostname: "postgres.postgres" @@ -8,20 +14,21 @@ config: outputVolume: storageClassName: "sdf-data-rubin" subPath: "shared/campaigns/users/usdf-cm-dev" + mountPath: "/sdf/group/rubin/shared/campaigns/users/usdf-cm-dev" + butler: + storageClassName: "sdf-data-rubin" + subPath: "repo/main" + mountPath: "/sdf/group/rubin/repo/main" image: - tag: "tickets-DM-48099-release" + tag: "tickets-DM-48560-resources" pullPolicy: Always worker: image: - tag: "tickets-DM-48099-release" + tag: "tickets-DM-48560-resources" pullPolicy: Always - butler: - mountPath: "/home/lsstsvc1/.config/butler" - dbUrl: "postgresql://usdf-butler.slac.stanford.edu:5432/lsstdb1" - dbUsername: rubin htcondor: config: - mountPath: "/home/lsstsvc1/.config/condor" + mountPath: "/usr/local/etc" contents: | CONDOR_HOST = sdfiana012.sdf.slac.stanford.edu COLLECTOR_HOST = sdfiana012.sdf.slac.stanford.edu diff --git a/applications/cm-service/values.yaml b/applications/cm-service/values.yaml index 6bbb216ec2..a199d14100 100644 --- a/applications/cm-service/values.yaml +++ b/applications/cm-service/values.yaml @@ -42,6 +42,17 @@ config: # -- URL path prefix pathPrefix: "/cm-service" + butler: + # -- If specified, name of storage class requested in butler remote area PVC + storageClassName: null + + # -- Minimum storage requested in the butler remote area PVC + storage: 1Gi + + # -- If specified, sub-path within bound PV to be mounted as butler remote area + subPath: null + + image: # -- Image to use for frontend containers repository: "ghcr.io/lsst-dm/cm-service" @@ -131,16 +142,6 @@ worker: # -- Tolerations for the worker pods tolerations: [] - butler: - # -- If specified, location for htcondor config file to be injected into worker containers - mountPath: null - - # -- URL of a Butler Registry Database - dbUrl: null - - # -- Name of a user to use with the Butler Registry Database - dbUsername: null - htcondor: host: