From 1939d7735d7ddf4b05f5fcd3c22dee093296088c Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 18 Apr 2024 14:23:09 -0700 Subject: [PATCH] bring the aux worker code over --- bin/worker | 8 +- docker-compose.yml | 63 ++++++++----- ops/production-deploy.tmpl.yaml | 151 +++++++++++++++++++++++++++++++ ops/staging-deploy.tmpl.yaml | 152 ++++++++++++++++++++++++++++++++ 4 files changed, 351 insertions(+), 23 deletions(-) diff --git a/bin/worker b/bin/worker index edfb2576..e605849a 100755 --- a/bin/worker +++ b/bin/worker @@ -9,4 +9,10 @@ else puts 'DATABASE_URL not set, no pool change needed' end -exec "echo $DATABASE_URL && bundle exec good_job start" \ No newline at end of file +if ENV['AUX_WORKER'] + # Run all queues (including :auxiliary) + exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='*' bundle exec good_job" +else + # Run all queues (excluding :auxiliary) + exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='-auxiliary' bundle exec good_job" +end diff --git a/docker-compose.yml b/docker-compose.yml index 5c7cc75c..d62ef5ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,30 @@ x-app: &app - cache:/app/samvera/hyrax-webapp/tmp/cache:cached - .:/app/samvera +x-app-worker: &app-worker + <<: *app + extends: + file: hyrax-webapp/docker-compose.yml + service: worker + build: + context: . + target: hyku-worker + cache_from: + - ghcr.io/samvera/hyku/base:latest + - ghcr.io/samvera-labs/hyku_knapsack/worker:latest + args: + BUILDKIT_INLINE_CACHE: 1 + APP_PATH: ./hyrax-webapp + image: ghcr.io/scientist-softserv/adventist_knapsack/worker:${TAG:-latest} + # Uncomment command to access container with out starting bin/worker. Useful for debugging or updating Gemfile.lock + # command: sleep infinity + volumes: + - node_modules:/app/samvera/hyrax-webapp/node_modules:cached + - uploads:/app/samvera/hyrax-webapp/public/uploads:cached + - assets:/app/samvera/hyrax-webapp/public/assets:cached + - cache:/app/samvera/hyrax-webapp/tmp/cache:cached + - .:/app/samvera + volumes: assets: cache: @@ -91,28 +115,23 @@ services: # command: sleep infinity worker: - <<: *app - extends: - file: hyrax-webapp/docker-compose.yml - service: worker - build: - context: . - target: hyku-worker - cache_from: - - ghcr.io/samvera/hyku/base:latest - - ghcr.io/samvera-labs/hyku_knapsack/worker:latest - args: - BUILDKIT_INLINE_CACHE: 1 - APP_PATH: ./hyrax-webapp - image: ghcr.io/scientist-softserv/adventist_knapsack/worker:${TAG:-latest} - # Uncomment command to access container with out starting bin/worker. Useful for debugging or updating Gemfile.lock - # command: sleep infinity - volumes: - - node_modules:/app/samvera/hyrax-webapp/node_modules:cached - - uploads:/app/samvera/hyrax-webapp/public/uploads:cached - - assets:/app/samvera/hyrax-webapp/public/assets:cached - - cache:/app/samvera/hyrax-webapp/tmp/cache:cached - - .:/app/samvera + <<: *app-worker + + worker_aux: + <<: *app-worker + environment: + - AUX_WORKER="true" + - AUXILIARY_QUEUE_TENANTS="sdapi" # This line is what makes the knapsack include use the local code instead of the remote gem + - AWS_REGION=us-east-1 + - AWS_S3_BUCKET=space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh + - BUNDLE_BUNDLER_INJECT__GEM_PATH=/app/samvera/bundler.d + - BUNDLE_DISABLE_LOCAL_BRANCH_CHECK=true + - BUNDLE_LOCAL__HYKU_KNAPSACK=/app/samvera # This line is what makes the knapsack include use the local code instead of the remote gem + - FITS_SERVLET_URL=http://fits:8080/fits + - GOOD_JOB_CLEANUP_DISCARDED_JOBS=false + - GOOD_JOB_CLEANUP_INTERVAL_SECONDS=86400 + - GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO=604800 + - HYRAX_ACTIVE_JOB_QUEUE=good_job check_volumes: <<: *app diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml index 3a373499..a7ae0dc8 100644 --- a/ops/production-deploy.tmpl.yaml +++ b/ops/production-deploy.tmpl.yaml @@ -256,6 +256,157 @@ worker: extraVolumeMounts: *volMounts extraEnvVars: *envVars +# When adding/removing key-value pairs to this block, ensure the +# corresponding changes are made in the `extraDeploy` block below. +workerAux: + extraEnvVars: + # GOOD_JOB_QUEUES gets set in bin/worker + - name: AUX_WORKER + value: "true" + +extraDeploy: + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "hyrax.fullname" . }}-worker-aux + labels: + {{- include "hyrax.labels" . | nindent 4 }} + spec: + replicas: {{ .Values.worker.replicaCount }} + selector: + matchLabels: + {{- include "hyrax.workerSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hyrax.workerSelectorLabels" . | nindent 8 }} + spec: + initContainers: + - name: db-wait + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + env: + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }} + command: + - sh + - -c + - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379" + {{- if .Values.worker.extraInitContainers }} + {{- toYaml .Values.worker.extraInitContainers | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hyrax.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + {{- if .Values.solrExistingSecret }} + - secretRef: + name: {{ .Values.solrExistingSecret }} + {{- end }} + {{- with .Values.worker.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + {{- end }} + volumeMounts: + - name: derivatives + mountPath: /app/samvera/derivatives + - name: uploads + subPath: file_cache + mountPath: /app/samvera/file_cache + - name: uploads + subPath: uploads + mountPath: /app/samvera/uploads + {{- if .Values.applicationExistingClaim }} + - name: application + mountPath: /app/samvera/hyrax-webapp + {{- end }} + {{- with .Values.worker.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + {{- with .Values.extraContainerConfiguration }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + - name: "derivatives" + {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.derivativesVolume.existingClaim }} + {{- else if .Values.derivativesVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-derivatives + {{ else }} + emptyDir: {} + {{ end }} + - name: "uploads" + {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.uploadsVolume.existingClaim }} + {{- else if .Values.uploadsVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-uploads + {{ else }} + emptyDir: {} + {{ end }} + {{- if .Values.applicationExistingClaim }} + - name: "application" + persistentVolumeClaim: + claimName: {{ .Values.applicationExistingClaim }} + {{- end }} + {{- with .Values.worker.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + podSecurityContext: runAsUser: 1001 runAsGroup: 101 diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml index 82782dfe..eddfaa26 100644 --- a/ops/staging-deploy.tmpl.yaml +++ b/ops/staging-deploy.tmpl.yaml @@ -274,6 +274,158 @@ worker: extraVolumeMounts: *volMounts extraEnvVars: *envVars +# When adding/removing key-value pairs to this block, ensure the +# corresponding changes are made in the `extraDeploy` block below. +workerAux: + extraEnvVars: + # GOOD_JOB_QUEUES gets set in bin/worker + - name: AUX_WORKER + value: "true" + +extraDeploy: + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "hyrax.fullname" . }}-worker-aux + labels: + {{- include "hyrax.labels" . | nindent 4 }} + spec: + replicas: {{ .Values.worker.replicaCount }} + selector: + matchLabels: + {{- include "hyrax.workerSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hyrax.workerSelectorLabels" . | nindent 8 }} + spec: + initContainers: + - name: db-wait + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + env: + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }} + command: + - sh + - -c + - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379" + {{- if .Values.worker.extraInitContainers }} + {{- toYaml .Values.worker.extraInitContainers | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hyrax.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + {{- if .Values.solrExistingSecret }} + - secretRef: + name: {{ .Values.solrExistingSecret }} + {{- end }} + {{- with .Values.worker.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + {{- end }} + volumeMounts: + - name: derivatives + mountPath: /app/samvera/derivatives + - name: uploads + subPath: file_cache + mountPath: /app/samvera/file_cache + - name: uploads + subPath: uploads + mountPath: /app/samvera/uploads + {{- if .Values.applicationExistingClaim }} + - name: application + mountPath: /app/samvera/hyrax-webapp + {{- end }} + {{- with .Values.worker.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + {{- with .Values.extraContainerConfiguration }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + - name: "derivatives" + {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.derivativesVolume.existingClaim }} + {{- else if .Values.derivativesVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-derivatives + {{ else }} + emptyDir: {} + {{ end }} + - name: "uploads" + {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.uploadsVolume.existingClaim }} + {{- else if .Values.uploadsVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-uploads + {{ else }} + emptyDir: {} + {{ end }} + {{- if .Values.applicationExistingClaim }} + - name: "application" + persistentVolumeClaim: + claimName: {{ .Values.applicationExistingClaim }} + {{- end }} + {{- with .Values.worker.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + + podSecurityContext: runAsUser: 1001 runAsGroup: 101