diff --git a/.env b/.env index d61ae725a..441e0cba1 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ +AUXILIARY_WORKER_THREAD_COUNT=1 CAPYBARA_SERVER=web CHROME_HOSTNAME=seleniarm-hub DB_ADAPTER=postgresql diff --git a/app/jobs/create_large_derivatives_job.rb b/app/jobs/create_large_derivatives_job.rb index 6f678c40f..c068d5152 100644 --- a/app/jobs/create_large_derivatives_job.rb +++ b/app/jobs/create_large_derivatives_job.rb @@ -2,9 +2,9 @@ # CreateLargeDerivativesJob is intended to be used for resource-intensive derivative # generation (e.g. video processing). It is functionally similar to CreateDerivativesJob, -# except that it queues jobs in the :resource_intensive queue. +# except that it queues jobs in the :auxiliary queue. # -# The worker responsible for processing jobs in the :resource_intensive queue should be +# The worker responsible for processing jobs in the :auxiliary queue should be # configured to have more resources dedicated to it, especially CPU. Otherwise, the # `ffmpeg` commands that this job class eventually triggers could be throttled. # @@ -12,5 +12,5 @@ # @see Hydra::Derivatives::Processors::Ffmpeg # @see https://github.com/scientist-softserv/palni-palci/issues/852 class CreateLargeDerivativesJob < CreateDerivativesJob - queue_as :resource_intensive + queue_as :auxiliary end diff --git a/bin/helm_deploy b/bin/helm_deploy index 631b57b9f..9972877c6 100755 --- a/bin/helm_deploy +++ b/bin/helm_deploy @@ -23,7 +23,7 @@ WORKER_IMAGE="${WORKER_IMAGE:-ghcr.io/samvera/hyku/worker}" DEPLOY_TAG="${DEPLOY_TAG:-latest}" WORKER_TAG="${WORKER_TAG:-$DEPLOY_TAG}" -helm pull oci://ghcr.io/samvera/charts/hyrax --version 2.0.0 --untar --untardir charts +helm pull oci://ghcr.io/samvera/charts/hyrax --version 3.5.1 --untar --untardir charts helm repo update diff --git a/bin/worker b/bin/worker index b7605486b..a369ffc11 100755 --- a/bin/worker +++ b/bin/worker @@ -9,4 +9,8 @@ else puts 'DATABASE_URL not set, no pool change needed' end -exec "echo $DATABASE_URL && bundle exec sidekiq" +if ENV['SIDEKIQ_CONFIG'] + exec "echo $DATABASE_URL && bundle exec sidekiq -C #{ENV['SIDEKIQ_CONFIG']}" +else + exec "echo $DATABASE_URL && bundle exec sidekiq" +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index fb2531c82..a72d01223 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -3,4 +3,3 @@ - default - import - export - - resource_intensive diff --git a/config/sidekiq_auxiliary.yml b/config/sidekiq_auxiliary.yml new file mode 100644 index 000000000..24eee633e --- /dev/null +++ b/config/sidekiq_auxiliary.yml @@ -0,0 +1,7 @@ +--- +:concurrency: <%= ENV['AUXILIARY_WORKER_THREAD_COUNT'] %> +:queues: + - auxiliary + - default + - import + - export diff --git a/docker-compose.yml b/docker-compose.yml index 900b85592..a6d040d7a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,35 @@ x-app: &app networks: internal: +x-app-worker: &app-worker + <<: *app + build: + context: . + target: hyku-worker + args: + - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool + - HYKU_BULKRAX_ENABLED=true + cache_from: + - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest} + - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest} + image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest} + command: sh -l -c 'bundle && bundle exec sidekiq' + depends_on: + check_volumes: + condition: service_completed_successfully + initialize_app: + condition: service_completed_successfully + db: + condition: service_started + solr: + condition: service_started + fcrepo: + condition: service_started + redis: + condition: service_started + zoo: + condition: service_started + volumes: assets: cache: @@ -145,6 +174,8 @@ services: condition: service_started worker: condition: service_started + worker_auxiliary: + condition: service_started initialize_app: condition: service_completed_successfully # ports: @@ -153,33 +184,11 @@ services: - 3000 worker: - <<: *app - build: - context: . - target: hyku-worker - args: - - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool - - HYKU_BULKRAX_ENABLED=true - cache_from: - - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest} - - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest} - image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest} - command: sh -l -c 'bundle && bundle exec sidekiq' - depends_on: - check_volumes: - condition: service_completed_successfully - initialize_app: - condition: service_completed_successfully - db: - condition: service_started - solr: - condition: service_started - fcrepo: - condition: service_started - redis: - condition: service_started - zoo: - condition: service_started + <<: *app-worker + + worker_auxiliary: + <<: *app-worker + command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_auxiliary.yml' # Do not recurse through all of tmp. derivitives will make booting # very slow and eventually just time out as data grows diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 2080c4523..43656223f 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -47,14 +47,15 @@ ingress: - host: commons-archive.org paths: - path: / + pathType: ImplementationSpecific - host: "*.commons-archive.org" paths: - path: / - annotations: { - kubernetes.io/ingress.class: "nginx", - nginx.ingress.kubernetes.io/proxy-body-size: "0", - cert-manager.io/cluster-issuer: letsencrypt-production-dns - } + pathType: ImplementationSpecific + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + cert-manager.io/cluster-issuer: "letsencrypt-production-dns" tls: - hosts: - commons-archive.org @@ -191,6 +192,167 @@ worker: runAsGroup: 101 fsGroup: 101 fsGroupChangePolicy: "OnRootMismatch" + +# When adding/removing key-value pairs to this block, ensure the +# corresponding changes are made in the `extraDeploy` block below. +workerAuxiliary: + replicaCount: 1 + resources: + limits: + memory: "12Gi" + cpu: "4" + requests: + memory: "4Gi" + cpu: "2" + extraEnvVars: + - name: AUXILIARY_WORKER_THREAD_COUNT + value: "1" + - name: SIDEKIQ_CONFIG + value: "config/sidekiq_auxiliary.yml" + +extraDeploy: + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "hyrax.fullname" . }}-auxiliary-worker + labels: + {{- include "hyrax.labels" . | nindent 4 }} + spec: + replicas: {{ .Values.workerAuxiliary.replicaCount }} + selector: + matchLabels: + {{- include "hyrax.workerSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hyrax.workerSelectorLabels" . | nindent 8 }} + spec: + initContainers: + - name: db-wait + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + command: + - sh + - -c + - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379" + {{- if .Values.worker.extraInitContainers }} + {{- toYaml .Values.worker.extraInitContainers | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hyrax.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + {{- if .Values.solrExistingSecret }} + - secretRef: + name: {{ .Values.solrExistingSecret }} + {{- end }} + {{- with .Values.worker.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + {{- end }} + volumeMounts: + - name: derivatives + mountPath: /app/samvera/derivatives + - name: uploads + subPath: file_cache + mountPath: /app/samvera/file_cache + - name: uploads + subPath: uploads + mountPath: /app/samvera/uploads + {{- if .Values.applicationExistingClaim }} + - name: application + mountPath: /app/samvera/hyrax-webapp + {{- end }} + {{- with .Values.worker.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }} + {{- with .Values.extraContainerConfiguration }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + - name: "derivatives" + {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.derivativesVolume.existingClaim }} + {{- else if .Values.derivativesVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-derivatives + {{ else }} + emptyDir: {} + {{ end }} + - name: "uploads" + {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.uploadsVolume.existingClaim }} + {{- else if .Values.uploadsVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-uploads + {{ else }} + emptyDir: {} + {{ end }} + {{- if .Values.applicationExistingClaim }} + - name: "application" + persistentVolumeClaim: + claimName: {{ .Values.applicationExistingClaim }} + {{- end }} + {{- with .Values.worker.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + podSecurityContext: runAsUser: 1001 runAsGroup: 101 @@ -208,11 +370,16 @@ fcrepo: postgresql: enabled: false redis: - cluster: - enabled: false - password: $REDIS_PASSWORD + enabled: true + architecture: standalone + auth: + password: $REDIS_PASSWORD solr: enabled: false +fits: + enabled: true + servicePort: 8080 + subPath: /fits externalPostgresql: host: postgres-cluster-alpha-ha.postgres.svc.cluster.local @@ -226,7 +393,7 @@ externalSolrCollection: demo-palni-palci externalSolrPassword: $SOLR_ADMIN_PASSWORD global: - hyraxName: palni-palci-demo-pals + hyraxHostName: palni-palci-demo-pals nginx: enabled: true @@ -238,7 +405,7 @@ nginx: tag: 1.21.5-debian-10-r4 serverBlock: |- upstream rails_app { - server {{ .Values.global.hyraxName }}; + server {{ .Values.global.hyraxHostName }}; } map ${DOLLAR}status ${DOLLAR}loggable { diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml index 175e703a5..3338bc92e 100644 --- a/ops/production-deploy.tmpl.yaml +++ b/ops/production-deploy.tmpl.yaml @@ -47,14 +47,15 @@ ingress: - host: hykucommons.org paths: - path: / + pathType: ImplementationSpecific - host: "*.hykucommons.org" paths: - path: / - annotations: { - kubernetes.io/ingress.class: "nginx", - nginx.ingress.kubernetes.io/proxy-body-size: "0", - cert-manager.io/cluster-issuer: letsencrypt-production-dns - } + pathType: ImplementationSpecific + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + cert-manager.io/cluster-issuer: "letsencrypt-production-dns" tls: - hosts: - hykucommons.org @@ -200,7 +201,6 @@ worker: requests: memory: "2Gi" cpu: "150m" - extraVolumeMounts: *volMounts extraEnvVars: *envVars podSecurityContext: @@ -208,6 +208,167 @@ worker: runAsGroup: 101 fsGroup: 101 fsGroupChangePolicy: "OnRootMismatch" + +# When adding/removing key-value pairs to this block, ensure the +# corresponding changes are made in the `extraDeploy` block below. +workerAuxiliary: + replicaCount: 1 + resources: + limits: + memory: "12Gi" + cpu: "4" + requests: + memory: "4Gi" + cpu: "2" + extraEnvVars: + - name: AUXILIARY_WORKER_THREAD_COUNT + value: "1" + - name: SIDEKIQ_CONFIG + value: "config/sidekiq_auxiliary.yml" + +extraDeploy: + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "hyrax.fullname" . }}-auxiliary-worker + labels: + {{- include "hyrax.labels" . | nindent 4 }} + spec: + replicas: {{ .Values.workerAuxiliary.replicaCount }} + selector: + matchLabels: + {{- include "hyrax.workerSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hyrax.workerSelectorLabels" . | nindent 8 }} + spec: + initContainers: + - name: db-wait + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + command: + - sh + - -c + - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379" + {{- if .Values.worker.extraInitContainers }} + {{- toYaml .Values.worker.extraInitContainers | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hyrax.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + {{- if .Values.solrExistingSecret }} + - secretRef: + name: {{ .Values.solrExistingSecret }} + {{- end }} + {{- with .Values.worker.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + {{- end }} + volumeMounts: + - name: derivatives + mountPath: /app/samvera/derivatives + - name: uploads + subPath: file_cache + mountPath: /app/samvera/file_cache + - name: uploads + subPath: uploads + mountPath: /app/samvera/uploads + {{- if .Values.applicationExistingClaim }} + - name: application + mountPath: /app/samvera/hyrax-webapp + {{- end }} + {{- with .Values.worker.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }} + {{- with .Values.extraContainerConfiguration }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + - name: "derivatives" + {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.derivativesVolume.existingClaim }} + {{- else if .Values.derivativesVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-derivatives + {{ else }} + emptyDir: {} + {{ end }} + - name: "uploads" + {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.uploadsVolume.existingClaim }} + {{- else if .Values.uploadsVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-uploads + {{ else }} + emptyDir: {} + {{ end }} + {{- if .Values.applicationExistingClaim }} + - name: "application" + persistentVolumeClaim: + claimName: {{ .Values.applicationExistingClaim }} + {{- end }} + {{- with .Values.worker.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + podSecurityContext: runAsUser: 1001 runAsGroup: 101 @@ -225,11 +386,16 @@ fcrepo: postgresql: enabled: false redis: - cluster: - enabled: false - password: $REDIS_PASSWORD + enabled: true + architecture: standalone + auth: + password: $REDIS_PASSWORD solr: enabled: false +fits: + enabled: true + servicePort: 8080 + subPath: /fits externalPostgresql: host: postgres-cluster-alpha-ha.postgres.svc.cluster.local @@ -243,7 +409,7 @@ externalSolrCollection: production-palni-palci externalSolrPassword: $SOLR_ADMIN_PASSWORD global: - hyraxName: palni-palci-production-pals + hyraxHostName: palni-palci-production-pals nginx: enabled: true @@ -255,7 +421,7 @@ nginx: tag: 1.21.5-debian-10-r4 serverBlock: |- upstream rails_app { - server {{ .Values.global.hyraxName }}; + server {{ .Values.global.hyraxHostName }}; } map ${DOLLAR}status ${DOLLAR}loggable { diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml index 3c6d0ebd2..b93ee50ea 100644 --- a/ops/staging-deploy.tmpl.yaml +++ b/ops/staging-deploy.tmpl.yaml @@ -47,14 +47,15 @@ ingress: - host: palni-palci-staging.notch8.cloud paths: - path: / + pathType: ImplementationSpecific - host: "*.palni-palci-staging.notch8.cloud" paths: - path: / - annotations: { - kubernetes.io/ingress.class: "nginx", - nginx.ingress.kubernetes.io/proxy-body-size: "0", - cert-manager.io/cluster-issuer: letsencrypt-production-dns - } + pathType: ImplementationSpecific + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + cert-manager.io/cluster-issuer: "letsencrypt-production-dns" tls: - hosts: - palni-palci-staging.notch8.cloud @@ -182,7 +183,6 @@ worker: limits: memory: "2Gi" cpu: "1000m" - extraVolumeMounts: *volMounts extraEnvVars: *envVars podSecurityContext: @@ -190,6 +190,167 @@ worker: runAsGroup: 101 fsGroup: 101 fsGroupChangePolicy: "OnRootMismatch" + +# When adding/removing key-value pairs to this block, ensure the +# corresponding changes are made in the `extraDeploy` block below. +workerAuxiliary: + replicaCount: 1 + resources: + limits: + memory: "12Gi" + cpu: "4" + requests: + memory: "4Gi" + cpu: "2" + extraEnvVars: + - name: AUXILIARY_WORKER_THREAD_COUNT + value: "1" + - name: SIDEKIQ_CONFIG + value: "config/sidekiq_auxiliary.yml" + +extraDeploy: + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "hyrax.fullname" . }}-auxiliary-worker + labels: + {{- include "hyrax.labels" . | nindent 4 }} + spec: + replicas: {{ .Values.workerAuxiliary.replicaCount }} + selector: + matchLabels: + {{- include "hyrax.workerSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hyrax.workerSelectorLabels" . | nindent 8 }} + spec: + initContainers: + - name: db-wait + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + command: + - sh + - -c + - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379" + {{- if .Values.worker.extraInitContainers }} + {{- toYaml .Values.worker.extraInitContainers | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hyrax.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + {{- if .Values.solrExistingSecret }} + - secretRef: + name: {{ .Values.solrExistingSecret }} + {{- end }} + {{- with .Values.worker.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + {{- end }} + volumeMounts: + - name: derivatives + mountPath: /app/samvera/derivatives + - name: uploads + subPath: file_cache + mountPath: /app/samvera/file_cache + - name: uploads + subPath: uploads + mountPath: /app/samvera/uploads + {{- if .Values.applicationExistingClaim }} + - name: application + mountPath: /app/samvera/hyrax-webapp + {{- end }} + {{- with .Values.worker.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }} + {{- with .Values.extraContainerConfiguration }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + - name: "derivatives" + {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.derivativesVolume.existingClaim }} + {{- else if .Values.derivativesVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-derivatives + {{ else }} + emptyDir: {} + {{ end }} + - name: "uploads" + {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.uploadsVolume.existingClaim }} + {{- else if .Values.uploadsVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-uploads + {{ else }} + emptyDir: {} + {{ end }} + {{- if .Values.applicationExistingClaim }} + - name: "application" + persistentVolumeClaim: + claimName: {{ .Values.applicationExistingClaim }} + {{- end }} + {{- with .Values.worker.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + podSecurityContext: runAsUser: 1001 runAsGroup: 101 @@ -206,11 +367,16 @@ fcrepo: postgresql: enabled: false redis: - cluster: - enabled: false - password: $REDIS_PASSWORD + enabled: true + architecture: standalone + auth: + password: $REDIS_PASSWORD solr: enabled: false +fits: + enabled: true + servicePort: 8080 + subPath: /fits externalFcrepoHost: fcrepo.staging-fcrepo.svc.cluster.local externalPostgresql: diff --git a/spec/jobs/create_large_derivatives_job_spec.rb b/spec/jobs/create_large_derivatives_job_spec.rb index b39f6d82a..1ed67ef2c 100644 --- a/spec/jobs/create_large_derivatives_job_spec.rb +++ b/spec/jobs/create_large_derivatives_job_spec.rb @@ -19,10 +19,10 @@ allow(file_set).to receive(:update_index) end - it 'runs in the :resource_intensive queue' do + it 'runs in the :auxiliary queue' do expect { described_class.perform_later(file_set, file.id) } .to have_enqueued_job(described_class) - .on_queue('resource_intensive') + .on_queue('auxiliary') end # @see CreateDerivativesJobDecorator#perform