From 92956e3f178b3195ffd92c796161b3fd3847d1d9 Mon Sep 17 00:00:00 2001 From: LaRita Robinson Date: Thu, 15 Feb 2024 15:54:45 -0600 Subject: [PATCH] Split jobs to a new queue Based on an array of tenant names from a config var, this will send all jobs for that tenant to the "auxiliary" queue, and increase the priority by 110. This causes the worker to pick up that tenant's jobs first, before picking up lower priority jobs. Refs #721 --- .env | 2 +- Gemfile | 4 +-- Gemfile.lock | 10 +++--- app/jobs/application_job.rb | 55 ++++++++++++++++++++++++++++++--- config/initializers/good_job.rb | 33 +++++--------------- config/initializers/hyrax.rb | 2 ++ lib/active_job_tenant.rb | 8 +++++ ops/dev-deploy.tmpl.yaml | 4 +-- 8 files changed, 78 insertions(+), 40 deletions(-) diff --git a/.env b/.env index d190bec1..a3358471 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -AUXILIARY_QUEUE_PRIORITY=100 +AUXILIARY_QUEUE_TENANTS="sdapi" CHROME_HOSTNAME=chrome DATABASE_ADAPTER=postgresql DATABASE_HOST=db diff --git a/Gemfile b/Gemfile index ef3a9e0d..8593bc1b 100644 --- a/Gemfile +++ b/Gemfile @@ -89,7 +89,7 @@ group :development do gem 'spring-watcher-listen', '~> 2.0.0' end -gem 'bulkrax', '~> 5.0' +gem 'bulkrax', '~> 5.3.0' gem 'blacklight', '~> 6.7' gem 'blacklight_oai_provider', '~> 6.1', '>= 6.1.1' @@ -142,7 +142,7 @@ gem 'blacklight_range_limit', '6.5.0' gem 'derivative-rodeo', git: 'https://github.com/scientist-softserv/derivative_rodeo.git', branch: 'main' gem 'dog_biscuits', git: 'https://github.com/samvera-labs/dog_biscuits.git' gem 'hyrax-v2_graph_indexer', git: 'https://github.com/scientist-softserv/hyrax-v2_graph_indexer', branch: 'main' -gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'main' +gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'adventist-custom-queue' gem 'order_already' gem 'redis', '~> 4.0' gem 'redlock', '~> 1.2.1' diff --git a/Gemfile.lock b/Gemfile.lock index a76da264..97f1de06 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -31,8 +31,8 @@ GIT GIT remote: https://github.com/scientist-softserv/iiif_print.git - revision: e476998ab453afabf1bcb8afa059b4416af9b705 - branch: main + revision: a9b270c2f07af4cc11111e971b50e4aeb0dc44b6 + branch: adventist-custom-queue specs: iiif_print (1.0.0) blacklight_iiif_search (>= 1.0, < 3.0) @@ -158,7 +158,7 @@ GEM babel-transpiler (0.7.0) babel-source (>= 4.0, < 6) execjs (~> 2.0) - bagit (0.4.5) + bagit (0.4.6) docopt (~> 0.5.0) validatable (~> 1.6) bcp47 (0.3.3) @@ -216,7 +216,7 @@ GEM signet (~> 0.8) typhoeus builder (3.2.4) - bulkrax (5.3.0) + bulkrax (5.3.1) bagit (~> 0.4) coderay dry-monads (~> 1.4.0) @@ -1163,7 +1163,7 @@ DEPENDENCIES blacklight_oai_provider (~> 6.1, >= 6.1.1) blacklight_range_limit (= 6.5.0) bootstrap-datepicker-rails - bulkrax (~> 5.0) + bulkrax (~> 5.3.0) byebug capybara carrierwave-aws (~> 1.3) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 459f4f20..7e1269b3 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,18 +1,65 @@ # frozen_string_literal: true class ApplicationJob < ActiveJob::Base - before_enqueue :set_auxiliary_queue_priority + PRIORITY_QUEUE_NAME = :auxiliary + PRIORITY_QUEUE_ADDITIONAL_PRIORITY = 110 # limit to 5 attempts retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception| # Log error, do nothing, etc. end + queue_as Hyrax.config.ingest_queue_name + + queue_with_priority do + # we used to do these via GoodJobs config, but that was swallowing this queue_with_priority method. + # Note: Higher priority numbers run first in all versions of GoodJob v3.x and below. + # GoodJob v4.x will change job priority to give smaller numbers higher priority (default: 0), + # in accordance with Active Job's definition of priority. + case self + when Bulkrax::ScheduleRelationshipsJob + calculate_priority base: 50 + when CreateDerivativesJob + calculate_priority base: 40 + when CharacterizeJob + calculate_priority base: 30 + when Hyrax::GrantEditToMembersJob, ImportUrlJob, IngestJob + calculate_priority base: 10 + when AttachFilesToWorkJob + calculate_priority base: -1 + when Bulkrax::ImportWorkJob + calculate_priority base: -5 + when Bulkrax::ImportFileSetJob + calculate_priority base: -15 + when IiifPrint::Jobs::ChildWorksFromPdfJob + calculate_priority base: -17 + when Bulkrax::CreateRelationshipsJob, Bulkrax::ImporterJob, IiifPrint::Jobs::CreateRelationshipsJob + calculate_priority base: -20 + when ContentDepositEventJob, ContentUpdateEventJob + calculate_priority base: -50 + else + calculate_priority base: 0 + end + end + private - def set_auxiliary_queue_priority - return unless queue_name.to_sym == :auxiliary + def redirect_priority_jobs + return :ingest unless priority_tenants_array.include? tenant_name + PRIORITY_QUEUE_NAME + end + + def calculate_priority(base:) + return base unless priority_tenants_array.include? tenant_name + PRIORITY_QUEUE_ADDITIONAL_PRIORITY + base + end + + def priority_tenants_array + @priority_array ||= ENV.fetch('AUXILIARY_QUEUE_TENANTS', '').split(',') + end - self.priority = ENV.fetch('AUXILIARY_QUEUE_PRIORITY', 100).to_i + def tenant_name + return nil unless current_tenant + Account.find_by(tenant: current_tenant)&.name end end diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb index bc999be3..00373fbf 100644 --- a/config/initializers/good_job.rb +++ b/config/initializers/good_job.rb @@ -31,29 +31,10 @@ end # Wrapping this in an after_initialize block to ensure that all constants are loaded -Rails.application.config.after_initialize do - # baseline of 0, higher is sooner - - # Commented out the following two jobs because they were - # specfically used for the sdapi ingests. - # see sdapi_ingest_script directory and - # ref: https://github.com/scientist-softserv/adventist-dl/issues/468 - # CollectionMembershipJob.priority = 70 - # UpdateCollectionMembershipJob.priority = 60 - Bulkrax::ScheduleRelationshipsJob.priority = 50 - CreateDerivativesJob.priority = 40 - CharacterizeJob.priority = 30 - Hyrax::GrantEditToMembersJob.priority = 10 - ImportUrlJob.priority = 10 - IngestJob.priority = 10 - ApplicationJob.priority = 0 - AttachFilesToWorkJob.priority = -1 - Bulkrax::ImportWorkJob.priority = -5 - Bulkrax::ImportFileSetJob.priority = -15 - IiifPrint::Jobs::ChildWorksFromPdfJob.priority = -17 - Bulkrax::CreateRelationshipsJob.priority = -20 - Bulkrax::ImporterJob.priority = -20 - IiifPrint::Jobs::CreateRelationshipsJob.priority = -20 - ContentDepositEventJob.priority = -50 - ContentUpdateEventJob.priority = -50 -end +# Rails.application.config.after_initialize do +# # baseline of 0, higher is sooner + +# Job prioritization has been moved to application_job. Using +# the GoodJob config swallowed queue_with_priority behavior. + +# end diff --git a/config/initializers/hyrax.rb b/config/initializers/hyrax.rb index 5c630cec..85fd7191 100644 --- a/config/initializers/hyrax.rb +++ b/config/initializers/hyrax.rb @@ -1,4 +1,6 @@ Hyrax.config do |config| + config.ingest_queue_name = -> { redirect_priority_jobs } + config.register_curation_concern :generic_work # Injected via `rails g hyrax:work Image` config.register_curation_concern :image diff --git a/lib/active_job_tenant.rb b/lib/active_job_tenant.rb index ad99f3b3..fec07c64 100644 --- a/lib/active_job_tenant.rb +++ b/lib/active_job_tenant.rb @@ -10,6 +10,14 @@ module ActiveJobTenant end module ClassMethods + def queue_as(part_name = nil, &block) + if part_name.is_a?(Proc) + self.queue_name = part_name + else + super + end + end + def deserialize(job_data) super.tap do |job| job.tenant = job_data['tenant'] diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml index a27d83ea..cd745715 100644 --- a/ops/dev-deploy.tmpl.yaml +++ b/ops/dev-deploy.tmpl.yaml @@ -93,8 +93,8 @@ extraEnvVars: &envVars value: /rest - name: FEDORA_URL value: http://fcrepo.default.svc.cluster.local:8080/rest - - name: AUXILIARY_QUEUE_PRIORITY - value: "100" + - name: AUXILIARY_QUEUE_TENANTS + value: "sdapi" - name: IN_DOCKER value: "true" - name: LD_LIBRARY_PATH