diff --git a/.env b/.env index d190bec1..a3358471 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -AUXILIARY_QUEUE_PRIORITY=100 +AUXILIARY_QUEUE_TENANTS="sdapi" CHROME_HOSTNAME=chrome DATABASE_ADAPTER=postgresql DATABASE_HOST=db diff --git a/Gemfile b/Gemfile index ef3a9e0d..8593bc1b 100644 --- a/Gemfile +++ b/Gemfile @@ -89,7 +89,7 @@ group :development do gem 'spring-watcher-listen', '~> 2.0.0' end -gem 'bulkrax', '~> 5.0' +gem 'bulkrax', '~> 5.3.0' gem 'blacklight', '~> 6.7' gem 'blacklight_oai_provider', '~> 6.1', '>= 6.1.1' @@ -142,7 +142,7 @@ gem 'blacklight_range_limit', '6.5.0' gem 'derivative-rodeo', git: 'https://github.com/scientist-softserv/derivative_rodeo.git', branch: 'main' gem 'dog_biscuits', git: 'https://github.com/samvera-labs/dog_biscuits.git' gem 'hyrax-v2_graph_indexer', git: 'https://github.com/scientist-softserv/hyrax-v2_graph_indexer', branch: 'main' -gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'main' +gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'adventist-custom-queue' gem 'order_already' gem 'redis', '~> 4.0' gem 'redlock', '~> 1.2.1' diff --git a/Gemfile.lock b/Gemfile.lock index a76da264..97f1de06 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -31,8 +31,8 @@ GIT GIT remote: https://github.com/scientist-softserv/iiif_print.git - revision: e476998ab453afabf1bcb8afa059b4416af9b705 - branch: main + revision: a9b270c2f07af4cc11111e971b50e4aeb0dc44b6 + branch: adventist-custom-queue specs: iiif_print (1.0.0) blacklight_iiif_search (>= 1.0, < 3.0) @@ -158,7 +158,7 @@ GEM babel-transpiler (0.7.0) babel-source (>= 4.0, < 6) execjs (~> 2.0) - bagit (0.4.5) + bagit (0.4.6) docopt (~> 0.5.0) validatable (~> 1.6) bcp47 (0.3.3) @@ -216,7 +216,7 @@ GEM signet (~> 0.8) typhoeus builder (3.2.4) - bulkrax (5.3.0) + bulkrax (5.3.1) bagit (~> 0.4) coderay dry-monads (~> 1.4.0) @@ -1163,7 +1163,7 @@ DEPENDENCIES blacklight_oai_provider (~> 6.1, >= 6.1.1) blacklight_range_limit (= 6.5.0) bootstrap-datepicker-rails - bulkrax (~> 5.0) + bulkrax (~> 5.3.0) byebug capybara carrierwave-aws (~> 1.3) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 459f4f20..7e1269b3 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,18 +1,65 @@ # frozen_string_literal: true class ApplicationJob < ActiveJob::Base - before_enqueue :set_auxiliary_queue_priority + PRIORITY_QUEUE_NAME = :auxiliary + PRIORITY_QUEUE_ADDITIONAL_PRIORITY = 110 # limit to 5 attempts retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception| # Log error, do nothing, etc. end + queue_as Hyrax.config.ingest_queue_name + + queue_with_priority do + # we used to do these via GoodJobs config, but that was swallowing this queue_with_priority method. + # Note: Higher priority numbers run first in all versions of GoodJob v3.x and below. + # GoodJob v4.x will change job priority to give smaller numbers higher priority (default: 0), + # in accordance with Active Job's definition of priority. + case self + when Bulkrax::ScheduleRelationshipsJob + calculate_priority base: 50 + when CreateDerivativesJob + calculate_priority base: 40 + when CharacterizeJob + calculate_priority base: 30 + when Hyrax::GrantEditToMembersJob, ImportUrlJob, IngestJob + calculate_priority base: 10 + when AttachFilesToWorkJob + calculate_priority base: -1 + when Bulkrax::ImportWorkJob + calculate_priority base: -5 + when Bulkrax::ImportFileSetJob + calculate_priority base: -15 + when IiifPrint::Jobs::ChildWorksFromPdfJob + calculate_priority base: -17 + when Bulkrax::CreateRelationshipsJob, Bulkrax::ImporterJob, IiifPrint::Jobs::CreateRelationshipsJob + calculate_priority base: -20 + when ContentDepositEventJob, ContentUpdateEventJob + calculate_priority base: -50 + else + calculate_priority base: 0 + end + end + private - def set_auxiliary_queue_priority - return unless queue_name.to_sym == :auxiliary + def redirect_priority_jobs + return :ingest unless priority_tenants_array.include? tenant_name + PRIORITY_QUEUE_NAME + end + + def calculate_priority(base:) + return base unless priority_tenants_array.include? tenant_name + PRIORITY_QUEUE_ADDITIONAL_PRIORITY + base + end + + def priority_tenants_array + @priority_array ||= ENV.fetch('AUXILIARY_QUEUE_TENANTS', '').split(',') + end - self.priority = ENV.fetch('AUXILIARY_QUEUE_PRIORITY', 100).to_i + def tenant_name + return nil unless current_tenant + Account.find_by(tenant: current_tenant)&.name end end diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb index bc999be3..00373fbf 100644 --- a/config/initializers/good_job.rb +++ b/config/initializers/good_job.rb @@ -31,29 +31,10 @@ end # Wrapping this in an after_initialize block to ensure that all constants are loaded -Rails.application.config.after_initialize do - # baseline of 0, higher is sooner - - # Commented out the following two jobs because they were - # specfically used for the sdapi ingests. - # see sdapi_ingest_script directory and - # ref: https://github.com/scientist-softserv/adventist-dl/issues/468 - # CollectionMembershipJob.priority = 70 - # UpdateCollectionMembershipJob.priority = 60 - Bulkrax::ScheduleRelationshipsJob.priority = 50 - CreateDerivativesJob.priority = 40 - CharacterizeJob.priority = 30 - Hyrax::GrantEditToMembersJob.priority = 10 - ImportUrlJob.priority = 10 - IngestJob.priority = 10 - ApplicationJob.priority = 0 - AttachFilesToWorkJob.priority = -1 - Bulkrax::ImportWorkJob.priority = -5 - Bulkrax::ImportFileSetJob.priority = -15 - IiifPrint::Jobs::ChildWorksFromPdfJob.priority = -17 - Bulkrax::CreateRelationshipsJob.priority = -20 - Bulkrax::ImporterJob.priority = -20 - IiifPrint::Jobs::CreateRelationshipsJob.priority = -20 - ContentDepositEventJob.priority = -50 - ContentUpdateEventJob.priority = -50 -end +# Rails.application.config.after_initialize do +# # baseline of 0, higher is sooner + +# Job prioritization has been moved to application_job. Using +# the GoodJob config swallowed queue_with_priority behavior. + +# end diff --git a/config/initializers/hyrax.rb b/config/initializers/hyrax.rb index 5c630cec..85fd7191 100644 --- a/config/initializers/hyrax.rb +++ b/config/initializers/hyrax.rb @@ -1,4 +1,6 @@ Hyrax.config do |config| + config.ingest_queue_name = -> { redirect_priority_jobs } + config.register_curation_concern :generic_work # Injected via `rails g hyrax:work Image` config.register_curation_concern :image diff --git a/lib/active_job_tenant.rb b/lib/active_job_tenant.rb index ad99f3b3..fec07c64 100644 --- a/lib/active_job_tenant.rb +++ b/lib/active_job_tenant.rb @@ -10,6 +10,14 @@ module ActiveJobTenant end module ClassMethods + def queue_as(part_name = nil, &block) + if part_name.is_a?(Proc) + self.queue_name = part_name + else + super + end + end + def deserialize(job_data) super.tap do |job| job.tenant = job_data['tenant'] diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml index a27d83ea..cd745715 100644 --- a/ops/dev-deploy.tmpl.yaml +++ b/ops/dev-deploy.tmpl.yaml @@ -93,8 +93,8 @@ extraEnvVars: &envVars value: /rest - name: FEDORA_URL value: http://fcrepo.default.svc.cluster.local:8080/rest - - name: AUXILIARY_QUEUE_PRIORITY - value: "100" + - name: AUXILIARY_QUEUE_TENANTS + value: "sdapi" - name: IN_DOCKER value: "true" - name: LD_LIBRARY_PATH