Split jobs to a new queue

Based on an array of tenant names from a config var, this will send all
jobs for that tenant to the "auxiliary" queue, and increase the priority
by 110.

This causes the worker to pick up that tenant's jobs first, before
picking up lower priority jobs.

Refs #721

.env

@@ -1,4 +1,4 @@
-AUXILIARY_QUEUE_PRIORITY=100
+AUXILIARY_QUEUE_TENANTS="sdapi"
 CHROME_HOSTNAME=chrome
 DATABASE_ADAPTER=postgresql
 DATABASE_HOST=db

Gemfile

@@ -89,7 +89,7 @@ group :development do
   gem 'spring-watcher-listen', '~> 2.0.0'
 end
 
-gem 'bulkrax', '~> 5.0'
+gem 'bulkrax', '~> 5.3.0'
 
 gem 'blacklight', '~> 6.7'
 gem 'blacklight_oai_provider', '~> 6.1', '>= 6.1.1'
@@ -142,7 +142,7 @@ gem 'blacklight_range_limit', '6.5.0'
 gem 'derivative-rodeo', git: 'https://github.com/scientist-softserv/derivative_rodeo.git', branch: 'main'
 gem 'dog_biscuits', git: 'https://github.com/samvera-labs/dog_biscuits.git'
 gem 'hyrax-v2_graph_indexer', git: 'https://github.com/scientist-softserv/hyrax-v2_graph_indexer', branch: 'main'
-gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'main'
+gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'adventist-custom-queue'
 gem 'order_already'
 gem 'redis', '~> 4.0'
 gem 'redlock', '~> 1.2.1'

Gemfile.lock

@@ -31,8 +31,8 @@ GIT
 
 GIT
   remote: https://github.com/scientist-softserv/iiif_print.git
-  revision: e476998ab453afabf1bcb8afa059b4416af9b705
-  branch: main
+  revision: a9b270c2f07af4cc11111e971b50e4aeb0dc44b6
+  branch: adventist-custom-queue
   specs:
     iiif_print (1.0.0)
       blacklight_iiif_search (>= 1.0, < 3.0)
@@ -158,7 +158,7 @@ GEM
     babel-transpiler (0.7.0)
       babel-source (>= 4.0, < 6)
       execjs (~> 2.0)
-    bagit (0.4.5)
+    bagit (0.4.6)
       docopt (~> 0.5.0)
       validatable (~> 1.6)
     bcp47 (0.3.3)
@@ -216,7 +216,7 @@ GEM
       signet (~> 0.8)
       typhoeus
     builder (3.2.4)
-    bulkrax (5.3.0)
+    bulkrax (5.3.1)
       bagit (~> 0.4)
       coderay
       dry-monads (~> 1.4.0)
@@ -1163,7 +1163,7 @@ DEPENDENCIES
   blacklight_oai_provider (~> 6.1, >= 6.1.1)
   blacklight_range_limit (= 6.5.0)
   bootstrap-datepicker-rails
-  bulkrax (~> 5.0)
+  bulkrax (~> 5.3.0)
   byebug
   capybara
   carrierwave-aws (~> 1.3)

app/jobs/application_job.rb

@@ -1,18 +1,65 @@
 # frozen_string_literal: true
 
 class ApplicationJob < ActiveJob::Base
-  before_enqueue :set_auxiliary_queue_priority
+  PRIORITY_QUEUE_NAME = :auxiliary
+  PRIORITY_QUEUE_ADDITIONAL_PRIORITY = 110
 
   # limit to 5 attempts
   retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception|
     # Log error, do nothing, etc.
   end
 
+  queue_as Hyrax.config.ingest_queue_name
+
+  queue_with_priority do
+    # we used to do these via GoodJobs config, but that was swallowing this queue_with_priority method.
+    # Note: Higher priority numbers run first in all versions of GoodJob v3.x and below.
+    # GoodJob v4.x will change job priority to give smaller numbers higher priority (default: 0),
+    # in accordance with Active Job's definition of priority.
+    case self
+    when  Bulkrax::ScheduleRelationshipsJob
+      calculate_priority base: 50
+    when  CreateDerivativesJob
+      calculate_priority base: 40
+    when  CharacterizeJob
+      calculate_priority base: 30
+    when  Hyrax::GrantEditToMembersJob, ImportUrlJob, IngestJob
+      calculate_priority base: 10
+    when  AttachFilesToWorkJob
+      calculate_priority base: -1
+    when  Bulkrax::ImportWorkJob
+      calculate_priority base: -5
+    when  Bulkrax::ImportFileSetJob
+      calculate_priority base: -15
+    when  IiifPrint::Jobs::ChildWorksFromPdfJob
+      calculate_priority base: -17
+    when  Bulkrax::CreateRelationshipsJob, Bulkrax::ImporterJob, IiifPrint::Jobs::CreateRelationshipsJob
+      calculate_priority base: -20
+    when  ContentDepositEventJob, ContentUpdateEventJob
+      calculate_priority base: -50
+    else
+      calculate_priority base: 0
+    end
+  end
+
   private
 
-    def set_auxiliary_queue_priority
-      return unless queue_name.to_sym == :auxiliary
+    def redirect_priority_jobs
+      return :ingest unless priority_tenants_array.include? tenant_name
+      PRIORITY_QUEUE_NAME
+    end
+
+    def calculate_priority(base:)
+      return base unless priority_tenants_array.include? tenant_name
+      PRIORITY_QUEUE_ADDITIONAL_PRIORITY + base
+    end
+
+    def priority_tenants_array
+      @priority_array ||= ENV.fetch('AUXILIARY_QUEUE_TENANTS', '').split(',')
+    end
 
-      self.priority = ENV.fetch('AUXILIARY_QUEUE_PRIORITY', 100).to_i
+    def tenant_name
+      return nil unless current_tenant
+      Account.find_by(tenant: current_tenant)&.name
     end
 end

config/initializers/good_job.rb

@@ -31,29 +31,10 @@
 end
 
 # Wrapping this in an after_initialize block to ensure that all constants are loaded
-Rails.application.config.after_initialize do
-  # baseline of 0, higher is sooner
-
-  # Commented out the following two jobs because they were
-  # specfically used for the sdapi ingests.
-  # see sdapi_ingest_script directory and
-  # ref: https://github.com/scientist-softserv/adventist-dl/issues/468
-  # CollectionMembershipJob.priority = 70
-  # UpdateCollectionMembershipJob.priority = 60
-  Bulkrax::ScheduleRelationshipsJob.priority = 50
-  CreateDerivativesJob.priority = 40
-  CharacterizeJob.priority = 30
-  Hyrax::GrantEditToMembersJob.priority = 10
-  ImportUrlJob.priority = 10
-  IngestJob.priority = 10
-  ApplicationJob.priority = 0
-  AttachFilesToWorkJob.priority = -1
-  Bulkrax::ImportWorkJob.priority = -5
-  Bulkrax::ImportFileSetJob.priority = -15
-  IiifPrint::Jobs::ChildWorksFromPdfJob.priority = -17
-  Bulkrax::CreateRelationshipsJob.priority = -20
-  Bulkrax::ImporterJob.priority = -20
-  IiifPrint::Jobs::CreateRelationshipsJob.priority = -20
-  ContentDepositEventJob.priority = -50
-  ContentUpdateEventJob.priority = -50
-end
+# Rails.application.config.after_initialize do
+#   # baseline of 0, higher is sooner
+
+#   Job prioritization has been moved to application_job. Using
+#   the GoodJob config swallowed queue_with_priority behavior.
+
+# end

config/initializers/hyrax.rb

@@ -1,4 +1,6 @@
 Hyrax.config do |config|
+  config.ingest_queue_name = -> { redirect_priority_jobs }
+
   config.register_curation_concern :generic_work
   # Injected via `rails g hyrax:work Image`
   config.register_curation_concern :image

lib/active_job_tenant.rb

@@ -10,6 +10,14 @@ module ActiveJobTenant
   end
 
   module ClassMethods
+    def queue_as(part_name = nil, &block)
+      if part_name.is_a?(Proc)
+        self.queue_name = part_name
+      else
+        super
+      end
+    end
+
     def deserialize(job_data)
       super.tap do |job|
         job.tenant = job_data['tenant']

ops/dev-deploy.tmpl.yaml

@@ -93,8 +93,8 @@ extraEnvVars: &envVars
     value: /rest
   - name: FEDORA_URL
     value: http://fcrepo.default.svc.cluster.local:8080/rest
-  - name: AUXILIARY_QUEUE_PRIORITY
-    value: "100"
+  - name: AUXILIARY_QUEUE_TENANTS
+    value: "sdapi"
   - name: IN_DOCKER
     value: "true"
   - name: LD_LIBRARY_PATH