handle collection release

Author: terrywbrady

src-colladmin/actions/ingest_collection_locks_action.rb

@@ -37,7 +37,11 @@ def table_headers
   end
 
   def table_types
-    ['', 'colllist', '', '', 'colllock', 'dataint', 'collqitems']
+    if ZookeeperListAction.migration_m1?
+      ['', 'colllist', '', '', 'colllock', 'dataint', 'collqitems-mrtzk']
+    else
+      ['', 'colllist', '', '', 'colllock', 'dataint', 'collqitems-legacy']
+    end
   end
 
   def table_rows(_body)

src-colladmin/actions/queue_action.rb

@@ -11,81 +11,4 @@ def initialize(config, action, path, myparams, endpoint)
     endpoint = endpoint.gsub(/coll$/, coll) unless coll.empty?
     super(config, action, path, myparams, endpoint)
   end
-end
-
-# Collection Admin Task class - see config/actions.yml for description
-class CollIterateQueueAction < ZookeeperAction
-  def initialize(config, action, path, myparams)
-    super(config, action, path, myparams)
-    @coll = myparams.fetch('coll', '')
-  end
-
-  def perform_action
-    if ZookeeperListAction.migration_m1?
-      ql = QueueList.new(@zk, { deletable: true })
-      puts "PROF: #{ql.profiles.keys}"
-    else
-      MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk) do |job|
-        puts "TEST COLL #{@coll}: #{job}"
-      end
-    end
-    { message: 'queue release submitted' }.to_json
-  end
-end
-
-# Collection Admin Task class - see config/actions.yml for description
-class IterateQueueAction < ZookeeperAction
-  def initialize(config, action, path, myparams)
-    super(config, action, path, myparams)
-    @queue = myparams.fetch('queue', 'na')
-    @reload_path = myparams.fetch('reload_path', 'na')
-  end
-
-  def legacy_delete(job)
-    status = job.fetch(:status, '')
-    path = job.fetch(:path, '')
-    return unless %w[Completed Deleted].include?(status)
-    return if path.empty?
-
-    @zk.delete(path)
-  end
-
-  def perform_action
-    if @queue == 'queues-acc' && ZookeeperListAction.migration_m3?
-      MerrittZK::Access.list_jobs_as_json(@zk).each do |job|
-        qn = job.fetch(:queueNode, MerrittZK::Access::SMALL).gsub(%r{^/access/}, '')
-        j = MerrittZK::Access.new(qn, job.fetch(:id, ''))
-        j.load(@zk)
-        next unless j.status.deletable?
-
-        j.delete(@zk)
-      end
-    elsif @queue == 'queues-acc'
-      MerrittZK::LegacyAccessJob.list_jobs_as_json(@zk).each do |job|
-        legacy_delete(job)
-      end
-    elsif @queue == 'queues-inv' && ZookeeperListAction.migration_m1?
-      # no action
-    elsif @queue == 'queues-inv'
-      MerrittZK::LegacyInventoryJob.list_jobs_as_json(@zk).each do |job|
-        legacy_delete(job)
-      end
-    elsif ZookeeperListAction.migration_m1?
-      ql = QueueList.new(@zk, { deletable: true })
-      ql.batches.each_key do |bid|
-        batch = MerrittZK::Batch.find_batch_by_uuid(@zk, bid)
-        batch.load(@zk)
-        next unless batch.status.deletable?
-
-        batch.delete(@zk)
-      end
-    else
-      MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk).each do |job|
-        legacy_delete(job)
-      end
-    end
-    {
-      redirect_location: "/web/collIndex.html?path=#{@reload_path}"
-    }.to_json
-  end
-end
+end

src-colladmin/actions/zookeeper_action.rb

@@ -194,39 +194,39 @@ def prefix
 
   attr_reader :qpath
 
-  def bytes
-    data = @zk.get(qpath)
+  def bytes(p)
+    data = @zk.get(p)
     return if data.nil?
 
     data[0].bytes
   end
 
-  def orig_stat
-    return if bytes.nil?
+  def orig_stat(p)
+    return if bytes(p).nil?
 
-    bytes[0]
+    bytes(p)[0]
   end
 
-  def orig_stat_name
-    return if orig_stat.nil?
+  def orig_stat_name(p)
+    return if orig_stat(p).nil?
 
-    legacy_status_vals[orig_stat]
+    legacy_status_vals[orig_stat(p)]
   end
 
-  def write_status(status)
-    pbytes = bytes
+  def write_status(p, status)
+    pbytes = bytes(p)
     pbytes[0] = status
-    @zk.set(qpath, pbytes.pack('CCCCCCCCCc*'))
+    @zk.set(p, pbytes.pack('CCCCCCCCCc*'))
   end
 
-  def set_status(status)
+  def set_legacy_status(p, status)
     i = legacy_status_vals.find_index(status)
     return if i.nil?
 
-    orig_name = orig_stat_name
+    orig_name = orig_stat_name(p)
     return { message: 'Illegal status' }.to_json unless check_status(orig_name)
 
-    write_status(i)
+    write_status(p, i)
     { message: "Status #{orig_name} -- > #{status}" }.to_json
   end
 
@@ -239,7 +239,7 @@ def check_status(_status)
 # Legacy Ingest queue action
 class ZkRequeueLegacyAction < LegacyZkAction
   def perform_action
-    set_status('Pending')
+    set_legacy_status(qpath, 'Pending')
   end
 
   def check_status(status)
@@ -251,7 +251,7 @@ def check_status(status)
 # Legacy Ingest queue action
 class ZkDeleteLegacyAction < LegacyZkAction
   def perform_action
-    set_status('Deleted')
+    set_legacy_status(qpath, 'Deleted')
   end
 
   def check_status(status)
@@ -263,7 +263,7 @@ def check_status(status)
 # Legacy Ingest queue action
 class ZkHoldLegacyAction < LegacyZkAction
   def perform_action
-    set_status('Held')
+    set_legacy_status(qpath, 'Held')
   end
 
   def check_status(status)
@@ -275,7 +275,7 @@ def check_status(status)
 # Legacy Ingest queue action
 class ZkReleaseLegacyAction < LegacyZkAction
   def perform_action
-    set_status('Pending')
+    set_legacy_status(qpath, 'Pending')
   end
 
   def check_status(status)
@@ -290,11 +290,108 @@ def perform_action
     if ZookeeperListAction.migration_m1?
       jobs = MerrittZK::Job.list_jobs_as_json(@zk)
     else
-      MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk)
+      jobs = MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk)
     end
     jobs.each do |po|
       register_item(QueueEntry.new(po))
     end
     convert_json_to_table('')
   end
 end
+
+# Collection Admin Task class - see config/actions.yml for description
+class CollIterateQueueM1Action < ZkM1Action
+  def initialize(config, action, path, myparams)
+    super(config, action, path, myparams)
+    @coll = myparams.fetch('coll', '')
+  end
+
+  def perform_action
+    count = 0
+    ql = QueueList.new(@zk, { held: true })
+    ql.jobs.each do |j|
+      job = MerrittZK::Job.new(j.queueId)
+      job.load(@zk)
+      job.set_status(@zk, MerrittZK::JobState::Pending)
+      count += 1
+    end
+    { message: "queue release submitted for #{count}" }.to_json
+  end
+end
+
+# Collection Admin Task class - see config/actions.yml for description
+class CollIterateQueueLegacyAction < LegacyZkAction
+  def initialize(config, action, path, myparams)
+    super(config, action, path, myparams)
+    @coll = myparams.fetch('coll', '')
+  end
+
+  def perform_action
+    count = 0
+    MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk).each do |j|
+      job = QueueEntry.new(j)
+      next unless job.qstatus == 'Held'
+      next unless job.profile == @coll
+      set_legacy_status("/ingest/#{job.queueId}", 'Pending')
+      count += 1
+    end
+    { message: "queue release submitted for #{count}" }.to_json
+  end
+end
+
+# Collection Admin Task class - see config/actions.yml for description
+class IterateQueueAction < ZookeeperAction
+  def initialize(config, action, path, myparams)
+    super(config, action, path, myparams)
+    @queue = myparams.fetch('queue', 'na')
+    @reload_path = myparams.fetch('reload_path', 'na')
+  end
+
+  def legacy_delete(job)
+    status = job.fetch(:status, '')
+    path = job.fetch(:path, '')
+    return unless %w[Completed Deleted].include?(status)
+    return if path.empty?
+
+    @zk.delete(path)
+  end
+
+  def perform_action
+    if @queue == 'queues-acc' && ZookeeperListAction.migration_m3?
+      MerrittZK::Access.list_jobs_as_json(@zk).each do |job|
+        qn = job.fetch(:queueNode, MerrittZK::Access::SMALL).gsub(%r{^/access/}, '')
+        j = MerrittZK::Access.new(qn, job.fetch(:id, ''))
+        j.load(@zk)
+        next unless j.status.deletable?
+
+        j.delete(@zk)
+      end
+    elsif @queue == 'queues-acc'
+      MerrittZK::LegacyAccessJob.list_jobs_as_json(@zk).each do |job|
+        legacy_delete(job)
+      end
+    elsif @queue == 'queues-inv' && ZookeeperListAction.migration_m1?
+      # no action
+    elsif @queue == 'queues-inv'
+      MerrittZK::LegacyInventoryJob.list_jobs_as_json(@zk).each do |job|
+        legacy_delete(job)
+      end
+    elsif ZookeeperListAction.migration_m1?
+      ql = QueueList.new(@zk, { deletable: true })
+      ql.batches.each_key do |bid|
+        batch = MerrittZK::Batch.find_batch_by_uuid(@zk, bid)
+        batch.load(@zk)
+        next unless batch.status.deletable?
+
+        batch.delete(@zk)
+      end
+    else
+      MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk).each do |job|
+        legacy_delete(job)
+      end
+    end
+    {
+      redirect_location: "/web/collIndex.html?path=#{@reload_path}"
+    }.to_json
+  end
+end

src-colladmin/config/actions.yml

@@ -555,10 +555,22 @@ lock-coll:
   documentation: |
     Ingest: POST /admin/submission/freeze/{coll}
   migration: m2
-release-coll-queue-items:
+release-coll-queue-items-m1:
   link-title: Release any held items for a collection
-  class: CollIterateQueueAction
-  params: ["admin/release-all/queue/coll"]
+  class: CollIterateQueueM1Action
+  category: Queue Management
+  sensitivity: reversible change
+  testing: automatable
+  description: |
+    Release any held items for a collection.
+  report-datatypes:
+  - collqitems
+  documentation: |
+    Ingest: POST /admin/release-all/queue/{coll}
+  migration: m2
+release-coll-queue-items-legacy:
+  link-title: Release any held items for a collection (legacy)
+  class: CollIterateQueueLegacyAction
   category: Queue Management
   sensitivity: reversible change
   testing: automatable

src-colladmin/lib/queue.rb

@@ -155,6 +155,10 @@ def to_table_row
     arr
   end
 
+  def queueId
+    get_value(:queueId)
+  end
+
   def bid
     get_value(:bid)
   end
@@ -233,6 +237,15 @@ def initialize(zk, filter = {})
     end
     jobs.each do |j|
       job = QueueEntry.new(j)
+      if @filter.key?(:batch)
+        next unless job.bid == @filter[:batch]
+      end
+      if @filter.fetch(:deletable, false)
+        next unless %w[Completed Deleted].include?(job.qstatus)
+      end
+      if @filter.fetch(:held, false)
+        next unless %w[Held].include?(job.qstatus)
+      end
       @jobs << job
       qb = @batches.fetch(job.bid, QueueBatch.new(job.bid, job.user))
       qb.add_job(job)

src-common/template/api-table.js

@@ -818,8 +818,11 @@ function format(cell, v, type, merritt_path) {
   } else if (type == 'release-legacy'  && v != '') {
     p = colladmin_root + "/lambda?path=release-queue-item-legacy&queue-path="+v;
     makeLink(cell, 'Release', "javascript:ajax_invoke('"+encodeURIComponent(p)+"')").addClass("ajax");
-  } else if (type == 'collqitems'  && v != '') {
-    p = colladmin_root + "/lambda?path=release-coll-queue-items&coll="+v;
+  } else if (type == 'collqitems-mrtzk'  && v != '') {
+    p = colladmin_root + "/lambda?path=release-coll-queue-items-m1&coll="+v;
+    makeLink(cell, 'Release Items', "javascript:ajax_invoke('"+encodeURIComponent(p)+"')").addClass("ajax");
+  } else if (type == 'collqitems-legacy'  && v != '') {
+    p = colladmin_root + "/lambda?path=release-coll-queue-items-legacy&coll="+v;
     makeLink(cell, 'Release Items', "javascript:ajax_invoke('"+encodeURIComponent(p)+"')").addClass("ajax");
   } else if (type == 'colllock') {
     var arr = v.split(",");