requeue support

mysql-ruby-lambda/Gemfile.lock

@@ -13,7 +13,7 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.925.0)
+    aws-partitions (1.926.0)
     aws-sdk-core (3.194.2)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
@@ -37,7 +37,7 @@ GEM
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.63.4)
+    rubocop (1.63.5)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)

src-admintool/Gemfile.lock

@@ -13,7 +13,7 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.925.0)
+    aws-partitions (1.926.0)
     aws-sdk-core (3.194.2)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
@@ -44,7 +44,7 @@ GEM
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.63.4)
+    rubocop (1.63.5)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)

src-colladmin/Gemfile

@@ -12,7 +12,7 @@ gem 'aws-sdk-lambda'
 gem 'aws-sdk-s3'
 gem 'aws-sdk-ssm'
 gem 'csv'
-gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.k'
+gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.m'
 gem 'rest-client'
 gem 'rubocop'
 gem 'uc3-ssm', '0.3.10', git: 'https://github.com/CDLUC3/uc3-ssm.git', branch: 'main'

src-colladmin/Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/CDLUC3/mrt-zk.git
-  revision: 562b8017b32c10473d5dd3986eb4c2243bc064e4
-  tag: 1.0.3.k
+  revision: 54820652b218e2108759db0f937abc31aa829e1c
+  tag: 1.0.3.m
   specs:
     mrt-zk (1.0.1)
       zk (~> 1.10.0)
@@ -22,13 +22,13 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.925.0)
+    aws-partitions (1.926.0)
     aws-sdk-core (3.194.2)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-ec2 (1.455.0)
+    aws-sdk-ec2 (1.456.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-kms (1.80.0)
@@ -73,7 +73,7 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.6)
-    rubocop (1.63.4)
+    rubocop (1.63.5)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)

src-colladmin/actions/access_queue_action.rb

@@ -8,9 +8,9 @@ class AccessQueueAction < ZookeeperListAction
   def perform_action
     jobs = []
     if ZookeeperListAction.migration_m3?
-      jobs = MerrittZK::Access.list_jobs(@zk)
+      jobs = MerrittZK::Access.list_jobs_as_json(@zk)
     else
-      jobs = MerrittZK::LegacyAccessJob.list_jobs(@zk)
+      jobs = MerrittZK::LegacyAccessJob.list_jobs_as_json(@zk)
     end
     jobs.each do |po|
       register_item(AccQueueEntry.new(po))

src-colladmin/actions/inventory_queue_action.rb

@@ -6,7 +6,7 @@
 # Collection Admin Task class - see config/actions.yml for description
 class InventoryQueueAction < ZookeeperListAction
   def perform_action
-    jobs = ZookeeperListAction.migration_m1? ? [] : MerrittZK::LegacyInventoryJob.list_jobs(@zk)
+    jobs = ZookeeperListAction.migration_m1? ? [] : MerrittZK::LegacyInventoryJob.list_jobs_as_json(@zk)
     jobs.each do |po|
       register_item(InvQueueEntry.new(po))
     end

src-colladmin/actions/queue_action.rb

@@ -25,7 +25,7 @@ def perform_action
       ql = QueueList.new(@zk, { deletable: true })
       puts "PROF: #{ql.profiles.keys}"
     else
-      MerrittZK::LegacyIngestJob.list_jobs(@zk) do |job|
+      MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk) do |job|
         puts "TEST COLL #{@coll}: #{job}"
       end
     end
@@ -52,7 +52,7 @@ def legacy_delete(job)
 
   def perform_action
     if @queue == 'queues-acc' && ZookeeperListAction.migration_m3?
-      MerrittZK::Access.list_jobs(@zk).each do |job|
+      MerrittZK::Access.list_jobs_as_json(@zk).each do |job|
         qn = job.fetch(:queueNode, MerrittZK::Access::SMALL).gsub(%r{^/access/}, '')
         j = MerrittZK::Access.new(qn, job.fetch(:id, ''))
         j.load(@zk)
@@ -61,13 +61,13 @@ def perform_action
         j.delete(@zk)
       end
     elsif @queue == 'queues-acc'
-      MerrittZK::LegacyAccessJob.list_jobs(@zk).each do |job|
+      MerrittZK::LegacyAccessJob.list_jobs_as_json(@zk).each do |job|
         legacy_delete(job)
       end
     elsif @queue == 'queues-inv' && ZookeeperListAction.migration_m1?
       # no action
     elsif @queue == 'queues-inv'
-      MerrittZK::LegacyInventoryJob.list_jobs(@zk).each do |job|
+      MerrittZK::LegacyInventoryJob.list_jobs_as_json(@zk).each do |job|
         legacy_delete(job)
       end
     elsif ZookeeperListAction.migration_m1?
@@ -80,7 +80,7 @@ def perform_action
         batch.delete(@zk)
       end
     else
-      MerrittZK::LegacyIngestJob.list_jobs(@zk).each do |job|
+      MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk).each do |job|
         legacy_delete(job)
       end
     end

src-colladmin/actions/zookeeper_action.rb

@@ -100,20 +100,86 @@ def perform_action
   end
 end
 
+## Base class for new style action
+class ZkM1Action < ZookeeperAction
+  def get_id
+    @qpath.split('/')[-1]
+  end
+
+  def get_access_queue
+    @qpath.split('/')[-2]
+  end
+
+  def perform_action
+    { message: "path: #{@path}; qpath: #{@qpath}" }.to_json
+  end
+end
+
 ## Queue manipulation action using new mrt-zk
-class ZkRequeueM1Action < ZookeeperAction
+class ZkRequeueM1Action < ZkM1Action
+  def perform_action
+    if @qpath =~ /access/
+      acc = MerrittZK::Access.new(get_access_queue, get_id)
+      acc.load(@zk)
+      acc.set_status(@zk, MerrittZK::AccessState::Pending)
+      { message: "Acc #{acc.id} requeue not yet implemented " }.to_json
+    else
+      job = MerrittZK::Job.new(get_id)
+      job.load(@zk)
+      js = job.json_property(@zk, MerrittZK::ZkKeys::STATUS)
+      laststat = js.fetch(:last_successful_status, '')
+      case laststat
+      when 'Estimating', '', nil
+        job.set_status(@zk, MerrittZK::JobState::Estimating)
+      when 'Downloading'
+        job.set_status(@zk, MerrittZK::JobState::Downloading)
+      when 'Processing'
+        job.set_status(@zk, MerrittZK::JobState::Processing)
+      when 'Recording'
+        job.set_status(@zk, MerrittZK::JobState::Recording)
+      when 'Notify'
+        job.set_status(@zk, MerrittZK::JobState::Notify)
+      end
+      { message: "Job #{job.id} requeued to status #{job.status_name}" }.to_json
+    end
+  end
 end
 
 ## Queue manipulation action using new mrt-zk
-class ZkDeleteM1Action < ZookeeperAction
+class ZkDeleteM1Action < ZkM1Action
+  def perform_action
+    if @qpath =~ /access/
+      acc = MerrittZK::Access.new(get_access_queue, get_id)
+      acc.load(@zk)
+      acc.set_status(@zk, MerrittZK::AccessState::Deleted)
+      { message: "Acc #{@acc.id} deleted" }.to_json
+    else
+      job = MerrittZK::Job.new(get_id)
+      job.load(@zk)
+      job.set_status(@zk, MerrittZK::JobState::Deleted)
+      { message: "Job #{@job.id} deleted" }.to_json
+    end
+  end
 end
 
 ## Queue manipulation action using new mrt-zk
-class ZkHoldM1Action < ZookeeperAction
+class ZkHoldM1Action < ZkM1Action
+  def perform_action
+    job = MerrittZK::Job.new(get_id)
+    job.load(@zk)
+    job.set_status(@zk, MerrittZK::JobState::Held)
+    { message: "Job #{@qpath} held" }.to_json
+  end
 end
 
 ## Queue manipulation action using new mrt-zk
-class ZkReleaseM1Action < ZookeeperAction
+class ZkReleaseM1Action < ZkM1Action
+  def perform_action
+    job = MerrittZK::Job.new(get_id)
+    job.load(@zk)
+    job.set_status(@zk, MerrittZK::JobState::Pending)
+    { message: "Job #{@qpath} released" }.to_json
+  end
 end
 
 ## Legacy Queue manipulation action using new mrt-zk
@@ -220,7 +286,12 @@ def check_status(status)
 ## Class for reading the legacy Merritt Ingest Queue
 class IngestQueueZookeeperAction < ZookeeperListAction
   def perform_action
-    jobs = ZookeeperListAction.migration_m1? ? MerrittZK::Job.list_jobs(@zk) : MerrittZK::LegacyIngestJob.list_jobs(@zk)
+    jobs = []
+    if ZookeeperListAction.migration_m1?
+      jobs = MerrittZK::Job.list_jobs_as_json(@zk)
+    else
+      MerrittZK::LegacyIngestJob.list_jobs_as_json(@zk)
+    end
     jobs.each do |po|
       register_item(QueueEntry.new(po))
     end

src-colladmin/lib/queue.rb

@@ -225,7 +225,12 @@ def initialize(zk, filter = {})
     @profiles = {}
     @filter = filter
 
-    jobs = ZookeeperListAction.migration_m1? ? MerrittZK::Job.list_jobs(zk) : MerrittZK::LegacyIngestJob.list_jobs(zk)
+    jobs = []
+    if ZookeeperListAction.migration_m1?
+      jobs = MerrittZK::Job.list_jobs_as_json(zk)
+    else
+      jobs = MerrittZK::LegacyIngestJob.list_jobs_as_json(zk)
+    end
     jobs.each do |j|
       job = QueueEntry.new(j)
       @jobs << job

src-common/Gemfile.lock

@@ -13,7 +13,7 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.925.0)
+    aws-partitions (1.926.0)
     aws-sdk-core (3.194.2)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
@@ -43,7 +43,7 @@ GEM
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.63.4)
+    rubocop (1.63.5)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)