support inv queue

Author: terrywbrady

cognito-lambda-nonvpc/Gemfile.lock

@@ -3,11 +3,11 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.924.0)
+    aws-partitions (1.925.0)
     aws-sdk-cognitoidentityprovider (1.90.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.194.1)
+    aws-sdk-core (3.194.2)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)

src-colladmin/Gemfile

@@ -12,7 +12,7 @@ gem 'aws-sdk-lambda'
 gem 'aws-sdk-s3'
 gem 'aws-sdk-ssm'
 gem 'csv'
-gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.d'
+gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.g'
 gem 'rest-client'
 gem 'rubocop'
 gem 'uc3-ssm', '0.3.10', git: 'https://github.com/CDLUC3/uc3-ssm.git', branch: 'main'

src-colladmin/Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/CDLUC3/mrt-zk.git
-  revision: 4297c486b446942d4ae748908f7a838b68e44428
-  tag: 1.0.3.d
+  revision: 9dd3b3e45de97160d2c6331c4152a5e1ab03f83f
+  tag: 1.0.3.g
   specs:
     mrt-zk (1.0.1)
       zk (~> 1.10.0)

src-colladmin/actions/zookeeper_action.rb

@@ -86,19 +86,19 @@ class ZookeeperAction < AdminAction
   def initialize(config, action, path, myparams)
     super(config, action, path, myparams)
     @zk = ZK.new(get_zookeeper_conn)
-    @qid = myparams.fetch('queue-path', '')
+    @qpath = myparams.fetch('queue-path', '')
   end
 
   def get_zookeeper_conn
     @config.fetch('zookeeper', '')
   end
 
   def perform_action
-    { message: "No response for #{@path}: #{@qid}" }.to_json
+    { message: "No response for #{@path}: #{@qpath}" }.to_json
   rescue StandardError => e
     log(e.message)
     log(e.backtrace)
-    { error: "#{e.message} for #{@path}: #{@qid}" }.to_json
+    { error: "#{e.message} for #{@path}: #{@qpath}" }.to_json
   end
 end
 
@@ -121,19 +121,19 @@ class ZkReleaseM1Action < ZookeeperAction
 ## Legacy Queue manipulation action using new mrt-zk
 class LegacyZkAction < ZookeeperAction
   def status_vals
-    MerrittZK::LegacyIngestJob.status_vals
+    MerrittZK::LegacyItem::STATUS_VALS
   end
 
   def prefix
     'na'
   end
 
-  def path
-    "/#{prefix}/#{@qid}"
+  def qpath
+    @qpath
   end
 
   def bytes
-    data = @zk.get(path)
+    data = @zk.get(qpath)
     return if data.nil?
 
     data[0].bytes
@@ -154,7 +154,7 @@ def orig_stat_name
   def write_status(status)
     pbytes = bytes
     pbytes[0] = status
-    @zk.set(path, pbytes.pack('CCCCCCCCCc*'))
+    @zk.set(qpath, pbytes.pack('CCCCCCCCCc*'))
   end
 
   def set_status(status)
@@ -175,15 +175,7 @@ def check_status(_status)
 
 ##
 # Legacy Ingest queue action
-class LegacyIngestZkAction < LegacyZkAction
-  def prefix
-    'ingest'
-  end
-end
-
-##
-# Legacy Ingest queue action
-class ZkRequeueLegacyIngestAction < LegacyIngestZkAction
+class ZkRequeueLegacyAction < LegacyZkAction
   def perform_action
     set_status('Pending')
   end
@@ -195,7 +187,7 @@ def check_status(status)
 
 ##
 # Legacy Ingest queue action
-class ZkDeleteLegacyIngestAction < LegacyIngestZkAction
+class ZkDeleteLegacyAction < LegacyZkAction
   def perform_action
     set_status('Deleted')
   end
@@ -207,7 +199,7 @@ def check_status(status)
 
 ##
 # Legacy Ingest queue action
-class ZkHoldLegacyIngestAction < LegacyIngestZkAction
+class ZkHoldLegacyAction < LegacyZkAction
   def perform_action
     set_status('Held')
   end
@@ -219,7 +211,7 @@ def check_status(status)
 
 ##
 # Legacy Ingest queue action
-class ZkReleaseLegacyIngestAction < LegacyIngestZkAction
+class ZkReleaseLegacyAction < LegacyZkAction
   def perform_action
     set_status('Pending')
   end
@@ -236,7 +228,7 @@ def zk_path
   end
 
   def status_vals
-    MerrittZK::LegacyIngestJob.status_vals
+    MerrittZK::LegacyItem::STATUS_VALS
   end
 
   def is_json
@@ -260,7 +252,7 @@ def zk_path
   end
 
   def status_vals
-    MerrittZK::LegacyInventoryJob.status_vals
+    MerrittZK::LegacyItem::STATUS_VALS
   end
 
   def is_json

src-colladmin/config/actions.yml

@@ -933,7 +933,7 @@ queue-delete-mrtzk:
   migration: m1
 queue-delete-legacy:
   link-title: Remove an item from a Zookeeper queue
-  class: ZkDeleteLegacyIngestAction
+  class: ZkDeleteLegacyAction
   category: Queue Management
   sensitivity: irreversible change
   testing: manual
@@ -973,7 +973,7 @@ requeue-mrtzk:
   migration: m1
 requeue-legacy:
   link-title: Re-queue an item from a Zookeeper queue
-  class: ZkRequeueLegacyIngestAction
+  class: ZkRequeueLegacyAction
   category: Queue Management
   sensitivity: irreversible change
   testing: manual
@@ -1013,7 +1013,7 @@ hold-queue-item-mrtzk:
   migration: m1
 hold-queue-item-legacy:
   link-title: Move a pending item from in Zookeeper queue to a Held status
-  class: ZkHoldLegacyIngestAction
+  class: ZkHoldLegacyAction
   category: Queue Management
   sensitivity: irreversible change
   testing: manual
@@ -1053,7 +1053,7 @@ release-queue-item-mrtzk:
   migration: m1
 release-queue-item-legacy:
   link-title: Release a held item from in Zookeeper queue to a Pending status
-  class: ZkReleaseLegacyIngestAction
+  class: ZkReleaseLegacyAction
   category: Queue Management
   sensitivity: irreversible change
   testing: manual

src-colladmin/lib/inv_queue.rb

@@ -14,23 +14,23 @@ def initialize(json)
     super()
     add_property(
       :queueNode,
-      MerrittJsonProperty.new('Queue Node').lookup_value(json, '', 'queueNode')
+      MerrittJsonProperty.new('Queue Node').lookup_value(json, '', :queueNode)
     )
     add_property(
       :manifestURL,
-      MerrittJsonProperty.new('Manifest URL').lookup_value(json, '', 'manifestURL')
+      MerrittJsonProperty.new('Manifest URL').lookup_value(json, '', :manifestURL)
     )
     add_property(
       :date,
-      MerrittJsonProperty.new('Date').lookup_time_value(json, '', 'date')
+      MerrittJsonProperty.new('Date').lookup_time_value(json, '', :date)
     )
     add_property(
       :qstatus,
-      MerrittJsonProperty.new('QStatus').lookup_value(json, '', 'status')
+      MerrittJsonProperty.new('QStatus').lookup_value(json, '', :status)
     )
     add_property(
       :queueId,
-      MerrittJsonProperty.new('Queue ID').lookup_value(json, '', 'iD')
+      MerrittJsonProperty.new('Queue ID').lookup_value(json, '', :id)
     )
     qs = get_value(:qstatus, '')
     qt = get_value(:date, '')
@@ -60,11 +60,11 @@ def initialize(json)
 
     add_property(
       :qdelete,
-      MerrittJsonProperty.new('Queue Del', get_queue_path(requeue: false))
+      MerrittJsonProperty.new('Queue Del', get_del_queue_path_m1)
     )
     add_property(
       :requeue,
-      MerrittJsonProperty.new('Requeue', get_queue_path(requeue: true))
+      MerrittJsonProperty.new('Requeue', get_requeue_path_m1)
     )
   end
 
@@ -80,8 +80,13 @@ def self.table_types
       type = ''
       type = 'status' if sym == :status
       type = 'datetime' if sym == :date
-      type = 'qdelete' if sym == :qdelete
-      type = 'requeue' if sym == :requeue
+      if $migration == :m1
+        type = 'qdelete-mrtzk' if sym == :qdelete
+        type = 'requeue-mrtzk' if sym == :requeue
+      else
+        type = 'qdelete-legacy' if sym == :qdelete
+        type = 'requeue-legacy' if sym == :requeue
+      end
       arr.append(type)
     end
     arr
@@ -104,69 +109,3 @@ def date
     get_value(:date)
   end
 end
-
-# inventory queue
-class InventoryQueue < MerrittJson
-  def initialize(queue_list, body)
-    data = JSON.parse(body)
-    data = fetch_hash_val(data, 'que:queueState')
-    data = fetch_hash_val(data, 'que:queueEntries')
-    list = fetch_array_val(data, 'que:queueEntryState')
-    list.each do |obj|
-      q = InvQueueEntry.new(obj)
-      queue_list.manifests.append(q)
-    end
-    super()
-  end
-end
-
-# list of all inventory queues - only one exists
-class InvQueueList < MerrittJson
-  def initialize(ingest_server, body, _filter = {})
-    super()
-    @ingest_server = ingest_server
-    @body = body
-    @manifests = []
-    retrieve_queues
-  end
-
-  def self.get_queue_list(ingest_server, filter = {})
-    qjson = HttpGetJson.new(ingest_server, 'admin/queues-inv')
-    InvQueueList.new(ingest_server, qjson.body, filter)
-  end
-
-  def retrieve_queues
-    data = JSON.parse(@body)
-    data = fetch_hash_val(data, 'ingq:ingestQueueNameState')
-    data = fetch_hash_val(data, 'ingq:ingestQueueName')
-    fetch_array_val(data, 'ingq:ingestQueue').each do |qjson|
-      node = fetch_hash_val(qjson, 'ingq:node')
-      begin
-        qjson = HttpGetJson.new(@ingest_server, "admin/queue-inv#{node}")
-        next unless qjson.status == 200
-
-        InventoryQueue.new(self, qjson.body)
-      rescue StandardError => e
-        LambdaBase.log(e.message)
-        LambdaBase.log(e.backtrace)
-      end
-    end
-  end
-
-  attr_reader :manifests, :body
-
-  def to_table
-    table = []
-    ms = @manifests.sort do |a, b|
-      if a.status == b.status
-        b.date <=> a.date
-      else
-        AdminTask.status_sort_val(a.status) <=> AdminTask.status_sort_val(b.status)
-      end
-    end
-    ms.each_with_index do |q, _i|
-      table.append(q.to_table_row)
-    end
-    table
-  end
-end

src-colladmin/lib/queue_json.rb

@@ -8,6 +8,10 @@ def get_queue_node
     get_value(:queueNode, '')
   end
 
+  def path
+    "#{get_queue_node}/#{get_value(:queueId, '')}"
+  end
+
   def get_queue_path(requeue: false)
     st = get_value(:qstatus, '')
     case st
@@ -33,14 +37,14 @@ def get_del_queue_path_m1
     st = get_value(:qstatus, '')
     return '' unless %w[Failed Completed Held].include?(st)
 
-    get_value(:queueId, '')
+    path
   end
 
   def get_requeue_path_m1
     st = get_value(:qstatus, '')
     return '' unless %w[Consumed Failed].include?(st)
 
-    get_value(:queueId, '')
+    path
   end
 
   def get_hold_path(release: false)
@@ -52,20 +56,20 @@ def get_hold_path(release: false)
     else
       return ''
     end
-    "#{get_queue_node}/#{get_value(:queueId, '')}"
+    path
   end
 
   def get_hold_path_m1
     st = get_value(:qstatus, '')
     return '' unless %w[Pending].include?(st)
 
-    get_value(:queueId, '')
+    path
   end
 
   def get_release_path_m1
     st = get_value(:qstatus, '')
     return '' unless %w[Held].include?(st)
 
-    get_value(:queueId, '')
+    path
   end
 end

src-common/template/api-table.css

@@ -297,19 +297,7 @@ td.requeue a,
 td.hold a, 
 td.release a, 
 td.colllock a, 
-td.collqitems a,
-td.qdelete-mrtzk a, 
-td.requeue-mrtzk a, 
-td.hold-mrtzk a, 
-td.release-mrtzk a, 
-td.colllock-mrtzk a, 
-td.collqitems-mrtzk a,
-td.qdelete-legacy a, 
-td.requeue-legacy a, 
-td.hold-legacy a, 
-td.release-legacy a, 
-td.colllock-legacy a, 
-td.collqitems-legacy a {
+td.collqitems a {
   display: block;
   text-align: center;
   border-radius: 5px;

src-testdriver/Gemfile.lock

@@ -13,8 +13,8 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.924.0)
-    aws-sdk-core (3.194.1)
+    aws-partitions (1.925.0)
+    aws-sdk-core (3.194.2)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)