zk action migration versions

Author: terrywbrady

cognito-lambda-nonvpc/Gemfile.lock

@@ -3,11 +3,11 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.923.0)
+    aws-partitions (1.924.0)
     aws-sdk-cognitoidentityprovider (1.90.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.194.0)
+    aws-sdk-core (3.194.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)

mysql-ruby-lambda/Gemfile.lock

@@ -13,8 +13,8 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.923.0)
-    aws-sdk-core (3.194.0)
+    aws-partitions (1.924.0)
+    aws-sdk-core (3.194.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)

src-admintool/Gemfile.lock

@@ -13,8 +13,8 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.923.0)
-    aws-sdk-core (3.194.0)
+    aws-partitions (1.924.0)
+    aws-sdk-core (3.194.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)

src-colladmin/Gemfile

@@ -12,7 +12,7 @@ gem 'aws-sdk-lambda'
 gem 'aws-sdk-s3'
 gem 'aws-sdk-ssm'
 gem 'csv'
-gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.2.a'
+gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.c'
 gem 'rest-client'
 gem 'rubocop'
 gem 'uc3-ssm', '0.3.10', git: 'https://github.com/CDLUC3/uc3-ssm.git', branch: 'main'

src-colladmin/Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/CDLUC3/mrt-zk.git
-  revision: 0666a075c9026c8f81868a1d81c8c4572296c9c1
-  tag: 1.0.2.a
+  revision: 0e70ab21c3588d8519366039905e5255b8e4ec58
+  tag: 1.0.3.c
   specs:
     mrt-zk (1.0.1)
       zk (~> 1.10.0)
@@ -22,13 +22,13 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.923.0)
-    aws-sdk-core (3.194.0)
+    aws-partitions (1.924.0)
+    aws-sdk-core (3.194.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-ec2 (1.454.0)
+    aws-sdk-ec2 (1.455.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-kms (1.80.0)

src-colladmin/actions/replication_action.rb

@@ -224,7 +224,6 @@ def perform_action
         [coll, nodenum]
       ).each do |r|
         endpoint = "delete/#{nodenum}/#{CGI.escape(r[0])}"
-        puts endpoint
         begin
           qjson = HttpDeleteJson.new(get_replic_server, endpoint)
           puts qjson.status

src-colladmin/actions/zookeeper_action.rb

@@ -32,7 +32,7 @@ def to_table
 end
 
 ## Base class for actions that interact directly with Zookeeper using the mrt-zk library
-class ZookeeperAction < AdminAction
+class ZookeeperListAction < AdminAction
   def initialize(config, action, path, myparams, _filters)
     super(config, action, path, myparams)
     @filters = {}
@@ -69,6 +69,7 @@ def register_item(item)
   end
 
   def perform_action
+    $migration = migration_level if migration_m1?
     jobs = migration_m1? ? MerrittZK::Job.list_jobs(@zk) : MerrittZK::LegacyIngestJob.list_jobs(@zk)
     jobs.each do |po|
       register_item(QueueEntry.new(po))
@@ -81,13 +82,87 @@ def table_rows(_body)
   end
 
   def get_zookeeper_conn
-    # @config.fetch('zookeeper', '').split(',').first
     @config.fetch('zookeeper', '')
   end
 end
 
+class ZookeeperAction < AdminAction
+  def initialize(config, action, path, myparams)
+    super(config, action, path, myparams)
+    @zk = ZK.new(get_zookeeper_conn)
+    @qid = myparams.fetch('queue-path', '')
+  end
+
+  def get_zookeeper_conn
+    @config.fetch('zookeeper', '')
+  end
+
+  def perform_action
+    { message: "No response for #{@path}: #{@qid}" }.to_json
+  rescue StandardError => e
+    log(e.message)
+    log(e.backtrace)
+    { error: "#{e.message} for #{@path}: #{@qid}" }.to_json
+  end
+end
+
+class ZkRequeueM1Action < ZookeeperAction
+end
+
+class ZkDeleteM1Action < ZookeeperAction
+end
+
+class ZkHoldM1Action < ZookeeperAction
+end
+
+class ZkReleaseM1Action < ZookeeperAction
+end
+
+class LegacyZkAction < ZookeeperAction
+  def status_vals
+    %w[Pending Consumed Deleted Failed Completed Held]
+  end
+
+  def set_status(status)
+    path = "/ingest/#{@qid}"
+    i = status_vals.find_index(status)
+    retun if i.nil?
+    data = @zk.get(path)
+    return if data.nil?
+    bytes = data[0].bytes
+    orig = bytes[0]
+    bytes[0] = i 
+    @zk.set(path, bytes.pack("CCCCCCCCCc*"))
+    {message: "Status #{orig} -- > #{status}"}.to_json
+  end
+end
+
+class ZkRequeueM2Action < LegacyZkAction
+  def perform_action
+    set_status('Pending')
+  end
+end
+
+class ZkDeleteM2Action < LegacyZkAction
+  def perform_action
+    set_status('Deleted')
+  end
+end
+
+class ZkHoldM2Action < LegacyZkAction
+  def perform_action
+    set_status('Held')
+  end
+end
+
+class ZkReleaseM2Action < LegacyZkAction
+  def perform_action
+    set_status('Pending')
+  end
+end
+
 ## Class for reading the legacy Merritt Ingest Queue
-class IngestQueueZookeeperAction < ZookeeperAction
+class IngestQueueZookeeperAction < ZookeeperListAction
   def zk_path
     '/ingest'
   end

src-colladmin/config/actions.yml

@@ -918,6 +918,32 @@ queue-delete:
   documentation: |
     Ingest: POST /admin/deleteq
   migration: m1, m2, m3
+queue-delete-m1:
+  link-title: Remove an item from a Zookeeper queue
+  class: ZkDeleteM1Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Remove an item from a Zookeeper queue.
+  report-datatypes:
+  - qdelete
+  documentation: |
+    Zk: tbd
+  migration: m1
+queue-delete-m2:
+  link-title: Remove an item from a Zookeeper queue
+  class: ZkDeleteM2Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Remove an item from a Zookeeper queue.
+  report-datatypes:
+  - qdelete
+  documentation: |
+    Zk: tbd
+  migration: m1
 requeue:
   link-title: Re-queue an item from a Zookeeper queue
   class: QueueAction
@@ -932,6 +958,32 @@ requeue:
   documentation: |
     Ingest: POST /admin/requeue
   migration: m1, m2, m3
+requeue-m1:
+  link-title: Re-queue an item from a Zookeeper queue
+  class: ZkRequeueM1Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Re-queue an item from a Zookeeper queue.
+  report-datatypes:
+  - requeue
+  documentation: |
+    ZK tbd
+  migration: m1
+requeue-m2:
+  link-title: Re-queue an item from a Zookeeper queue
+  class: ZkRequeueM2Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Re-queue an item from a Zookeeper queue.
+  report-datatypes:
+  - requeue
+  documentation: |
+    ZK tbd
+  migration: m1
 hold-queue-item:
   link-title: Move a pending item from in Zookeeper queue to a Held status
   class: QueueAction
@@ -946,6 +998,32 @@ hold-queue-item:
   documentation: |
     Ingest: POST /admin/hold
   migration: m1, m2
+hold-queue-item-m1:
+  link-title: Move a pending item from in Zookeeper queue to a Held status
+  class: ZkHoldM1Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Move a pending item from in Zookeeper queue to a Held status.
+  report-datatypes:
+  - hold
+  documentation: |
+    Zk: tbd
+  migration: m1
+hold-queue-item-m2:
+  link-title: Move a pending item from in Zookeeper queue to a Held status
+  class: ZkHoldM2Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Move a pending item from in Zookeeper queue to a Held status.
+  report-datatypes:
+  - hold
+  documentation: |
+    Zk: tbd
+  migration: m1
 release-queue-item:
   link-title: Release a held item from in Zookeeper queue to a Pending status
   class: QueueAction
@@ -960,6 +1038,32 @@ release-queue-item:
   documentation: |
     Ingest: POST /admin/release
   migration: m1, m2
+release-queue-item-m1:
+  link-title: Release a held item from in Zookeeper queue to a Pending status
+  class: ZkReleaseM1Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Release a held item from in Zookeeper queue to a Pending status.
+  report-datatypes:
+  - release
+  documentation: |
+    Zk: tbd
+  migration: m1
+release-queue-item-m2:
+  link-title: Release a held item from in Zookeeper queue to a Pending status
+  class: ZkReleaseM2Action
+  category: Queue Management
+  sensitivity: irreversible change
+  testing: manual
+  description: |
+    Release a held item from in Zookeeper queue to a Pending status.
+  report-datatypes:
+  - release
+  documentation: |
+    Zk: tbd
+  migration: m1
 cleanup-queue:
   link-title: Cleanup Deleted and Completed items from a Zookeeper Queue
   class: IterateQueueAction