hold/release access queue

CDLUC3 · May 15, 2024 · 547b4c4 · 547b4c4
1 parent a5c24f0
commit 547b4c4
Show file tree

Hide file tree

Showing 11 changed files with 108 additions and 94 deletions.
diff --git a/cognito-lambda-nonvpc/Gemfile.lock b/cognito-lambda-nonvpc/Gemfile.lock
@@ -3,11 +3,11 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.925.0)
-    aws-sdk-cognitoidentityprovider (1.90.0)
+    aws-partitions (1.929.0)
+    aws-sdk-cognitoidentityprovider (1.92.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.194.2)
+    aws-sdk-core (3.196.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
@@ -23,9 +23,9 @@ GEM
       racc
     racc (1.7.3)
     rainbow (3.1.1)
-    regexp_parser (2.9.0)
+    regexp_parser (2.9.1)
     rexml (3.2.6)
-    rubocop (1.63.4)
+    rubocop (1.63.5)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)

diff --git a/mysql-ruby-lambda/Gemfile.lock b/mysql-ruby-lambda/Gemfile.lock
@@ -13,13 +13,13 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.926.0)
-    aws-sdk-core (3.194.2)
+    aws-partitions (1.929.0)
+    aws-sdk-core (3.196.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-ssm (1.168.0)
+    aws-sdk-ssm (1.169.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.8.0)
@@ -35,7 +35,7 @@ GEM
       racc
     racc (1.7.3)
     rainbow (3.1.1)
-    regexp_parser (2.9.0)
+    regexp_parser (2.9.1)
     rexml (3.2.6)
     rubocop (1.63.5)
       json (~> 2.3)

diff --git a/src-admintool/Gemfile.lock b/src-admintool/Gemfile.lock
@@ -13,20 +13,20 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.926.0)
-    aws-sdk-core (3.194.2)
+    aws-partitions (1.929.0)
+    aws-sdk-core (3.196.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-kms (1.80.0)
+    aws-sdk-kms (1.81.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.149.1)
+    aws-sdk-s3 (1.151.0)
       aws-sdk-core (~> 3, >= 3.194.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.8)
-    aws-sdk-ssm (1.168.0)
+    aws-sdk-ssm (1.169.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.8.0)
@@ -42,7 +42,7 @@ GEM
       racc
     racc (1.7.3)
     rainbow (3.1.1)
-    regexp_parser (2.9.0)
+    regexp_parser (2.9.1)
     rexml (3.2.6)
     rubocop (1.63.5)
       json (~> 2.3)

diff --git a/src-admintool/web/index.html b/src-admintool/web/index.html
@@ -465,12 +465,12 @@ <h3>Storage Configuration</h3>
       <h3>Access Queues</h3>
       <ul>
         <li class="job"><a href="{{COLLADMIN_HOME}}?path=acc-queues">Access Queues</a></li>
-        <li class="report"><a href="{{COLLADMIN_HOME}}?path=storage-set-flag&op=state&object=LargeAccessHold">Get Large Access State</a></li>
-        <li class="action"><a href="{{COLLADMIN_HOME}}?path=storage-set-flag&op=set&object=LargeAccessHold">Pause Large Access</a></li>
-        <li class="action"><a href="{{COLLADMIN_HOME}}?path=storage-set-flag&op=clear&object=LargeAccessHold">Clear Large Access</a></li>
-        <li class="report"><a href="{{COLLADMIN_HOME}}?path=storage-set-flag&op=state&object=SmallAccessHold">Get Small Access State</a></li>
-        <li class="action"><a href="{{COLLADMIN_HOME}}?path=storage-set-flag&op=set&object=SmallAccessHold">Pause Small Access</a></li>
-        <li class="action"><a href="{{COLLADMIN_HOME}}?path=storage-set-flag&op=clear&object=SmallAccessHold">Clear Small Access</a></li>
+        <li class="report"><a href="{{COLLADMIN_HOME}}?path=access-lock-control&op=state&object=LargeAccessHold">Get Large Access State</a></li>
+        <li class="action"><a href="{{COLLADMIN_HOME}}?path=access-lock-control&op=set&object=LargeAccessHold">Pause Large Access</a></li>
+        <li class="action"><a href="{{COLLADMIN_HOME}}?path=access-lock-control&op=clear&object=LargeAccessHold">Clear Large Access</a></li>
+        <li class="report"><a href="{{COLLADMIN_HOME}}?path=access-lock-control&op=state&object=SmallAccessHold">Get Small Access State</a></li>
+        <li class="action"><a href="{{COLLADMIN_HOME}}?path=access-lock-control&op=set&object=SmallAccessHold">Pause Small Access</a></li>
+        <li class="action"><a href="{{COLLADMIN_HOME}}?path=access-lock-control&op=clear&object=SmallAccessHold">Clear Small Access</a></li>
       </ul>
     </div>
 

diff --git a/src-colladmin/Gemfile b/src-colladmin/Gemfile
@@ -12,7 +12,7 @@ gem 'aws-sdk-lambda'
 gem 'aws-sdk-s3'
 gem 'aws-sdk-ssm'
 gem 'csv'
-gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.m'
+gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.n'
 gem 'rest-client'
 gem 'rubocop'
 gem 'uc3-ssm', '0.3.10', git: 'https://github.com/CDLUC3/uc3-ssm.git', branch: 'main'

diff --git a/src-colladmin/Gemfile.lock b/src-colladmin/Gemfile.lock
@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/CDLUC3/mrt-zk.git
-  revision: 54820652b218e2108759db0f937abc31aa829e1c
-  tag: 1.0.3.m
+  revision: c47b62b41e93ce2db34243a6e5011af4de5ffa50
+  tag: 1.0.3.n
   specs:
     mrt-zk (1.0.1)
       zk (~> 1.10.0)
@@ -22,26 +22,26 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.926.0)
-    aws-sdk-core (3.194.2)
+    aws-partitions (1.929.0)
+    aws-sdk-core (3.196.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-ec2 (1.456.0)
+    aws-sdk-ec2 (1.457.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.80.0)
+    aws-sdk-kms (1.81.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-lambda (1.119.0)
+    aws-sdk-lambda (1.120.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.149.1)
+    aws-sdk-s3 (1.151.0)
       aws-sdk-core (~> 3, >= 3.194.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.8)
-    aws-sdk-ssm (1.168.0)
+    aws-sdk-ssm (1.169.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.8.0)
@@ -66,7 +66,7 @@ GEM
       racc
     racc (1.7.3)
     rainbow (3.1.1)
-    regexp_parser (2.9.0)
+    regexp_parser (2.9.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)

diff --git a/src-colladmin/actions/storage_action.rb b/src-colladmin/actions/storage_action.rb
@@ -245,50 +245,27 @@ def perform_action
       return res
     end
 
-    if @path == 'storage-rebuild-inventory'
-      # INV DELETE object/ARK
-      srvc = get_inventory_service
-      endpoint = "/object/#{CGI.escape(ark)}"
-      return { message: 'Inventory service undefined' }.to_json if srvc.empty?
-      return { message: 'Empty Ark' }.to_json if ark.empty?
+    return unless @path == 'storage-rebuild-inventory'
 
-      qjson = HttpDeleteJson.new(srvc, endpoint)
-      return { message: 'Inventory delete failed' }.to_json if qjson.status != 200
+    # INV DELETE object/ARK
+    srvc = get_inventory_service
+    endpoint = "/object/#{CGI.escape(ark)}"
+    return { message: 'Inventory service undefined' }.to_json if srvc.empty?
+    return { message: 'Empty Ark' }.to_json if ark.empty?
 
-      endpoint = '/add'
-      data = {
-        'url' => "#{get_storage_service}/manifest/#{nodenum}/#{CGI.escape(ark)}",
-        'responseForm' => 'json'
-      }
+    qjson = HttpDeleteJson.new(srvc, endpoint)
+    return { message: 'Inventory delete failed' }.to_json if qjson.status != 200
 
-      qjson = HttpPostMultipartJson.new(srvc, endpoint, data)
-      return { message: 'Inventory recreate failed' }.to_json if qjson.status != 200
+    endpoint = '/add'
+    data = {
+      'url' => "#{get_storage_service}/manifest/#{nodenum}/#{CGI.escape(ark)}",
+      'responseForm' => 'json'
+    }
 
-      { message: 'Inventory Delete and Recreate Successful' }.to_json
-    end
-
-    # if @path == "storage-update-manifest"
-    # end
-    return unless @path == 'storage-set-flag'
+    qjson = HttpPostMultipartJson.new(srvc, endpoint, data)
+    return { message: 'Inventory recreate failed' }.to_json if qjson.status != 200
 
-    op = @myparams.fetch('op', 'set')
-    op = 'state' unless %w[set clear state].include?(op)
-    qobj = @myparams.fetch('object', '')
-    srvc = get_access_service
-    endpoint = "/flag/#{op}/access/#{qobj}?t=json"
-    qjson = HttpPostJson.new(srvc, endpoint)
-    return message_as_table('Access ZK flag set failed').to_json if qjson.status != 200
-
-    ts = JSON.parse(qjson.body).fetch('tok:zooTokenState', {})
-    tss = ts.fetch('tok:tokenStatus', '')
-    tso = ts.fetch('tok:zooFlagPath', 'na')
-    reload_path = @myparams.fetch('reload_path', '')
-    unless reload_path.empty?
-      return {
-        redirect_location: "/web/collIndex.html?path=#{reload_path}"
-      }.to_json
-    end
-    message_as_table("Token result: #{tso}=#{tss}").to_json
+    { message: 'Inventory Delete and Recreate Successful' }.to_json
   end
 
   def get_storage_service

diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb
@@ -396,3 +396,40 @@ def perform_action
     }.to_json
   end
 end
+
+## Control Access Queue Hold/Release (legacy and mrt-zk)
+class AccessLockAction < ZookeeperAction
+  def initialize(config, action, path, myparams)
+    super(config, action, path, myparams)
+    @op = @myparams.fetch('op', 'set')
+    @op = 'state' unless %w[set clear state].include?(@op)
+    @flag = @myparams.fetch('object', '')
+  end
+
+  def largeq?
+    @flag == 'LargeAccessHold'
+  end
+
+  def perform_action
+    lockpath = 'tbd'
+    if ZookeeperListAction.migration_m3?
+      lockpath = largeq? ? MerrittZK::Locks::LOCKS_QUEUE_ACCESS_LARGE : MerrittZK::Locks::LOCKS_QUEUE_ACCESS_SMALL
+      case @op
+      when 'set'
+        largeq? ? MerrittZK::Locks.lock_large_access_queue(@zk) : MerrittZK::Locks.lock_small_access_queue(@zk)
+      when 'clear'
+        largeq? ? MerrittZK::Locks.unlock_large_access_queue(@zk) : MerrittZK::Locks.unlock_small_access_queue(@zk)
+      end
+    else
+      lockpath = "/mrt.lock/access/#{@flag}"
+      case @op
+      when 'set'
+        @zk.create(lockpath, data: nil) unless exists?(lockpath)
+      when 'clear'
+        @zk.delete(lockpath) if @zk.exists?(lockpath)
+      end
+    end
+    state = @zk.exists?(lockpath) ? 'Held' : 'Released'
+    message_as_table("Lock #{@op} status result: #{lockpath}=#{state}").to_json
+  end
+end
diff --git a/src-colladmin/config/actions.yml b/src-colladmin/config/actions.yml
@@ -115,7 +115,7 @@ queues:
   documentation: |
     Ingest: GET /admin/queues
     Ingest: GET /admin/queue/{node}
-  migration: m1, m2
+  migration: m1
 ingest-queue-by-profile:
   link-title: Ingest Queue Activity by Profile
   breadcrumb: bp_ingest
@@ -128,7 +128,7 @@ ingest-queue-by-profile:
   documentation: |
     Ingest: GET /admin/queues
     Ingest: GET /admin/queue/{node}
-  migration: m1, m2
+  migration: m1
 inv-queues:
   link-title: Inventory Queue Jobs
   breadcrumb: bp_inventory
@@ -142,7 +142,7 @@ inv-queues:
   documentation: |
     Ingest: GET /admin/queues-inv
     Ingest: GET /admin/queue-inv/{node}
-  migration: m1, m2
+  migration: m1
 acc-queues:
   link-title: Access Queue Jobs
   breadcrumb: bp_storage
@@ -159,7 +159,7 @@ acc-queues:
   documentation: |
     Ingest: GET /admin/queues-acc
     Ingest: GET /admin/queue-acc/{node}
-  migration: m1, m3
+  migration: m1
 batchFolders:
   link-title: Ingest Batch Folders
   breadcrumb: bp_ingest
@@ -196,7 +196,7 @@ batch:
   documentation: |
     Ingest: GET /admin/queues
     Ingest: GET /admin/queue/{node}
-  migration: m1, m2
+  migration: m1
 job:
   link-title: Ingest Job
   breadcrumb: bp_ingest
@@ -496,7 +496,7 @@ ingest-locks:
     Display the ingest storage locks existing for object arks.
   documentation: |
     Ingest: GET /admin/lock/mrt.lock
-  migration: m1, m2
+  migration: m1
 
 # Operations that change system state - Reversible
 
@@ -567,7 +567,7 @@ release-coll-queue-items-m1:
   - collqitems
   documentation: |
     Ingest: POST /admin/release-all/queue/{coll}
-  migration: m2
+  migration: m1
 release-coll-queue-items-legacy:
   link-title: Release any held items for a collection (legacy)
   class: CollIterateQueueLegacyAction
@@ -580,7 +580,7 @@ release-coll-queue-items-legacy:
   - collqitems
   documentation: |
     Ingest: POST /admin/release-all/queue/{coll}
-  migration: m2
+  migration: m1
 storage-force-audit-for-object:
   link-title: Trigger audit processing for all files in an object on a specific node
   class: StorageAction
@@ -1032,7 +1032,7 @@ cleanup-queue:
   - release
   documentation: |
     Ingest: POST /admin/cleanupq/{queue}
-  migration: m1, m2, m3
+  migration: m1
 apply-review-changes:
   link-title: Apply bulk changes to storage scan results by uploading a CSV containing a set of desired changes
   class: ReplicationAction
@@ -1111,17 +1111,17 @@ storage-del-node-for-collection:
   documentation: |
     SQL: DELETE FROM inv_collections_inv_nodes WHERE inv_collection_id = ?
   migration: m0
-storage-set-flag:
+access-lock-control:
   link-title: Storage set zookeeper flag
-  class: StorageAction
+  class: AccessLockAction
   category: Queue Management
   sensitivity: reversible change
   testing: manual
   description: |
     Change the state of a zookeeper flag (SET or CLEAR).
   documentation: |
     ZK: set boolean flag
-  migration: m1, m3?
+  migration: m3
 replic-delete-coll-batch-from-node:
   link-title: Delete a batch of objects from the collection from storage node
   class: ReplicationAction