diff --git a/cognito-lambda-nonvpc/Gemfile.lock b/cognito-lambda-nonvpc/Gemfile.lock index 5896d38..deaa1bd 100644 --- a/cognito-lambda-nonvpc/Gemfile.lock +++ b/cognito-lambda-nonvpc/Gemfile.lock @@ -3,11 +3,11 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.925.0) - aws-sdk-cognitoidentityprovider (1.90.0) + aws-partitions (1.929.0) + aws-sdk-cognitoidentityprovider (1.92.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.194.2) + aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) @@ -23,9 +23,9 @@ GEM racc racc (1.7.3) rainbow (3.1.1) - regexp_parser (2.9.0) + regexp_parser (2.9.1) rexml (3.2.6) - rubocop (1.63.4) + rubocop (1.63.5) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) diff --git a/mysql-ruby-lambda/Gemfile.lock b/mysql-ruby-lambda/Gemfile.lock index 0097ec5..8daf26f 100644 --- a/mysql-ruby-lambda/Gemfile.lock +++ b/mysql-ruby-lambda/Gemfile.lock @@ -13,13 +13,13 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.926.0) - aws-sdk-core (3.194.2) + aws-partitions (1.929.0) + aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-ssm (1.168.0) + aws-sdk-ssm (1.169.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) @@ -35,7 +35,7 @@ GEM racc racc (1.7.3) rainbow (3.1.1) - regexp_parser (2.9.0) + regexp_parser (2.9.1) rexml (3.2.6) rubocop (1.63.5) json (~> 2.3) diff --git a/src-admintool/Gemfile.lock b/src-admintool/Gemfile.lock index a5921d2..066823e 100644 --- a/src-admintool/Gemfile.lock +++ b/src-admintool/Gemfile.lock @@ -13,20 +13,20 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.926.0) - aws-sdk-core (3.194.2) + aws-partitions (1.929.0) + aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.80.0) + aws-sdk-kms (1.81.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.149.1) + aws-sdk-s3 (1.151.0) aws-sdk-core (~> 3, >= 3.194.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) - aws-sdk-ssm (1.168.0) + aws-sdk-ssm (1.169.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) @@ -42,7 +42,7 @@ GEM racc racc (1.7.3) rainbow (3.1.1) - regexp_parser (2.9.0) + regexp_parser (2.9.1) rexml (3.2.6) rubocop (1.63.5) json (~> 2.3) diff --git a/src-admintool/web/index.html b/src-admintool/web/index.html index 7f63005..f6f908e 100644 --- a/src-admintool/web/index.html +++ b/src-admintool/web/index.html @@ -465,12 +465,12 @@

Storage Configuration

Access Queues

diff --git a/src-colladmin/Gemfile b/src-colladmin/Gemfile index d71981f..f2ceb1c 100644 --- a/src-colladmin/Gemfile +++ b/src-colladmin/Gemfile @@ -12,7 +12,7 @@ gem 'aws-sdk-lambda' gem 'aws-sdk-s3' gem 'aws-sdk-ssm' gem 'csv' -gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.m' +gem 'mrt-zk', git: 'https://github.com/CDLUC3/mrt-zk.git', tag: '1.0.3.n' gem 'rest-client' gem 'rubocop' gem 'uc3-ssm', '0.3.10', git: 'https://github.com/CDLUC3/uc3-ssm.git', branch: 'main' diff --git a/src-colladmin/Gemfile.lock b/src-colladmin/Gemfile.lock index 47e492d..755d5ae 100644 --- a/src-colladmin/Gemfile.lock +++ b/src-colladmin/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/CDLUC3/mrt-zk.git - revision: 54820652b218e2108759db0f937abc31aa829e1c - tag: 1.0.3.m + revision: c47b62b41e93ce2db34243a6e5011af4de5ffa50 + tag: 1.0.3.n specs: mrt-zk (1.0.1) zk (~> 1.10.0) @@ -22,26 +22,26 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.926.0) - aws-sdk-core (3.194.2) + aws-partitions (1.929.0) + aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-ec2 (1.456.0) + aws-sdk-ec2 (1.457.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.80.0) + aws-sdk-kms (1.81.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-lambda (1.119.0) + aws-sdk-lambda (1.120.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.149.1) + aws-sdk-s3 (1.151.0) aws-sdk-core (~> 3, >= 3.194.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) - aws-sdk-ssm (1.168.0) + aws-sdk-ssm (1.169.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) @@ -66,7 +66,7 @@ GEM racc racc (1.7.3) rainbow (3.1.1) - regexp_parser (2.9.0) + regexp_parser (2.9.1) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) diff --git a/src-colladmin/actions/storage_action.rb b/src-colladmin/actions/storage_action.rb index ccc0887..6d0472a 100644 --- a/src-colladmin/actions/storage_action.rb +++ b/src-colladmin/actions/storage_action.rb @@ -245,50 +245,27 @@ def perform_action return res end - if @path == 'storage-rebuild-inventory' - # INV DELETE object/ARK - srvc = get_inventory_service - endpoint = "/object/#{CGI.escape(ark)}" - return { message: 'Inventory service undefined' }.to_json if srvc.empty? - return { message: 'Empty Ark' }.to_json if ark.empty? + return unless @path == 'storage-rebuild-inventory' - qjson = HttpDeleteJson.new(srvc, endpoint) - return { message: 'Inventory delete failed' }.to_json if qjson.status != 200 + # INV DELETE object/ARK + srvc = get_inventory_service + endpoint = "/object/#{CGI.escape(ark)}" + return { message: 'Inventory service undefined' }.to_json if srvc.empty? + return { message: 'Empty Ark' }.to_json if ark.empty? - endpoint = '/add' - data = { - 'url' => "#{get_storage_service}/manifest/#{nodenum}/#{CGI.escape(ark)}", - 'responseForm' => 'json' - } + qjson = HttpDeleteJson.new(srvc, endpoint) + return { message: 'Inventory delete failed' }.to_json if qjson.status != 200 - qjson = HttpPostMultipartJson.new(srvc, endpoint, data) - return { message: 'Inventory recreate failed' }.to_json if qjson.status != 200 + endpoint = '/add' + data = { + 'url' => "#{get_storage_service}/manifest/#{nodenum}/#{CGI.escape(ark)}", + 'responseForm' => 'json' + } - { message: 'Inventory Delete and Recreate Successful' }.to_json - end - - # if @path == "storage-update-manifest" - # end - return unless @path == 'storage-set-flag' + qjson = HttpPostMultipartJson.new(srvc, endpoint, data) + return { message: 'Inventory recreate failed' }.to_json if qjson.status != 200 - op = @myparams.fetch('op', 'set') - op = 'state' unless %w[set clear state].include?(op) - qobj = @myparams.fetch('object', '') - srvc = get_access_service - endpoint = "/flag/#{op}/access/#{qobj}?t=json" - qjson = HttpPostJson.new(srvc, endpoint) - return message_as_table('Access ZK flag set failed').to_json if qjson.status != 200 - - ts = JSON.parse(qjson.body).fetch('tok:zooTokenState', {}) - tss = ts.fetch('tok:tokenStatus', '') - tso = ts.fetch('tok:zooFlagPath', 'na') - reload_path = @myparams.fetch('reload_path', '') - unless reload_path.empty? - return { - redirect_location: "/web/collIndex.html?path=#{reload_path}" - }.to_json - end - message_as_table("Token result: #{tso}=#{tss}").to_json + { message: 'Inventory Delete and Recreate Successful' }.to_json end def get_storage_service diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb index 8582ae0..eee8f95 100644 --- a/src-colladmin/actions/zookeeper_action.rb +++ b/src-colladmin/actions/zookeeper_action.rb @@ -396,3 +396,40 @@ def perform_action }.to_json end end + +## Control Access Queue Hold/Release (legacy and mrt-zk) +class AccessLockAction < ZookeeperAction + def initialize(config, action, path, myparams) + super(config, action, path, myparams) + @op = @myparams.fetch('op', 'set') + @op = 'state' unless %w[set clear state].include?(@op) + @flag = @myparams.fetch('object', '') + end + + def largeq? + @flag == 'LargeAccessHold' + end + + def perform_action + lockpath = 'tbd' + if ZookeeperListAction.migration_m3? + lockpath = largeq? ? MerrittZK::Locks::LOCKS_QUEUE_ACCESS_LARGE : MerrittZK::Locks::LOCKS_QUEUE_ACCESS_SMALL + case @op + when 'set' + largeq? ? MerrittZK::Locks.lock_large_access_queue(@zk) : MerrittZK::Locks.lock_small_access_queue(@zk) + when 'clear' + largeq? ? MerrittZK::Locks.unlock_large_access_queue(@zk) : MerrittZK::Locks.unlock_small_access_queue(@zk) + end + else + lockpath = "/mrt.lock/access/#{@flag}" + case @op + when 'set' + @zk.create(lockpath, data: nil) unless exists?(lockpath) + when 'clear' + @zk.delete(lockpath) if @zk.exists?(lockpath) + end + end + state = @zk.exists?(lockpath) ? 'Held' : 'Released' + message_as_table("Lock #{@op} status result: #{lockpath}=#{state}").to_json + end +end diff --git a/src-colladmin/config/actions.yml b/src-colladmin/config/actions.yml index 6240197..b521dde 100644 --- a/src-colladmin/config/actions.yml +++ b/src-colladmin/config/actions.yml @@ -115,7 +115,7 @@ queues: documentation: | Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m1, m2 + migration: m1 ingest-queue-by-profile: link-title: Ingest Queue Activity by Profile breadcrumb: bp_ingest @@ -128,7 +128,7 @@ ingest-queue-by-profile: documentation: | Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m1, m2 + migration: m1 inv-queues: link-title: Inventory Queue Jobs breadcrumb: bp_inventory @@ -142,7 +142,7 @@ inv-queues: documentation: | Ingest: GET /admin/queues-inv Ingest: GET /admin/queue-inv/{node} - migration: m1, m2 + migration: m1 acc-queues: link-title: Access Queue Jobs breadcrumb: bp_storage @@ -159,7 +159,7 @@ acc-queues: documentation: | Ingest: GET /admin/queues-acc Ingest: GET /admin/queue-acc/{node} - migration: m1, m3 + migration: m1 batchFolders: link-title: Ingest Batch Folders breadcrumb: bp_ingest @@ -196,7 +196,7 @@ batch: documentation: | Ingest: GET /admin/queues Ingest: GET /admin/queue/{node} - migration: m1, m2 + migration: m1 job: link-title: Ingest Job breadcrumb: bp_ingest @@ -496,7 +496,7 @@ ingest-locks: Display the ingest storage locks existing for object arks. documentation: | Ingest: GET /admin/lock/mrt.lock - migration: m1, m2 + migration: m1 # Operations that change system state - Reversible @@ -567,7 +567,7 @@ release-coll-queue-items-m1: - collqitems documentation: | Ingest: POST /admin/release-all/queue/{coll} - migration: m2 + migration: m1 release-coll-queue-items-legacy: link-title: Release any held items for a collection (legacy) class: CollIterateQueueLegacyAction @@ -580,7 +580,7 @@ release-coll-queue-items-legacy: - collqitems documentation: | Ingest: POST /admin/release-all/queue/{coll} - migration: m2 + migration: m1 storage-force-audit-for-object: link-title: Trigger audit processing for all files in an object on a specific node class: StorageAction @@ -1032,7 +1032,7 @@ cleanup-queue: - release documentation: | Ingest: POST /admin/cleanupq/{queue} - migration: m1, m2, m3 + migration: m1 apply-review-changes: link-title: Apply bulk changes to storage scan results by uploading a CSV containing a set of desired changes class: ReplicationAction @@ -1111,9 +1111,9 @@ storage-del-node-for-collection: documentation: | SQL: DELETE FROM inv_collections_inv_nodes WHERE inv_collection_id = ? migration: m0 -storage-set-flag: +access-lock-control: link-title: Storage set zookeeper flag - class: StorageAction + class: AccessLockAction category: Queue Management sensitivity: reversible change testing: manual @@ -1121,7 +1121,7 @@ storage-set-flag: Change the state of a zookeeper flag (SET or CLEAR). documentation: | ZK: set boolean flag - migration: m1, m3? + migration: m3 replic-delete-coll-batch-from-node: link-title: Delete a batch of objects from the collection from storage node class: ReplicationAction diff --git a/src-common/Gemfile.lock b/src-common/Gemfile.lock index fc6ec6c..73c78b9 100644 --- a/src-common/Gemfile.lock +++ b/src-common/Gemfile.lock @@ -13,13 +13,13 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.926.0) - aws-sdk-core (3.194.2) + aws-partitions (1.929.0) + aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-ssm (1.168.0) + aws-sdk-ssm (1.169.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) @@ -41,7 +41,7 @@ GEM racc racc (1.7.3) rainbow (3.1.1) - regexp_parser (2.9.0) + regexp_parser (2.9.1) rexml (3.2.6) rubocop (1.63.5) json (~> 2.3) diff --git a/src-testdriver/Gemfile.lock b/src-testdriver/Gemfile.lock index 0790828..8653057 100644 --- a/src-testdriver/Gemfile.lock +++ b/src-testdriver/Gemfile.lock @@ -13,16 +13,16 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.925.0) - aws-sdk-core (3.194.2) + aws-partitions (1.929.0) + aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-lambda (1.119.0) + aws-sdk-lambda (1.120.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-ssm (1.168.0) + aws-sdk-ssm (1.169.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) @@ -36,7 +36,7 @@ GEM language_server-protocol (3.17.0.3) logger (1.6.0) mini_portile2 (2.8.6) - nokogiri (1.16.4) + nokogiri (1.16.5) mini_portile2 (~> 2.8.2) racc (~> 1.4) parallel (1.24.0) @@ -45,7 +45,7 @@ GEM racc racc (1.7.3) rainbow (3.1.1) - regexp_parser (2.9.0) + regexp_parser (2.9.1) rexml (3.2.6) rspec (3.13.0) rspec-core (~> 3.13.0) @@ -60,7 +60,7 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.1) - rubocop (1.63.4) + rubocop (1.63.5) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10)