From ffd65c103eb1328721782d4ca70394cdb5a2ce53 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Tue, 5 Mar 2024 17:11:57 -0800 Subject: [PATCH 01/24] init read from zk --- cognito-lambda-nonvpc/.dockerignore | 3 +- mysql-ruby-lambda/.dockerignore | 3 +- simulate-lambda-alb/.dockerignore | 3 +- src-admintool/.dockerignore | 3 +- src-colladmin/.dockerignore | 3 +- src-colladmin/Dockerfile | 2 +- src-colladmin/Gemfile | 2 + src-colladmin/actions/ingest_queue_action.rb | 12 +-- src-colladmin/actions/zookeeper_action.rb | 78 ++++++++++++++++++++ src-colladmin/config/database.ssm.yml | 2 + src-colladmin/lib/queue.rb | 32 +++++--- src-common/.dockerignore | 3 +- 12 files changed, 119 insertions(+), 27 deletions(-) create mode 100644 src-colladmin/actions/zookeeper_action.rb diff --git a/cognito-lambda-nonvpc/.dockerignore b/cognito-lambda-nonvpc/.dockerignore index 02457453..e76e3b51 100644 --- a/cognito-lambda-nonvpc/.dockerignore +++ b/cognito-lambda-nonvpc/.dockerignore @@ -1,4 +1,5 @@ .bundle/ vendor/ README.md -Dockerfile \ No newline at end of file +Dockerfile +Gemfile.lock \ No newline at end of file diff --git a/mysql-ruby-lambda/.dockerignore b/mysql-ruby-lambda/.dockerignore index 02457453..e76e3b51 100644 --- a/mysql-ruby-lambda/.dockerignore +++ b/mysql-ruby-lambda/.dockerignore @@ -1,4 +1,5 @@ .bundle/ vendor/ README.md -Dockerfile \ No newline at end of file +Dockerfile +Gemfile.lock \ No newline at end of file diff --git a/simulate-lambda-alb/.dockerignore b/simulate-lambda-alb/.dockerignore index 02457453..e76e3b51 100644 --- a/simulate-lambda-alb/.dockerignore +++ b/simulate-lambda-alb/.dockerignore @@ -1,4 +1,5 @@ .bundle/ vendor/ README.md -Dockerfile \ No newline at end of file +Dockerfile +Gemfile.lock \ No newline at end of file diff --git a/src-admintool/.dockerignore b/src-admintool/.dockerignore index 06a6d30f..e80a4c5e 100644 --- a/src-admintool/.dockerignore +++ b/src-admintool/.dockerignore @@ -1,2 +1,3 @@ Dockerfile -vendor/ \ No newline at end of file +vendor/ +Gemfile.lock \ No newline at end of file diff --git a/src-colladmin/.dockerignore b/src-colladmin/.dockerignore index 923e3ed5..9de7653d 100644 --- a/src-colladmin/.dockerignore +++ b/src-colladmin/.dockerignore @@ -1,3 +1,4 @@ Dockerfile README.md -vendor/ \ No newline at end of file +vendor/ +Gemfile.lock \ No newline at end of file diff --git a/src-colladmin/Dockerfile b/src-colladmin/Dockerfile index 7db0e7be..fead6063 100644 --- a/src-colladmin/Dockerfile +++ b/src-colladmin/Dockerfile @@ -13,7 +13,7 @@ FROM ${ECR_REGISTRY}/uc3-mrt-admin-common RUN yum -y update # Build rest-client gem from source -RUN yum -y install gcc-c++ +RUN yum -y install gcc-c++ && yum -y groupinstall "Development Tools" # Add Admin Tool Code to the image COPY . /var/task/ diff --git a/src-colladmin/Gemfile b/src-colladmin/Gemfile index a558ed49..def92993 100644 --- a/src-colladmin/Gemfile +++ b/src-colladmin/Gemfile @@ -16,3 +16,5 @@ gem 'rest-client' gem 'rubocop' gem 'uc3-ssm', '0.3.10', git: 'https://github.com/CDLUC3/uc3-ssm.git', branch: 'main' gem 'unf_ext', '~> 0.0.7.7' +gem 'zk' +gem 'zookeeper' diff --git a/src-colladmin/actions/ingest_queue_action.rb b/src-colladmin/actions/ingest_queue_action.rb index df4e8219..43cdbf2e 100644 --- a/src-colladmin/actions/ingest_queue_action.rb +++ b/src-colladmin/actions/ingest_queue_action.rb @@ -1,19 +1,19 @@ # frozen_string_literal: true -require_relative 'forward_to_ingest_action' +require_relative 'zookeeper_action' # Collection Admin Task class - see config/actions.yml for description -class IngestQueueAction < ForwardToIngestAction +class IngestQueueAction < ZookeeperAction def initialize(config, action, path, myparams) @batch = myparams.fetch('batch', '') @profile = myparams.fetch('profile', '') @qstatus = myparams.fetch('qstatus', '') - super(config, action, path, myparams, 'admin/queues') @filter = { batch: @batch, profile: @profile, qstatus: @qstatus } + super(config, action, path, myparams, @filter) @batches = {} end @@ -29,12 +29,6 @@ def table_types QueueEntry.table_types end - def table_rows(body) - queue_list = QueueList.new(get_ingest_server, body, @filter) - @batches = queue_list.batches - queue_list.to_table - end - def has_table true end diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb new file mode 100644 index 00000000..da85cc50 --- /dev/null +++ b/src-colladmin/actions/zookeeper_action.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require_relative 'action' +require_relative '../lib/queue' +require 'zk' + +# Collection Admin Task class - see config/actions.yml for description +class ZkList + def initialize + @jobs = [] + end + + def add_job(job) + @jobs.push(job) + end + + def to_table + table = [] + js = @jobs.sort do |a, b| + if a.status == b.status + b.date <=> a.date + else + AdminTask.status_sort_val(a.status) <=> AdminTask.status_sort_val(b.status) + end + end + js.each_with_index do |q, _i| + table.append(q.to_table_row) + end + table + end +end + +class ZookeeperAction < AdminAction + @@status_vals = ['Pending', 'Consumed', 'Deleted', 'Completed', 'Failed', 'Resolved'] + def initialize(config, action, path, myparams, filters) + super(config, action, path, myparams) + @filters = {} + @path = '/ingest' + @zk = ZK.new(get_zookeeper_conn) + @jobs = ZkList.new + end + + def perform_action + @zk.children(@path).each do |cp| + puts cp + arr = @zk.get("#{@path}/#{cp}") + next if arr[0].nil? + data = arr[0].bytes + return if data.length < 9 + status = data[0] + # https://stackoverflow.com/a/68855488/3846548 + t = data[1..8].inject(0) {|m, b| (m << 8) + b } + time = Time.at(t/1000) + payload=data[9..].pack('c*') + begin + json = JSON.parse(payload) + json['queueNode'] = 'ingest' + json['id'] = cp + json['date'] = time + json['status'] = @@status_vals[status] + puts json.to_json + @jobs.add_job(QueueEntry.new(json)) + #puts JSON.pretty_generate(json) + rescue => exception + #puts exception + end + end + convert_json_to_table('') + end + + def table_rows(_body) + @jobs.to_table + end + + def get_zookeeper_conn + @config.fetch('zookeeper', '').split(',').first + end +end diff --git a/src-colladmin/config/database.ssm.yml b/src-colladmin/config/database.ssm.yml index daeda4d2..10e9643a 100644 --- a/src-colladmin/config/database.ssm.yml +++ b/src-colladmin/config/database.ssm.yml @@ -173,6 +173,7 @@ default: keylist: 7001:scanlist/9502.log disable-scan-nodenums: "{!SSM: colladmin/disable-scan-nodenums !DEFAULT: 0}" uc3inv_home: "{!SSM: admintool/uc3inv_home !DEFAULT: ''}" + zookeeper: "{!SSM: inventory/zoo/queueService}" # use personal credentials when configuring a local environment # set up a script to set these variables docker: @@ -212,4 +213,5 @@ docker: # Provide state endpoints for specific subservices - not applicable for docker endpoints: uc3inv_home: "" + zookeeper: "zoo:2181" diff --git a/src-colladmin/lib/queue.rb b/src-colladmin/lib/queue.rb index ac55a0b3..fb02a0c5 100644 --- a/src-colladmin/lib/queue.rb +++ b/src-colladmin/lib/queue.rb @@ -11,51 +11,61 @@ def self.placeholder end def initialize(json) + # not yet used + #{ + # "queuePriority": "03", + # "responseForm": "xml", + # "localID": "2024_03_04_1717_v1file", + # "update": true + #} + super() # until July 2023, Merritt had 3 separate queues identified as a queue node add_property( :queueNode, - MerrittJsonProperty.new('Ingest Worker').lookup_value(json, 'que', 'queueNode') + MerrittJsonProperty.new('Ingest Worker').lookup_value(json, '', 'queueNode') ) add_property( :bid, - MerrittJsonProperty.new('Batch').lookup_value(json, 'que', 'batchID') + MerrittJsonProperty.new('Batch').lookup_value(json, '', 'batchID') ) add_property( :job, - MerrittJsonProperty.new('Job').lookup_value(json, 'que', 'jobID') + MerrittJsonProperty.new('Job').lookup_value(json, '', 'jobID') ) add_property( :profile, - MerrittJsonProperty.new('Profile').lookup_value(json, 'que', 'profile') + MerrittJsonProperty.new('Profile').lookup_value(json, '', 'profile') ) + # insert binary time field add_property( :date, - MerrittJsonProperty.new('Date').lookup_time_value(json, 'que', 'date') + MerrittJsonProperty.new('Date').lookup_time_value(json, '', 'date') ) add_property( :user, - MerrittJsonProperty.new('User').lookup_value(json, 'que', 'user') + MerrittJsonProperty.new('User').lookup_value(json, '', 'submitter') ) add_property( :title, - MerrittJsonProperty.new('Title').lookup_value(json, 'que', 'objectTitle') + MerrittJsonProperty.new('Title').lookup_value(json, '', 'title') ) add_property( :file_type, - MerrittJsonProperty.new('File Type').lookup_value(json, 'que', 'file_type') + MerrittJsonProperty.new('File Type').lookup_value(json, '', 'type') ) + # insert status from binary field add_property( :qstatus, - MerrittJsonProperty.new('QStatus').lookup_value(json, 'que', 'status') + MerrittJsonProperty.new('QStatus').lookup_value(json, '', 'status') ) add_property( :queue, - MerrittJsonProperty.new('Name').lookup_value(json, 'que', 'name') + MerrittJsonProperty.new('Name').lookup_value(json, '', 'filename') ) add_property( :queueId, - MerrittJsonProperty.new('Queue ID').lookup_value(json, 'que', 'iD') + MerrittJsonProperty.new('Queue ID').lookup_value(json, '', 'id') ) # extract the ingest worker node from the queue id string qid = get_value(:queueId, '') diff --git a/src-common/.dockerignore b/src-common/.dockerignore index 139884ec..dbc4dd99 100644 --- a/src-common/.dockerignore +++ b/src-common/.dockerignore @@ -1,4 +1,5 @@ Dockerfile README.md docker-compose.yml -vendor/ \ No newline at end of file +vendor/ +Gemfile.lock \ No newline at end of file From f4b9c84e9894a9347c54acfec8564ca2b43aa89b Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Thu, 7 Mar 2024 17:22:42 -0800 Subject: [PATCH 02/24] gem updates --- cognito-lambda-nonvpc/Gemfile.lock | 18 +++++++++--------- mysql-ruby-lambda/Gemfile.lock | 20 ++++++++++---------- simulate-lambda-alb/Gemfile.lock | 25 ++++++++++++++----------- src-admintool/Gemfile.lock | 28 ++++++++++++++-------------- 4 files changed, 47 insertions(+), 44 deletions(-) diff --git a/cognito-lambda-nonvpc/Gemfile.lock b/cognito-lambda-nonvpc/Gemfile.lock index 1dce9193..35e246ec 100644 --- a/cognito-lambda-nonvpc/Gemfile.lock +++ b/cognito-lambda-nonvpc/Gemfile.lock @@ -3,11 +3,11 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.881.0) - aws-sdk-cognitoidentityprovider (1.85.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-partitions (1.895.0) + aws-sdk-cognitoidentityprovider (1.87.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.190.3) + aws-sdk-core (3.191.3) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) @@ -18,14 +18,14 @@ GEM json (2.7.1) language_server-protocol (3.17.0.3) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -33,11 +33,11 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) unicode-display_width (2.5.0) diff --git a/mysql-ruby-lambda/Gemfile.lock b/mysql-ruby-lambda/Gemfile.lock index c8caace9..f90526d8 100644 --- a/mysql-ruby-lambda/Gemfile.lock +++ b/mysql-ruby-lambda/Gemfile.lock @@ -13,14 +13,14 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.881.0) - aws-sdk-core (3.190.3) + aws-partitions (1.895.0) + aws-sdk-core (3.191.3) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-ssm (1.162.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-sdk-ssm (1.165.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) aws-eventstream (~> 1, >= 1.0.2) @@ -28,16 +28,16 @@ GEM json (2.7.1) language_server-protocol (3.17.0.3) logger (1.6.0) - mysql2 (0.5.5) + mysql2 (0.5.6) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -45,11 +45,11 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) unicode-display_width (2.5.0) yaml (0.2.1) diff --git a/simulate-lambda-alb/Gemfile.lock b/simulate-lambda-alb/Gemfile.lock index 1e616124..dd2135e5 100644 --- a/simulate-lambda-alb/Gemfile.lock +++ b/simulate-lambda-alb/Gemfile.lock @@ -9,18 +9,20 @@ GEM mustermann (3.0.0) ruby2_keywords (~> 0.0.1) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) - rack (2.2.8) - rack-protection (3.2.0) + rack (3.0.9.1) + rack-protection (4.0.0) base64 (>= 0.1.0) - rack (~> 2.2, >= 2.2.4) + rack (>= 3.0.0, < 4) + rack-session (2.0.0) + rack (>= 3.0.0) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -28,17 +30,18 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) - sinatra (3.2.0) + sinatra (4.0.0) mustermann (~> 3.0) - rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.2.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.0.0) + rack-session (>= 2.0.0, < 3) tilt (~> 2.0) tilt (2.3.0) unicode-display_width (2.5.0) diff --git a/src-admintool/Gemfile.lock b/src-admintool/Gemfile.lock index 5a098220..3cbb4f28 100644 --- a/src-admintool/Gemfile.lock +++ b/src-admintool/Gemfile.lock @@ -13,21 +13,21 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.881.0) - aws-sdk-core (3.190.3) + aws-partitions (1.895.0) + aws-sdk-core (3.191.3) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.76.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-sdk-kms (1.77.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.142.0) - aws-sdk-core (~> 3, >= 3.189.0) + aws-sdk-s3 (1.143.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) - aws-sdk-ssm (1.162.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-sdk-ssm (1.165.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) aws-eventstream (~> 1, >= 1.0.2) @@ -35,16 +35,16 @@ GEM json (2.7.1) language_server-protocol (3.17.0.3) logger (1.6.0) - mysql2 (0.5.5) + mysql2 (0.5.6) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -52,11 +52,11 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) unicode-display_width (2.5.0) yaml (0.2.1) From 6e4a4964c7e1a6408226c296c18d4a3d7f3d0c4d Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Thu, 7 Mar 2024 17:22:42 -0800 Subject: [PATCH 03/24] gem updates --- cognito-lambda-nonvpc/Gemfile.lock | 18 +++++++++--------- mysql-ruby-lambda/Gemfile.lock | 20 ++++++++++---------- simulate-lambda-alb/Gemfile.lock | 25 ++++++++++++++----------- src-admintool/Gemfile.lock | 28 ++++++++++++++-------------- 4 files changed, 47 insertions(+), 44 deletions(-) diff --git a/cognito-lambda-nonvpc/Gemfile.lock b/cognito-lambda-nonvpc/Gemfile.lock index 1dce9193..35e246ec 100644 --- a/cognito-lambda-nonvpc/Gemfile.lock +++ b/cognito-lambda-nonvpc/Gemfile.lock @@ -3,11 +3,11 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.881.0) - aws-sdk-cognitoidentityprovider (1.85.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-partitions (1.895.0) + aws-sdk-cognitoidentityprovider (1.87.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.190.3) + aws-sdk-core (3.191.3) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) @@ -18,14 +18,14 @@ GEM json (2.7.1) language_server-protocol (3.17.0.3) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -33,11 +33,11 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) unicode-display_width (2.5.0) diff --git a/mysql-ruby-lambda/Gemfile.lock b/mysql-ruby-lambda/Gemfile.lock index c8caace9..f90526d8 100644 --- a/mysql-ruby-lambda/Gemfile.lock +++ b/mysql-ruby-lambda/Gemfile.lock @@ -13,14 +13,14 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.881.0) - aws-sdk-core (3.190.3) + aws-partitions (1.895.0) + aws-sdk-core (3.191.3) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-ssm (1.162.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-sdk-ssm (1.165.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) aws-eventstream (~> 1, >= 1.0.2) @@ -28,16 +28,16 @@ GEM json (2.7.1) language_server-protocol (3.17.0.3) logger (1.6.0) - mysql2 (0.5.5) + mysql2 (0.5.6) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -45,11 +45,11 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) unicode-display_width (2.5.0) yaml (0.2.1) diff --git a/simulate-lambda-alb/Gemfile.lock b/simulate-lambda-alb/Gemfile.lock index 1e616124..dd2135e5 100644 --- a/simulate-lambda-alb/Gemfile.lock +++ b/simulate-lambda-alb/Gemfile.lock @@ -9,18 +9,20 @@ GEM mustermann (3.0.0) ruby2_keywords (~> 0.0.1) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) - rack (2.2.8) - rack-protection (3.2.0) + rack (3.0.9.1) + rack-protection (4.0.0) base64 (>= 0.1.0) - rack (~> 2.2, >= 2.2.4) + rack (>= 3.0.0, < 4) + rack-session (2.0.0) + rack (>= 3.0.0) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -28,17 +30,18 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) - sinatra (3.2.0) + sinatra (4.0.0) mustermann (~> 3.0) - rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.2.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.0.0) + rack-session (>= 2.0.0, < 3) tilt (~> 2.0) tilt (2.3.0) unicode-display_width (2.5.0) diff --git a/src-admintool/Gemfile.lock b/src-admintool/Gemfile.lock index 5a098220..3cbb4f28 100644 --- a/src-admintool/Gemfile.lock +++ b/src-admintool/Gemfile.lock @@ -13,21 +13,21 @@ GEM specs: ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.881.0) - aws-sdk-core (3.190.3) + aws-partitions (1.895.0) + aws-sdk-core (3.191.3) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.76.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-sdk-kms (1.77.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.142.0) - aws-sdk-core (~> 3, >= 3.189.0) + aws-sdk-s3 (1.143.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) - aws-sdk-ssm (1.162.0) - aws-sdk-core (~> 3, >= 3.188.0) + aws-sdk-ssm (1.165.0) + aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) aws-eventstream (~> 1, >= 1.0.2) @@ -35,16 +35,16 @@ GEM json (2.7.1) language_server-protocol (3.17.0.3) logger (1.6.0) - mysql2 (0.5.5) + mysql2 (0.5.6) parallel (1.24.0) - parser (3.3.0.4) + parser (3.3.0.5) ast (~> 2.4.1) racc racc (1.7.3) rainbow (3.1.1) regexp_parser (2.9.0) rexml (3.2.6) - rubocop (1.60.1) + rubocop (1.62.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -52,11 +52,11 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.1) + parser (>= 3.3.0.4) ruby-progressbar (1.13.0) unicode-display_width (2.5.0) yaml (0.2.1) From 7cd02b40d0d1cb98cefd21e1488c9c9014dab1fe Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Fri, 8 Mar 2024 17:41:45 -0800 Subject: [PATCH 04/24] specialize zklists --- src-colladmin/actions/ingest_queue_action.rb | 2 +- src-colladmin/actions/zookeeper_action.rb | 120 ++++++++++++++----- 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/src-colladmin/actions/ingest_queue_action.rb b/src-colladmin/actions/ingest_queue_action.rb index 43cdbf2e..0fe595fd 100644 --- a/src-colladmin/actions/ingest_queue_action.rb +++ b/src-colladmin/actions/ingest_queue_action.rb @@ -3,7 +3,7 @@ require_relative 'zookeeper_action' # Collection Admin Task class - see config/actions.yml for description -class IngestQueueAction < ZookeeperAction +class IngestQueueAction < IngestQueueZookeeperAction def initialize(config, action, path, myparams) @batch = myparams.fetch('batch', '') @profile = myparams.fetch('profile', '') diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb index da85cc50..36a2a380 100644 --- a/src-colladmin/actions/zookeeper_action.rb +++ b/src-colladmin/actions/zookeeper_action.rb @@ -7,16 +7,16 @@ # Collection Admin Task class - see config/actions.yml for description class ZkList def initialize - @jobs = [] + @items = [] end - def add_job(job) - @jobs.push(job) + def add_item(item) + @items.push(item) end def to_table table = [] - js = @jobs.sort do |a, b| + js = @items.sort do |a, b| if a.status == b.status b.date <=> a.date else @@ -30,49 +30,109 @@ def to_table end end +class QueueItemReader + @@na = "NA" + def initialize(zk_action, id, payload) + @bytes = payload.nil? ? [] : payload.bytes + @is_json = zk_action.is_json + @status_vals = zk_action.status_vals + @queue_node = zk_action.zk_path + @id = id + end + + def status_byte + @bytes.empty? ? 0 : @bytes[0] + end + + def status + return @@na if status_byte > @status_vals.length + @status_vals[status_byte] + end + + def time + return nil if @bytes.length < 9 + # https://stackoverflow.com/a/68855488/3846548 + t = @bytes[1..8].inject(0) {|m, b| (m << 8) + b } + Time.at(t/1000) + end + + def payload_text + return "" if @bytes.length < 10 + @bytes[9..].pack('c*') + end + + def payload_object + if @is_json + json = JSON.parse(payload_text) + else + json = { + payload: payload_text + } + end + json['queueNode'] = @queue_node + json['id'] = @id + json['date'] = time + json['status'] = status + json + end + +end + + class ZookeeperAction < AdminAction - @@status_vals = ['Pending', 'Consumed', 'Deleted', 'Completed', 'Failed', 'Resolved'] def initialize(config, action, path, myparams, filters) super(config, action, path, myparams) @filters = {} - @path = '/ingest' @zk = ZK.new(get_zookeeper_conn) - @jobs = ZkList.new + @items = ZkList.new + end + + def zk_path + '/tbd' + end + + def status_vals + [] + end + + def is_json + false + end + + def items + @items end def perform_action - @zk.children(@path).each do |cp| + @zk.children(zk_path).each do |cp| puts cp - arr = @zk.get("#{@path}/#{cp}") - next if arr[0].nil? - data = arr[0].bytes - return if data.length < 9 - status = data[0] - # https://stackoverflow.com/a/68855488/3846548 - t = data[1..8].inject(0) {|m, b| (m << 8) + b } - time = Time.at(t/1000) - payload=data[9..].pack('c*') - begin - json = JSON.parse(payload) - json['queueNode'] = 'ingest' - json['id'] = cp - json['date'] = time - json['status'] = @@status_vals[status] - puts json.to_json - @jobs.add_job(QueueEntry.new(json)) - #puts JSON.pretty_generate(json) - rescue => exception - #puts exception - end + arr = @zk.get("#{zk_path}/#{cp}") + po = QueueItemReader.new(self, cp, arr[0]).payload_object + puts po.to_json + @items.add_item(QueueEntry.new(po)) end convert_json_to_table('') end def table_rows(_body) - @jobs.to_table + items.to_table end def get_zookeeper_conn @config.fetch('zookeeper', '').split(',').first end end + +class IngestQueueZookeeperAction < ZookeeperAction + def zk_path + '/ingest' + end + + def status_vals + ['Pending', 'Consumed', 'Deleted', 'Completed', 'Failed', 'Resolved'] + end + + def is_json + true + end +end \ No newline at end of file From 692c4dec2d4ef7a8610f47b86a79d07f1fd58373 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Mon, 11 Mar 2024 13:28:26 -0700 Subject: [PATCH 05/24] refine queue job filter --- src-colladmin/actions/ingest_queue_action.rb | 21 +++++++++++++++++++ .../actions/ingest_queue_profile_action.rb | 17 ++++++++++----- src-colladmin/actions/zookeeper_action.rb | 6 +++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src-colladmin/actions/ingest_queue_action.rb b/src-colladmin/actions/ingest_queue_action.rb index 0fe595fd..d760b468 100644 --- a/src-colladmin/actions/ingest_queue_action.rb +++ b/src-colladmin/actions/ingest_queue_action.rb @@ -17,6 +17,27 @@ def initialize(config, action, path, myparams) @batches = {} end + def filter_batch + @filter.fetch(:batch, '') + end + + def filter_profile + @filter.fetch(:profile, '') + end + + def filter_qstatus + @filter.fetch(:qstatus, '') + end + + def register_item(item) + return unless filter_batch.empty? || filter_batch == item.bid + return unless filter_profile.empty? || filter_profile == item.profile + return unless filter_qstatus.empty? || filter_qstatus == item.qstatus + super(item) + end + + + def get_title 'List Ingest Queues' end diff --git a/src-colladmin/actions/ingest_queue_profile_action.rb b/src-colladmin/actions/ingest_queue_profile_action.rb index e4a2c7b4..a67813b8 100644 --- a/src-colladmin/actions/ingest_queue_profile_action.rb +++ b/src-colladmin/actions/ingest_queue_profile_action.rb @@ -3,9 +3,10 @@ require_relative 'forward_to_ingest_action' # Collection Admin Task class - see config/actions.yml for description -class IngestQueueProfileCountAction < ForwardToIngestAction +class IngestQueueProfileCountAction < IngestQueueZookeeperAction def initialize(config, action, path, myparams) - super(config, action, path, myparams, 'admin/queues') + super(config, action, path, myparams, {}) + @profiles = {} end def get_title @@ -32,14 +33,20 @@ def table_types ] end + def register_item(item) + super(item) + k = "#{item.profile},#{item.qstatus}" + @profiles[k] = @profiles.fetch(k, []) + @profiles[k].append(item) + end + def table_rows(body) - queue_list = QueueList.new(get_ingest_server, body) arr = [] - queue_list.profiles.keys.sort.each do |k| + @profiles.keys.sort.each do |k| ka = k.split(',') qs = ka[1] profile = ka[0] - list = queue_list.profiles[k] + list = @profiles[k] count = list.length status = 'PASS' status = 'FAIL' if qs == 'Failed' diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb index 36a2a380..40780447 100644 --- a/src-colladmin/actions/zookeeper_action.rb +++ b/src-colladmin/actions/zookeeper_action.rb @@ -103,13 +103,17 @@ def items @items end + def register_item(item) + @items.add_item(item) + end + def perform_action @zk.children(zk_path).each do |cp| puts cp arr = @zk.get("#{zk_path}/#{cp}") po = QueueItemReader.new(self, cp, arr[0]).payload_object puts po.to_json - @items.add_item(QueueEntry.new(po)) + register_item(QueueEntry.new(po)) end convert_json_to_table('') end From 47a28938203479e40e144b7d04122ea557f46abf Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Wed, 13 Mar 2024 10:25:25 -0700 Subject: [PATCH 06/24] endpoint def --- src-colladmin/config/database.ssm.yml | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src-colladmin/config/database.ssm.yml b/src-colladmin/config/database.ssm.yml index daeda4d2..1f3104ad 100644 --- a/src-colladmin/config/database.ssm.yml +++ b/src-colladmin/config/database.ssm.yml @@ -72,6 +72,51 @@ default: +ALB: "{!ENV: MERRITT_PATH}/state.json" ldap: -ALB: "https://{!SSM: ldap/host}" + al2023prd: + match: '.*prd[0-9][0-9]$' + endpoints: + ingest: + state: "{!SSM: ingest/port}/state?t=json" + build-info: "{!SSM: ingest/port}/static/build.content.txt" + ALB: "{!SSM: colladmin/ingest-service}state?t=json" + store: + jsonstatus: "{!SSM: store/port}/jsonstatus" + state: "{!SSM: store/port}/state?t=json" + ping: "{!SSM: store/port}/ping?t=json" + hostname: "{!SSM: store/port}/hostname" + build-info: "{!SSM: store/port}/static/build.content.txt" + ALB: "{!SSM: colladmin/storage-service}/state?t=json" + access: + jsonstatus: "{!SSM: store/port}/jsonstatus" + state: "{!SSM: store/port}/state?t=json" + ping: "{!SSM: store/port}/ping?t=json" + hostname: "{!SSM: store/port}/hostname" + build-info: "{!SSM: store/port}/static/build.content.txt" + ALB: "{!SSM: colladmin/access-service}/state?t=json" + inventory: + state: "{!SSM: inventory/port}/state?t=json" + build-info: "{!SSM: inventory/port}/static/build.content.txt" + #start: "{!SSM: inventory/port}/service/start?t=json" + #stop: "{!SSM: inventory/port}/service/stop?t=json" + ALB: "{!SSM: colladmin/inventory-service}/state?t=json" + audit: + jsonstatus: "{!SSM: audit/port}/jsonstatus" + state: "{!SSM: audit/port}/state?t=json" + build-info: "{!SSM: audit/port}/static/build.content.txt" + #start: "{!SSM: audit/port}/service/start?t=json" + #stop: "{!SSM: audit/port}/service/stop?t=json" + replic: + jsonstatus: "{!SSM: replic/port}/jsonstatus" + state: "{!SSM: replic/port}/state?t=json" + build-info: "{!SSM: replic/port}/static/build.content.txt" + #start: "{!SSM: replic/port}/service/start?t=json" + #stop: "{!SSM: replic/port}/service/stop?t=json" + ui: + state: "/state.json" + audit_rep: "/state-audit-replic.json" + +ALB: "{!ENV: MERRITT_PATH}/state.json" + ldap: + -ALB: "https://{!SSM: ldap/host}" al2023: match: '.*stg[0-9][0-9]$' endpoints: @@ -124,6 +169,7 @@ default: state: "{!SSM: ingest/port}/state?t=json" build-info: "{!SSM: ingest/port}/static/build.content.txt" ALB: "{!SSM: colladmin/ingest-service}state?t=json" + -al2-config: "#default" store: jsonstatus: "{!SSM: store/port}/jsonstatus" state: "{!SSM: store/port}/state?t=json" @@ -131,6 +177,7 @@ default: hostname: "{!SSM: store/port}/hostname" build-info: "{!SSM: store/port}/static/build.content.txt" ALB: "{!SSM: colladmin/storage-service}/state?t=json" + -al2-config: "#default" access: jsonstatus: "{!SSM: store/port}/jsonstatus" state: "{!SSM: store/port}/state?t=json" @@ -138,30 +185,36 @@ default: hostname: "{!SSM: store/port}/hostname" build-info: "{!SSM: store/port}/static/build.content.txt" ALB: "{!SSM: colladmin/access-service}/state?t=json" + -al2-config: "#default" inventory: state: "{!SSM: inventory/port}/mrtinv/state?t=json" build-info: "{!SSM: inventory/port}/mrtinv/static/build.content.txt" start: "{!SSM: inventory/port}/mrtinv/service/start?t=json" stop: "{!SSM: inventory/port}/mrtinv/service/stop?t=json" ALB: "{!SSM: colladmin/inventory-service}/state?t=json" + -al2-config: "#default" audit: jsonstatus: "{!SSM: audit/port}/mrtaudit/jsonstatus" state: "{!SSM: audit/port}/mrtaudit/state?t=json" build-info: "{!SSM: audit/port}/mrtaudit/static/build.content.txt" start: "{!SSM: audit/port}/mrtaudit/service/start?t=json" stop: "{!SSM: audit/port}/mrtaudit/service/stop?t=json" + -al2-config: "#default" replic: jsonstatus: "{!SSM: replic/port}/mrtreplic/jsonstatus" state: "{!SSM: replic/port}/mrtreplic/state?t=json" build-info: "{!SSM: replic/port}/mrtreplic/static/build.content.txt" start: "{!SSM: replic/port}/mrtreplic/service/start?t=json" stop: "{!SSM: replic/port}/mrtreplic/service/stop?t=json" + -al2-config: "#default" ui: state: "/state.json" audit_rep: "/state-audit-replic.json" +ALB: "{!ENV: MERRITT_PATH}/state.json" + -al2-config: "#default" ldap: -ALB: "https://{!SSM: ldap/host}" + -al2-config: "#default" replic-service: "{!SSM: colladmin/replic-service}" storage-service: "{!SSM: colladmin/storage-service}" access-service: "{!SSM: colladmin/access-service}" From 6c391adf411f124835141810b7058783c0c79340 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Thu, 14 Mar 2024 15:43:47 -0700 Subject: [PATCH 07/24] update endpoints for audit/replic --- src-colladmin/config/database.ssm.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src-colladmin/config/database.ssm.yml b/src-colladmin/config/database.ssm.yml index 1f3104ad..44ed19c8 100644 --- a/src-colladmin/config/database.ssm.yml +++ b/src-colladmin/config/database.ssm.yml @@ -103,14 +103,14 @@ default: jsonstatus: "{!SSM: audit/port}/jsonstatus" state: "{!SSM: audit/port}/state?t=json" build-info: "{!SSM: audit/port}/static/build.content.txt" - #start: "{!SSM: audit/port}/service/start?t=json" - #stop: "{!SSM: audit/port}/service/stop?t=json" + start: "{!SSM: audit/port}/service/start?t=json" + stop: "{!SSM: audit/port}/service/stop?t=json" replic: jsonstatus: "{!SSM: replic/port}/jsonstatus" state: "{!SSM: replic/port}/state?t=json" build-info: "{!SSM: replic/port}/static/build.content.txt" - #start: "{!SSM: replic/port}/service/start?t=json" - #stop: "{!SSM: replic/port}/service/stop?t=json" + start: "{!SSM: replic/port}/service/start?t=json" + stop: "{!SSM: replic/port}/service/stop?t=json" ui: state: "/state.json" audit_rep: "/state-audit-replic.json" @@ -194,18 +194,8 @@ default: ALB: "{!SSM: colladmin/inventory-service}/state?t=json" -al2-config: "#default" audit: - jsonstatus: "{!SSM: audit/port}/mrtaudit/jsonstatus" - state: "{!SSM: audit/port}/mrtaudit/state?t=json" - build-info: "{!SSM: audit/port}/mrtaudit/static/build.content.txt" - start: "{!SSM: audit/port}/mrtaudit/service/start?t=json" - stop: "{!SSM: audit/port}/mrtaudit/service/stop?t=json" -al2-config: "#default" replic: - jsonstatus: "{!SSM: replic/port}/mrtreplic/jsonstatus" - state: "{!SSM: replic/port}/mrtreplic/state?t=json" - build-info: "{!SSM: replic/port}/mrtreplic/static/build.content.txt" - start: "{!SSM: replic/port}/mrtreplic/service/start?t=json" - stop: "{!SSM: replic/port}/mrtreplic/service/stop?t=json" -al2-config: "#default" ui: state: "/state.json" From c742c844257d043735167e422186d7111d7218e7 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Fri, 15 Mar 2024 12:24:11 -0700 Subject: [PATCH 08/24] list all uc3 boxes --- src-colladmin/actions/tag_uc3_action.rb | 187 ++++++++++++++++++++++++ src-colladmin/config/actions.yml | 13 ++ src-colladmin/lambda_function.rb | 1 + src-common/template/navmenu.html | 1 + 4 files changed, 202 insertions(+) create mode 100644 src-colladmin/actions/tag_uc3_action.rb diff --git a/src-colladmin/actions/tag_uc3_action.rb b/src-colladmin/actions/tag_uc3_action.rb new file mode 100644 index 00000000..106071e0 --- /dev/null +++ b/src-colladmin/actions/tag_uc3_action.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +require_relative 'action' +require 'aws-sdk-ec2' +require 'aws-sdk-ssm' + +# represents information about an EC2 instance +class Ec2InfoUc3 + + # See https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/EC2/Client.html#describe_instances-instance_method + def initialize(config, inst) + @config = config + @state = inst.state.name + @type = inst.instance_type + @publicip = inst.public_ip_address + @az = inst.placement.availability_zone + inst.tags.each do |tag| + @name = tag.value if tag.key == 'Name' + @subservice = tag.value if tag.key == 'Subservice' + end + end + + attr_reader :name + + def self.table_headers + [ + 'Name', + 'Subservice', + 'Type', + 'SERVER State', + 'IP', + 'AZ' + ] + end + + def self.table_types + [ + '', + '', + '', + '', + '', + '' + ] + end + + def table_row(action) + [ + @name, + @subservice, + @type, + @state, + @publicip, + @az + ] + end +end + +# Collection Admin Task class - see config/actions.yml for description +class TagUc3Action < AdminAction + def initialize(config, action, path, myparams) + super(config, action, path, myparams) + region = ENV['AWS_REGION'] || 'us-west-2' + @ec2 = Aws::EC2::Client.new( + region: region + ) + @ssm = Aws::SSM::Client.new( + region: region + ) + @title = 'Merritt EC2 Instances' + @instances = {} + @name = myparams.fetch('name', '') + @label = myparams.fetch('label', '') + @list_servers = @name.empty? + + data = @ec2.describe_instances({ + filters: [ + { + name: 'tag:Program', + values: [LambdaBase.tag_program] + }, + { + name: 'tag:Environment', + values: [LambdaBase.tag_environment] + } + ] + }) + + data.reservations.each do |res| + res.instances.each do |inst| + ec2 = Ec2InfoUc3.new(config, inst) + @instances[ec2.name] = ec2 + end + end + end + + def get_ssm(key) + @ssm.get_parameter({ name: "#{LambdaBase.ssm_root_path}#{key}" })[:parameter][:value] + rescue StandardError + '' + end + + def get_title + @title + end + + def table_headers + Ec2InfoUc3.table_headers + end + + def table_types + Ec2InfoUc3.table_types + end + + def endpoint_call + return unless @instances[@name] + + ec2 = @instances[@name] + url = ec2.urls.fetch(@label, '') + return 'No Url found' if url.empty? + + cli = HTTPClient.new + cli.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE + if @label == 'stop' || @label == 'start' + puts "POST #{url}" + resp = cli.post(url) + else + puts "GET #{url}" + resp = cli.get(url, follow_redirect: true) + end + ret = resp.body + return ret if @label == 'build-info' + + if resp.status == 200 + begin + JSON.parse(resp.body) + rescue StandardError + ret = { body: resp.dump }.to_json + end + else + ret = { + message: "Status #{resp.status} for #{url}" + }.to_json + end + ret + end + + def perform_action + return endpoint_call unless @list_servers + + evaluate_status(table_types, get_table_rows) + { + format: 'report', + title: get_title_with_pagination, + breadcrumb: get_breadcrumb, + headers: table_headers, + types: table_types, + data: get_table_rows, + filter_col: nil, + group_col: nil, + show_grand_total: false, + merritt_path: @merritt_path, + alternative_queries: get_alternative_queries_with_pagination, + iterate: false, + saveable: is_saveable?, + report_path: report_path, + chart: nil, + description: get_description + }.to_json + end + + def get_table_rows + rows = [] + @instances.keys.sort.each do |k| + rows.append(@instances[k].table_row(self)) + end + rows + end + + def has_table + @list_servers + end + + def get_alternative_queries + [] + end +end diff --git a/src-colladmin/config/actions.yml b/src-colladmin/config/actions.yml index feb9e829..4d6ea7fb 100644 --- a/src-colladmin/config/actions.yml +++ b/src-colladmin/config/actions.yml @@ -347,6 +347,19 @@ instances: - instances documentation: | Aws: Tag Query +uc3-instances: + link-title: List and Describe UC3 EC2 Servers + breadcrumb: bp_internals + class: TagUc3Action + category: Dev Ops + sensitivity: readonly + testing: automated + description: | + Generate a report describing the EC2 servers used by UC3. + report-datatypes: + - instances + documentation: | + Aws: Tag Query ssm-describe: link-title: List and Describe Merritt SSM Variables breadcrumb: bp_internals diff --git a/src-colladmin/lambda_function.rb b/src-colladmin/lambda_function.rb index ede93ec2..350f306b 100644 --- a/src-colladmin/lambda_function.rb +++ b/src-colladmin/lambda_function.rb @@ -25,6 +25,7 @@ require_relative 'actions/cognito_action' require_relative 'actions/storage_action' require_relative 'actions/tag_action' +require_relative 'actions/tag_uc3_action' require_relative 'actions/ssm_describe_action' require_relative 'actions/opensearch_describe_action' require_relative 'actions/replication_action' diff --git a/src-common/template/navmenu.html b/src-common/template/navmenu.html index 610b8f87..52e39360 100644 --- a/src-common/template/navmenu.html +++ b/src-common/template/navmenu.html @@ -105,6 +105,7 @@ Merritt Admin Objects Merritt User Accounts Merritt Servers + UC3 Servers SSM Parameter Registry OpenSearch Field Registry From a2479ad7866cdcd3ca354abd5862ea9e2b8c40f5 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Fri, 15 Mar 2024 12:30:57 -0700 Subject: [PATCH 09/24] tag uc3, incl service --- src-colladmin/actions/tag_uc3_action.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src-colladmin/actions/tag_uc3_action.rb b/src-colladmin/actions/tag_uc3_action.rb index 106071e0..f1f3ee20 100644 --- a/src-colladmin/actions/tag_uc3_action.rb +++ b/src-colladmin/actions/tag_uc3_action.rb @@ -6,7 +6,6 @@ # represents information about an EC2 instance class Ec2InfoUc3 - # See https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/EC2/Client.html#describe_instances-instance_method def initialize(config, inst) @config = config @@ -16,6 +15,7 @@ def initialize(config, inst) @az = inst.placement.availability_zone inst.tags.each do |tag| @name = tag.value if tag.key == 'Name' + @service = tag.value if tag.key == 'Service' @subservice = tag.value if tag.key == 'Subservice' end end @@ -25,6 +25,7 @@ def initialize(config, inst) def self.table_headers [ 'Name', + 'Service', 'Subservice', 'Type', 'SERVER State', @@ -40,13 +41,15 @@ def self.table_types '', '', '', + '', '' ] end - def table_row(action) + def table_row(_action) [ @name, + @service, @subservice, @type, @state, From 664c85dd6f7a90cc1fd0980e929daa9d60719941 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Wed, 20 Mar 2024 10:51:04 -0700 Subject: [PATCH 10/24] show current version --- src-admintool/queries/objectid_files_query.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src-admintool/queries/objectid_files_query.rb b/src-admintool/queries/objectid_files_query.rb index 356c9048..d467da37 100644 --- a/src-admintool/queries/objectid_files_query.rb +++ b/src-admintool/queries/objectid_files_query.rb @@ -16,6 +16,17 @@ def get_sql select o.ark, v.number, + ( + select max(vv.number) + from inv.inv_files ff + inner join inv.inv_versions vv + on ff.inv_version_id = vv.id + where + ff.inv_object_id = o.id + and exists ( + select 1 where ff.pathname=f.pathname + ) + ) as maxv, f.source, binary f.pathname, f.full_size, @@ -69,11 +80,11 @@ def get_params end def get_headers(_results) - ['Ark', 'Version', 'Source', 'Path', 'File Size', 'Created', 'Nodes', 'Unverified'] + ['Ark', 'Version', 'MaxVer', 'Source', 'Path', 'File Size', 'Created', 'Nodes', 'Unverified'] end def get_types(_results) - ['ark', '', '', 'name', 'bytes', 'datetime', '', ''] + ['ark', '', '', '', 'name', 'bytes', 'datetime', '', ''] end def get_alternative_queries From cb83d6e25234e82e19458a6ff8a924c31ddbe1c6 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Wed, 20 Mar 2024 14:09:26 -0700 Subject: [PATCH 11/24] access store srv upgrade --- src-admintool/queries/objectid_files_query.rb | 12 ++++++------ src-colladmin/config/database.ssm.yml | 12 ------------ 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/src-admintool/queries/objectid_files_query.rb b/src-admintool/queries/objectid_files_query.rb index d467da37..a39b2d5c 100644 --- a/src-admintool/queries/objectid_files_query.rb +++ b/src-admintool/queries/objectid_files_query.rb @@ -17,12 +17,12 @@ def get_sql o.ark, v.number, ( - select max(vv.number) - from inv.inv_files ff - inner join inv.inv_versions vv - on ff.inv_version_id = vv.id - where - ff.inv_object_id = o.id + select max(vv.number) + from inv.inv_files ff + inner join inv.inv_versions vv + on ff.inv_version_id = vv.id + where + ff.inv_object_id = o.id and exists ( select 1 where ff.pathname=f.pathname ) diff --git a/src-colladmin/config/database.ssm.yml b/src-colladmin/config/database.ssm.yml index 44ed19c8..217951e2 100644 --- a/src-colladmin/config/database.ssm.yml +++ b/src-colladmin/config/database.ssm.yml @@ -171,20 +171,8 @@ default: ALB: "{!SSM: colladmin/ingest-service}state?t=json" -al2-config: "#default" store: - jsonstatus: "{!SSM: store/port}/jsonstatus" - state: "{!SSM: store/port}/state?t=json" - ping: "{!SSM: store/port}/ping?t=json" - hostname: "{!SSM: store/port}/hostname" - build-info: "{!SSM: store/port}/static/build.content.txt" - ALB: "{!SSM: colladmin/storage-service}/state?t=json" -al2-config: "#default" access: - jsonstatus: "{!SSM: store/port}/jsonstatus" - state: "{!SSM: store/port}/state?t=json" - ping: "{!SSM: store/port}/ping?t=json" - hostname: "{!SSM: store/port}/hostname" - build-info: "{!SSM: store/port}/static/build.content.txt" - ALB: "{!SSM: colladmin/access-service}/state?t=json" -al2-config: "#default" inventory: state: "{!SSM: inventory/port}/mrtinv/state?t=json" From a9ef92b97583b22df96874cb30f3509059799e85 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Thu, 21 Mar 2024 13:23:18 -0700 Subject: [PATCH 12/24] ui server migration --- src-colladmin/config/database.ssm.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src-colladmin/config/database.ssm.yml b/src-colladmin/config/database.ssm.yml index 217951e2..1d9ae3e7 100644 --- a/src-colladmin/config/database.ssm.yml +++ b/src-colladmin/config/database.ssm.yml @@ -67,9 +67,7 @@ default: match: '.*x2-stg$' endpoints: ui: - state: "/state.json" - audit_rep: "/state-audit-replic.json" - +ALB: "{!ENV: MERRITT_PATH}/state.json" + -al2-config: "#default" ldap: -ALB: "https://{!SSM: ldap/host}" al2023prd: @@ -112,8 +110,8 @@ default: start: "{!SSM: replic/port}/service/start?t=json" stop: "{!SSM: replic/port}/service/stop?t=json" ui: - state: "/state.json" - audit_rep: "/state-audit-replic.json" + state: "{!SSM: ui/port}/state.json" + audit_rep: "{!SSM: ui/port}/state-audit-replic.json" +ALB: "{!ENV: MERRITT_PATH}/state.json" ldap: -ALB: "https://{!SSM: ldap/host}" @@ -157,8 +155,8 @@ default: start: "{!SSM: replic/port}/service/start?t=json" stop: "{!SSM: replic/port}/service/stop?t=json" ui: - state: "/state.json" - audit_rep: "/state-audit-replic.json" + state: "{!SSM: ui/port}/state.json" + audit_rep: "{!SSM: ui/port}/state-audit-replic.json" +ALB: "{!ENV: MERRITT_PATH}/state.json" ldap: -ALB: "https://{!SSM: ldap/host}" From c0a9d853f836d92e5e5da25c07875e25604356c7 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Tue, 2 Apr 2024 12:22:02 -0700 Subject: [PATCH 13/24] ruby base image update --- cognito-lambda-nonvpc/Dockerfile | 6 ++++-- mysql-ruby-lambda/Dockerfile | 4 ++-- simulate-lambda-alb/Dockerfile | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cognito-lambda-nonvpc/Dockerfile b/cognito-lambda-nonvpc/Dockerfile index c60d55db..43297faf 100644 --- a/cognito-lambda-nonvpc/Dockerfile +++ b/cognito-lambda-nonvpc/Dockerfile @@ -5,10 +5,12 @@ # This image is based on an architecture that matches the Lambda Ruby runtime # docker build -t ${ECR_REGISTRY}/cognito-lambda-nonvpc . -FROM public.ecr.aws/lambda/ruby:3 +FROM public.ecr.aws/lambda/ruby:3.2 + +RUN yum -y update && yum -y upgrade -RUN yum -y update RUN yum -y install gcc make git + RUN gem update bundler # Start with a Gemfile containing only MySQL diff --git a/mysql-ruby-lambda/Dockerfile b/mysql-ruby-lambda/Dockerfile index c0a4ca11..254bfae3 100644 --- a/mysql-ruby-lambda/Dockerfile +++ b/mysql-ruby-lambda/Dockerfile @@ -5,9 +5,9 @@ # This image is based on an architecture that matches the Lambda Ruby runtime # docker build -t ${ECR_REGISTRY}/mysql-ruby-lambda . -FROM public.ecr.aws/lambda/ruby:3 +FROM public.ecr.aws/lambda/ruby:3.2 -RUN yum -y update +RUN yum -y update && yum -y upgrade # Install an os-specific MySQL installation. # gcc and make are required to build mysql. diff --git a/simulate-lambda-alb/Dockerfile b/simulate-lambda-alb/Dockerfile index 07fff047..d8973b4e 100644 --- a/simulate-lambda-alb/Dockerfile +++ b/simulate-lambda-alb/Dockerfile @@ -7,9 +7,9 @@ # This image is not actively used # docker build -t ${ECR_REGISTRY}/simulate-lambda-alb . -FROM ruby:3 +FROM ruby:3.2 -RUN apt-get update -y && apt-get -y upgrade +RUN yum -y update && yum -y upgrade RUN gem install bundler From 6a2b2d5f278fa9a3d773048423a441a666003013 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Tue, 2 Apr 2024 13:35:46 -0700 Subject: [PATCH 14/24] fix ruby ver --- simulate-lambda-alb/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulate-lambda-alb/Dockerfile b/simulate-lambda-alb/Dockerfile index d8973b4e..3e9fb201 100644 --- a/simulate-lambda-alb/Dockerfile +++ b/simulate-lambda-alb/Dockerfile @@ -7,9 +7,9 @@ # This image is not actively used # docker build -t ${ECR_REGISTRY}/simulate-lambda-alb . -FROM ruby:3.2 +FROM ruby:3 -RUN yum -y update && yum -y upgrade +RUN apt-get -y update && apt-get -y upgrade RUN gem install bundler From 31cff577b1789d911c642048d43a56f6bbf840fc Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Fri, 5 Apr 2024 16:54:02 -0700 Subject: [PATCH 15/24] endpoint updates --- src-colladmin/config/database.ssm.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src-colladmin/config/database.ssm.yml b/src-colladmin/config/database.ssm.yml index 1d9ae3e7..19f54f7d 100644 --- a/src-colladmin/config/database.ssm.yml +++ b/src-colladmin/config/database.ssm.yml @@ -94,8 +94,8 @@ default: inventory: state: "{!SSM: inventory/port}/state?t=json" build-info: "{!SSM: inventory/port}/static/build.content.txt" - #start: "{!SSM: inventory/port}/service/start?t=json" - #stop: "{!SSM: inventory/port}/service/stop?t=json" + start: "{!SSM: inventory/port}/service/start?t=json" + stop: "{!SSM: inventory/port}/service/stop?t=json" ALB: "{!SSM: colladmin/inventory-service}/state?t=json" audit: jsonstatus: "{!SSM: audit/port}/jsonstatus" @@ -173,20 +173,12 @@ default: access: -al2-config: "#default" inventory: - state: "{!SSM: inventory/port}/mrtinv/state?t=json" - build-info: "{!SSM: inventory/port}/mrtinv/static/build.content.txt" - start: "{!SSM: inventory/port}/mrtinv/service/start?t=json" - stop: "{!SSM: inventory/port}/mrtinv/service/stop?t=json" - ALB: "{!SSM: colladmin/inventory-service}/state?t=json" -al2-config: "#default" audit: -al2-config: "#default" replic: -al2-config: "#default" ui: - state: "/state.json" - audit_rep: "/state-audit-replic.json" - +ALB: "{!ENV: MERRITT_PATH}/state.json" -al2-config: "#default" ldap: -ALB: "https://{!SSM: ldap/host}" From def2952227b75573b4a8a84d693d465665dcaf06 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Mon, 8 Apr 2024 16:22:21 -0700 Subject: [PATCH 16/24] prune report --- src-admintool/config/reports.yml | 7 ++ src-admintool/queries/prune_files_query.rb | 107 +++++++++++++++++++++ src-admintool/web/index.html | 17 +++- 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src-admintool/queries/prune_files_query.rb diff --git a/src-admintool/config/reports.yml b/src-admintool/config/reports.yml index 754ac549..287ac166 100644 --- a/src-admintool/config/reports.yml +++ b/src-admintool/config/reports.yml @@ -120,6 +120,13 @@ producer_files: category: files description: | Producer Files for a Mnemonic. +prune_candidates: + link-title: Prune Candidate Files for a Mnemonic + breadcrumb: bp_content_projects + class: PruneCandidateFilesQuery + category: files + description: | + Prune Candidate Files for a Mnemonic. daily-build-s3-list: link-title: Daily Build Links breadcrumb: bp_content_projects diff --git a/src-admintool/queries/prune_files_query.rb b/src-admintool/queries/prune_files_query.rb new file mode 100644 index 00000000..5d65a7da --- /dev/null +++ b/src-admintool/queries/prune_files_query.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +# Query class - see config/reports.yml for description +class PruneCandidateFilesQuery < AdminQuery + def initialize(query_factory, path, myparams) + super(query_factory, path, myparams) + @mnemonic = get_param('mnemonic', 'merritt_demo') + end + + def get_title + "Prune Candidate Files for Mnemonic #{@mnemonic}" + end + + def get_sql + %{ + select distinct + o.ark, + o.version_number, + substring_index(f.pathname, '?', 1) norm, + max(v.number), + count(distinct f.pathname), + count(distinct digest_value) + from + inv.inv_objects o + inner join + inv.inv_files f + on + f.inv_object_id = o.id + inner join + inv.inv_versions v + on + f.inv_version_id = v.id + where exists ( + select 1 + from + inv.inv_collections_inv_objects icio + inner join + inv.inv_collections c + on + c.id = icio.inv_collection_id + and + c.mnemonic= ? + where + icio.inv_object_id=o.id + ) + and + f.pathname like 'producer/%' + group by + ark,version_number,norm + having + count(distinct f.pathname) > 1 + and + count(distinct digest_value) > 1 + union + select distinct + o.ark, + o.version_number, + substring_index(f.pathname, '?', 1) norm, + max(v.number), + count(distinct f.pathname), + count(distinct digest_value) + from + inv.inv_objects o + inner join + inv.inv_files f + on + f.inv_object_id = o.id + inner join + inv.inv_versions v + on + f.inv_version_id = v.id + where exists ( + select 1 + from + inv.inv_collections_inv_objects icio + inner join + inv.inv_collections c + on + c.id = icio.inv_collection_id + and + c.mnemonic= ? + where + icio.inv_object_id=o.id + ) + and + f.pathname like 'producer/%' + group by + ark,version_number,norm + having + count(distinct f.pathname) = 1 + and + max(v.number) < o.version_number; + } + end + + def get_params + [@mnemonic, @mnemonic] + end + + def get_headers(_results) + ['Ark', 'Version', 'Path', 'Max Ver', 'Path Count', 'Digest Count'] + end + + def get_types(_results) + ['ark', '', 'name', '', '', ''] + end +end diff --git a/src-admintool/web/index.html b/src-admintool/web/index.html index 636fdb33..acaf2700 100644 --- a/src-admintool/web/index.html +++ b/src-admintool/web/index.html @@ -569,7 +569,22 @@

Inventory Comparison

- +

Prune Candidate Files

+
    +
  • +
    + Prune Candidate Files for a Mnemonic: + + + +
    +
+ +
From 78409116abc42b98522bdb1b9d6d3a577ce43d03 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Mon, 8 Apr 2024 16:24:28 -0700 Subject: [PATCH 17/24] rubocop --- src-admintool/queries/prune_files_query.rb | 76 +++++++++++----------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src-admintool/queries/prune_files_query.rb b/src-admintool/queries/prune_files_query.rb index 5d65a7da..2fc8a523 100644 --- a/src-admintool/queries/prune_files_query.rb +++ b/src-admintool/queries/prune_files_query.rb @@ -14,80 +14,80 @@ def get_title def get_sql %{ select distinct - o.ark, - o.version_number, + o.ark, + o.version_number, substring_index(f.pathname, '?', 1) norm, max(v.number), count(distinct f.pathname), count(distinct digest_value) - from - inv.inv_objects o + from + inv.inv_objects o inner join - inv.inv_files f - on - f.inv_object_id = o.id + inv.inv_files f + on + f.inv_object_id = o.id inner join inv.inv_versions v - on + on f.inv_version_id = v.id where exists ( - select 1 - from + select 1 + from inv.inv_collections_inv_objects icio - inner join - inv.inv_collections c - on - c.id = icio.inv_collection_id - and + inner join + inv.inv_collections c + on + c.id = icio.inv_collection_id + and c.mnemonic= ? where icio.inv_object_id=o.id ) - and + and f.pathname like 'producer/%' - group by + group by ark,version_number,norm - having + having count(distinct f.pathname) > 1 - and + and count(distinct digest_value) > 1 union select distinct - o.ark, - o.version_number, + o.ark, + o.version_number, substring_index(f.pathname, '?', 1) norm, max(v.number), count(distinct f.pathname), count(distinct digest_value) - from - inv.inv_objects o + from + inv.inv_objects o inner join - inv.inv_files f - on - f.inv_object_id = o.id + inv.inv_files f + on + f.inv_object_id = o.id inner join inv.inv_versions v - on + on f.inv_version_id = v.id where exists ( - select 1 - from + select 1 + from inv.inv_collections_inv_objects icio - inner join - inv.inv_collections c - on - c.id = icio.inv_collection_id - and + inner join + inv.inv_collections c + on + c.id = icio.inv_collection_id + and c.mnemonic= ? where icio.inv_object_id=o.id ) - and + and f.pathname like 'producer/%' - group by + group by ark,version_number,norm - having - count(distinct f.pathname) = 1 + having + count(distinct f.pathname) = 1 and max(v.number) < o.version_number; } From df48fba75a118180a0e400ac0ed91185ccae6ce8 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Mon, 8 Apr 2024 16:37:41 -0700 Subject: [PATCH 18/24] rubocop map logic --- .../actions/ingest_batch_folders_action.rb | 6 +--- .../actions/ingest_job_manifest_action.rb | 6 ++-- .../actions/ingest_job_metadata_action.rb | 6 +--- .../actions/opensearch_describe_action.rb | 6 ++-- src-colladmin/actions/replication_action.rb | 5 ++-- src-colladmin/actions/tag_action.rb | 6 ++-- src-colladmin/actions/tag_uc3_action.rb | 6 ++-- src-colladmin/lambda_function.rb | 6 ++-- src-colladmin/lib/acc_queue.rb | 6 ++-- src-colladmin/lib/admin_objects.rb | 8 +----- src-colladmin/lib/inv_queue.rb | 6 ++-- src-colladmin/lib/lock.rb | 6 ++-- src-colladmin/lib/profile.rb | 28 ++++++------------- src-colladmin/lib/queue.rb | 6 ++-- src-colladmin/lib/storage_nodes.rb | 18 +++++------- 15 files changed, 39 insertions(+), 86 deletions(-) diff --git a/src-colladmin/actions/ingest_batch_folders_action.rb b/src-colladmin/actions/ingest_batch_folders_action.rb index cc433595..49eaba72 100644 --- a/src-colladmin/actions/ingest_batch_folders_action.rb +++ b/src-colladmin/actions/ingest_batch_folders_action.rb @@ -163,7 +163,6 @@ def empty? end def to_table - table = [] bfs = @batch_folders.sort do |a, b| if a.status == b.status b.dtime <=> a.dtime @@ -171,10 +170,7 @@ def to_table AdminTask.status_sort_val(a.status) <=> AdminTask.status_sort_val(b.status) end end - bfs.each do |bf| - table.append(bf.table_row) - end - table + bfs.map(&:table_row) end def apply_queue_list(queue_list) diff --git a/src-colladmin/actions/ingest_job_manifest_action.rb b/src-colladmin/actions/ingest_job_manifest_action.rb index 61b5fd67..273d9df6 100644 --- a/src-colladmin/actions/ingest_job_manifest_action.rb +++ b/src-colladmin/actions/ingest_job_manifest_action.rb @@ -80,11 +80,9 @@ def initialize(json) end def self.table_headers - arr = [] - JobManifestEntry.placeholder.get_property_list.each do |sym| - arr.append(JobManifestEntry.placeholder.get_label(sym)) + JobManifestEntry.placeholder.get_property_list.map do |sym| + JobManifestEntry.placeholder.get_label(sym) end - arr end def self.table_types diff --git a/src-colladmin/actions/ingest_job_metadata_action.rb b/src-colladmin/actions/ingest_job_metadata_action.rb index 12bc64f8..486b4bf0 100644 --- a/src-colladmin/actions/ingest_job_metadata_action.rb +++ b/src-colladmin/actions/ingest_job_metadata_action.rb @@ -92,10 +92,6 @@ def initialize(body) end def to_table - rows = [] - @metadata.each do |jmr| - rows.append(jmr.to_table_row) - end - rows + @metadata.map(&:to_table_row) end end diff --git a/src-colladmin/actions/opensearch_describe_action.rb b/src-colladmin/actions/opensearch_describe_action.rb index be2e9a4c..ece5d714 100644 --- a/src-colladmin/actions/opensearch_describe_action.rb +++ b/src-colladmin/actions/opensearch_describe_action.rb @@ -71,7 +71,6 @@ def process_key(k, defstat) end def table_rows(_body) - rows = [] oskeys = {} @fields.each_key do |k| @@ -82,10 +81,9 @@ def table_rows(_body) oskeys[k] = process_key(k, 'WARN') end - oskeys.keys.sort.each do |k| - rows.append(oskeys[k]) + oskeys.keys.sort.map do |k| + oskeys[k] end - rows end def has_table diff --git a/src-colladmin/actions/replication_action.rb b/src-colladmin/actions/replication_action.rb index 15b34c9c..bc34a071 100644 --- a/src-colladmin/actions/replication_action.rb +++ b/src-colladmin/actions/replication_action.rb @@ -38,9 +38,8 @@ def maintidlist_params(status) end def maintidlist_placeholders - placeholders = [] - @myparams.fetch('maintidlist', '').split(',').each do |_id| - placeholders.append('?') + placeholders = @myparams.fetch('maintidlist', '').split(',').map do |_id| + '?' end placeholders.join(',') end diff --git a/src-colladmin/actions/tag_action.rb b/src-colladmin/actions/tag_action.rb index 9bd67f58..5534f0b5 100644 --- a/src-colladmin/actions/tag_action.rb +++ b/src-colladmin/actions/tag_action.rb @@ -236,11 +236,9 @@ def perform_action end def get_table_rows - rows = [] - @instances.keys.sort.each do |k| - rows.append(@instances[k].table_row(self)) + @instances.keys.sort.map do |k| + @instances[k].table_row(self) end - rows end def has_table diff --git a/src-colladmin/actions/tag_uc3_action.rb b/src-colladmin/actions/tag_uc3_action.rb index f1f3ee20..9d08dd83 100644 --- a/src-colladmin/actions/tag_uc3_action.rb +++ b/src-colladmin/actions/tag_uc3_action.rb @@ -173,11 +173,9 @@ def perform_action end def get_table_rows - rows = [] - @instances.keys.sort.each do |k| - rows.append(@instances[k].table_row(self)) + @instances.keys.sort.map do |k| + @instances[k].table_row(self) end - rows end def has_table diff --git a/src-colladmin/lambda_function.rb b/src-colladmin/lambda_function.rb index 350f306b..94dc603c 100644 --- a/src-colladmin/lambda_function.rb +++ b/src-colladmin/lambda_function.rb @@ -381,11 +381,9 @@ def get_actions_map actmap[path] = [] unless actmap.key?(path) actmap[path].push(act) end - maplist = [] - actmap.keys.sort.each do |p| - maplist.push({ path: p, actions: actmap[p] }) + actmap.keys.sort.map do |p| + { path: p, actions: actmap[p] } end - maplist end end end diff --git a/src-colladmin/lib/acc_queue.rb b/src-colladmin/lib/acc_queue.rb index 94517230..586793df 100644 --- a/src-colladmin/lib/acc_queue.rb +++ b/src-colladmin/lib/acc_queue.rb @@ -80,11 +80,9 @@ def initialize(json) end def self.table_headers - arr = [] - AccQueueEntry.placeholder.get_property_list.each do |sym| - arr.append(AccQueueEntry.placeholder.get_label(sym)) + AccQueueEntry.placeholder.get_property_list.map do |sym| + AccQueueEntry.placeholder.get_label(sym) end - arr end def self.table_types diff --git a/src-colladmin/lib/admin_objects.rb b/src-colladmin/lib/admin_objects.rb index 0e56fd7b..196a3798 100644 --- a/src-colladmin/lib/admin_objects.rb +++ b/src-colladmin/lib/admin_objects.rb @@ -261,13 +261,7 @@ def self.table_types end def table_rows - rows = [] - @profiles.each do |p| - rows.push( - p.to_table_row - ) - end - rows + @profiles.map(&:to_table_row) end attr_reader :profiles diff --git a/src-colladmin/lib/inv_queue.rb b/src-colladmin/lib/inv_queue.rb index 1670aee0..97e36901 100644 --- a/src-colladmin/lib/inv_queue.rb +++ b/src-colladmin/lib/inv_queue.rb @@ -69,11 +69,9 @@ def initialize(json) end def self.table_headers - arr = [] - InvQueueEntry.placeholder.get_property_list.each do |sym| - arr.append(InvQueueEntry.placeholder.get_label(sym)) + InvQueueEntry.placeholder.get_property_list.map do |sym| + InvQueueEntry.placeholder.get_label(sym) end - arr end def self.table_types diff --git a/src-colladmin/lib/lock.rb b/src-colladmin/lib/lock.rb index 54288248..2f47cfa5 100644 --- a/src-colladmin/lib/lock.rb +++ b/src-colladmin/lib/lock.rb @@ -25,11 +25,9 @@ def initialize(json) end def self.table_headers - arr = [] - LockEntry.placeholder.get_property_list.each do |sym| - arr.append(LockEntry.placeholder.get_label(sym)) + LockEntry.placeholder.get_property_list.map do |sym| + LockEntry.placeholder.get_label(sym) end - arr end def self.table_types diff --git a/src-colladmin/lib/profile.rb b/src-colladmin/lib/profile.rb index eb0b0c73..7aec8450 100644 --- a/src-colladmin/lib/profile.rb +++ b/src-colladmin/lib/profile.rb @@ -28,12 +28,7 @@ def initialize(body, collections) end def table_rows - rows = [] - @profiles.each do |p| - # next if p.is_template? - rows.append(p.summary_values) - end - rows + @profiles.map(&:summary_values) end def notification_map @@ -41,14 +36,12 @@ def notification_map @profiles.each do |p| map[p.get_value(:context)] = p.get_value(:contactsEmail).join(',') end - omap = [] - map.keys.sort.each do |k| - omap.push({ + map.keys.sort.map do |k| + { mnemonic: k, contacts: map[k] - }) + } end - omap end def recent_profiles @@ -60,14 +53,12 @@ def recent_profiles context: p.get_value(:context) } end - omap = [] - map.keys.sort.reverse.each do |k| - omap.push({ + map.keys.sort.reverse.map do |k| + { ark: map[k][:ark], name: "#{map[k][:context]}: #{map[k][:name]}" - }) + } end - omap end attr_reader :profiles @@ -330,9 +321,8 @@ def summary_types end def summary_headers - arr = [] - summary_symbols.each do |sym| - arr.append(get_label(sym)) + arr = summary_symbols.map do |sym| + get_label(sym) end arr.append('Score') arr.append('Collection') diff --git a/src-colladmin/lib/queue.rb b/src-colladmin/lib/queue.rb index ac55a0b3..e6e73fe8 100644 --- a/src-colladmin/lib/queue.rb +++ b/src-colladmin/lib/queue.rb @@ -102,11 +102,9 @@ def check_filter(filter) end def self.table_headers - arr = [] - QueueEntry.placeholder.get_property_list.each do |sym| - arr.append(QueueEntry.placeholder.get_label(sym)) + QueueEntry.placeholder.get_property_list.map do |sym| + QueueEntry.placeholder.get_label(sym) end - arr end def self.table_types diff --git a/src-colladmin/lib/storage_nodes.rb b/src-colladmin/lib/storage_nodes.rb index 96ed7a57..fa13e886 100644 --- a/src-colladmin/lib/storage_nodes.rb +++ b/src-colladmin/lib/storage_nodes.rb @@ -841,18 +841,16 @@ def owner_clause end def search - objects = [] - params = [] - @norm_search.each do |p| - params.append(p) + params = @norm_search.map do |p| + p end params.push(@owner) unless @owner.empty? run_query( get_sql, params - ).each do |r| - objects.push({ + ).map do |r| + { coll: r[0], owner: r[1], id: r[2], @@ -867,17 +865,15 @@ def search file_count_fmt: MerrittQuery.num_format(r[10]), billable_size: r[11].nil? ? 0 : r[11], billable_size_fmt: MerrittQuery.num_format(r[11].nil? ? 0 : r[11]) - }) + } end - objects end attr_reader :objects def get_placeholders - p = [] - @norm_search.each do |_s| - p.append('?') + p = @norm_search.map do |_s| + '?' end p.join(',') end From 3d3ce42210f657ceed75939fedc2e315a6a36cd5 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Tue, 9 Apr 2024 11:33:37 -0700 Subject: [PATCH 19/24] filter --- src-admintool/queries/prune_files_query.rb | 24 +++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src-admintool/queries/prune_files_query.rb b/src-admintool/queries/prune_files_query.rb index 2fc8a523..3cd05628 100644 --- a/src-admintool/queries/prune_files_query.rb +++ b/src-admintool/queries/prune_files_query.rb @@ -14,12 +14,19 @@ def get_title def get_sql %{ select distinct + 'urlparam' type, o.ark, o.version_number, substring_index(f.pathname, '?', 1) norm, + lower(substring_index(substring_index(f.pathname, '?', 1), '.', -1)) ext, max(v.number), count(distinct f.pathname), - count(distinct digest_value) + count(distinct digest_value), + case + when lower(substring_index(substring_index(f.pathname, '?', 1), '.', -1)) in ('txt', 'json', 'xml') + then 'INFO' + else 'WARN' + end as status from inv.inv_objects o inner join @@ -53,13 +60,20 @@ def get_sql count(distinct digest_value) > 1 union select distinct + 'deleted' type, o.ark, o.version_number, substring_index(f.pathname, '?', 1) norm, + lower(substring_index(substring_index(f.pathname, '?', 1), '.', -1)) ext, max(v.number), count(distinct f.pathname), - count(distinct digest_value) - from + count(distinct digest_value), + case + when lower(substring_index(substring_index(f.pathname, '?', 1), '.', -1)) in ('txt', 'json', 'xml') + then 'INFO' + else 'WARN' + end as status + from inv.inv_objects o inner join inv.inv_files f @@ -98,10 +112,10 @@ def get_params end def get_headers(_results) - ['Ark', 'Version', 'Path', 'Max Ver', 'Path Count', 'Digest Count'] + ['Type','Ark', 'Version', 'Path', 'Ext', 'Max Ver', 'Path Count', 'Digest Count', 'Status'] end def get_types(_results) - ['ark', '', 'name', '', '', ''] + ['', 'ark', 'data', 'name', '', 'data', 'data', 'data', 'status'] end end From 9b2b4aebc17b8475afb542b993631b60102a7fec Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Tue, 9 Apr 2024 11:38:37 -0700 Subject: [PATCH 20/24] rubocop --- src-admintool/queries/prune_files_query.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-admintool/queries/prune_files_query.rb b/src-admintool/queries/prune_files_query.rb index 3cd05628..c6684814 100644 --- a/src-admintool/queries/prune_files_query.rb +++ b/src-admintool/queries/prune_files_query.rb @@ -112,7 +112,7 @@ def get_params end def get_headers(_results) - ['Type','Ark', 'Version', 'Path', 'Ext', 'Max Ver', 'Path Count', 'Digest Count', 'Status'] + ['Type', 'Ark', 'Version', 'Path', 'Ext', 'Max Ver', 'Path Count', 'Digest Count', 'Status'] end def get_types(_results) From 9cdaf9ed88ea5b628cd61bdd8bee5a1bee6f6642 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Wed, 10 Apr 2024 13:03:07 -0700 Subject: [PATCH 21/24] rm table --- merrit-billing/schema.sql | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/merrit-billing/schema.sql b/merrit-billing/schema.sql index b8ef3efa..e4209660 100644 --- a/merrit-billing/schema.sql +++ b/merrit-billing/schema.sql @@ -108,19 +108,6 @@ CREATE TABLE ingests_completed ( INDEX ingest_date(ingest_date) ); -/* -DROP TABLE IF EXISTS node_counts; -*/ -CREATE TABLE node_counts ( - inv_node_id int, - number int, - object_count bigint, - object_count_primary bigint, - object_count_secondary bigint, - file_count bigint, - billable_size bigint, - index node_id(inv_node_id) -); /* DROP TABLE IF EXISTS daily_node_counts; From 018ce655d21676f048d61ec92e7e28f84aa3ec36 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Tue, 5 Mar 2024 17:11:57 -0800 Subject: [PATCH 22/24] init read from zk --- cognito-lambda-nonvpc/.dockerignore | 3 +- mysql-ruby-lambda/.dockerignore | 3 +- simulate-lambda-alb/.dockerignore | 3 +- src-admintool/.dockerignore | 3 +- src-colladmin/.dockerignore | 3 +- src-colladmin/Dockerfile | 2 +- src-colladmin/Gemfile | 2 + src-colladmin/actions/ingest_queue_action.rb | 12 +-- src-colladmin/actions/zookeeper_action.rb | 78 ++++++++++++++++++++ src-colladmin/config/database.ssm.yml | 2 + src-colladmin/lib/queue.rb | 32 +++++--- src-common/.dockerignore | 3 +- 12 files changed, 119 insertions(+), 27 deletions(-) create mode 100644 src-colladmin/actions/zookeeper_action.rb diff --git a/cognito-lambda-nonvpc/.dockerignore b/cognito-lambda-nonvpc/.dockerignore index 02457453..e76e3b51 100644 --- a/cognito-lambda-nonvpc/.dockerignore +++ b/cognito-lambda-nonvpc/.dockerignore @@ -1,4 +1,5 @@ .bundle/ vendor/ README.md -Dockerfile \ No newline at end of file +Dockerfile +Gemfile.lock \ No newline at end of file diff --git a/mysql-ruby-lambda/.dockerignore b/mysql-ruby-lambda/.dockerignore index 02457453..e76e3b51 100644 --- a/mysql-ruby-lambda/.dockerignore +++ b/mysql-ruby-lambda/.dockerignore @@ -1,4 +1,5 @@ .bundle/ vendor/ README.md -Dockerfile \ No newline at end of file +Dockerfile +Gemfile.lock \ No newline at end of file diff --git a/simulate-lambda-alb/.dockerignore b/simulate-lambda-alb/.dockerignore index 02457453..e76e3b51 100644 --- a/simulate-lambda-alb/.dockerignore +++ b/simulate-lambda-alb/.dockerignore @@ -1,4 +1,5 @@ .bundle/ vendor/ README.md -Dockerfile \ No newline at end of file +Dockerfile +Gemfile.lock \ No newline at end of file diff --git a/src-admintool/.dockerignore b/src-admintool/.dockerignore index 06a6d30f..e80a4c5e 100644 --- a/src-admintool/.dockerignore +++ b/src-admintool/.dockerignore @@ -1,2 +1,3 @@ Dockerfile -vendor/ \ No newline at end of file +vendor/ +Gemfile.lock \ No newline at end of file diff --git a/src-colladmin/.dockerignore b/src-colladmin/.dockerignore index 923e3ed5..9de7653d 100644 --- a/src-colladmin/.dockerignore +++ b/src-colladmin/.dockerignore @@ -1,3 +1,4 @@ Dockerfile README.md -vendor/ \ No newline at end of file +vendor/ +Gemfile.lock \ No newline at end of file diff --git a/src-colladmin/Dockerfile b/src-colladmin/Dockerfile index 7db0e7be..fead6063 100644 --- a/src-colladmin/Dockerfile +++ b/src-colladmin/Dockerfile @@ -13,7 +13,7 @@ FROM ${ECR_REGISTRY}/uc3-mrt-admin-common RUN yum -y update # Build rest-client gem from source -RUN yum -y install gcc-c++ +RUN yum -y install gcc-c++ && yum -y groupinstall "Development Tools" # Add Admin Tool Code to the image COPY . /var/task/ diff --git a/src-colladmin/Gemfile b/src-colladmin/Gemfile index a558ed49..def92993 100644 --- a/src-colladmin/Gemfile +++ b/src-colladmin/Gemfile @@ -16,3 +16,5 @@ gem 'rest-client' gem 'rubocop' gem 'uc3-ssm', '0.3.10', git: 'https://github.com/CDLUC3/uc3-ssm.git', branch: 'main' gem 'unf_ext', '~> 0.0.7.7' +gem 'zk' +gem 'zookeeper' diff --git a/src-colladmin/actions/ingest_queue_action.rb b/src-colladmin/actions/ingest_queue_action.rb index df4e8219..43cdbf2e 100644 --- a/src-colladmin/actions/ingest_queue_action.rb +++ b/src-colladmin/actions/ingest_queue_action.rb @@ -1,19 +1,19 @@ # frozen_string_literal: true -require_relative 'forward_to_ingest_action' +require_relative 'zookeeper_action' # Collection Admin Task class - see config/actions.yml for description -class IngestQueueAction < ForwardToIngestAction +class IngestQueueAction < ZookeeperAction def initialize(config, action, path, myparams) @batch = myparams.fetch('batch', '') @profile = myparams.fetch('profile', '') @qstatus = myparams.fetch('qstatus', '') - super(config, action, path, myparams, 'admin/queues') @filter = { batch: @batch, profile: @profile, qstatus: @qstatus } + super(config, action, path, myparams, @filter) @batches = {} end @@ -29,12 +29,6 @@ def table_types QueueEntry.table_types end - def table_rows(body) - queue_list = QueueList.new(get_ingest_server, body, @filter) - @batches = queue_list.batches - queue_list.to_table - end - def has_table true end diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb new file mode 100644 index 00000000..da85cc50 --- /dev/null +++ b/src-colladmin/actions/zookeeper_action.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require_relative 'action' +require_relative '../lib/queue' +require 'zk' + +# Collection Admin Task class - see config/actions.yml for description +class ZkList + def initialize + @jobs = [] + end + + def add_job(job) + @jobs.push(job) + end + + def to_table + table = [] + js = @jobs.sort do |a, b| + if a.status == b.status + b.date <=> a.date + else + AdminTask.status_sort_val(a.status) <=> AdminTask.status_sort_val(b.status) + end + end + js.each_with_index do |q, _i| + table.append(q.to_table_row) + end + table + end +end + +class ZookeeperAction < AdminAction + @@status_vals = ['Pending', 'Consumed', 'Deleted', 'Completed', 'Failed', 'Resolved'] + def initialize(config, action, path, myparams, filters) + super(config, action, path, myparams) + @filters = {} + @path = '/ingest' + @zk = ZK.new(get_zookeeper_conn) + @jobs = ZkList.new + end + + def perform_action + @zk.children(@path).each do |cp| + puts cp + arr = @zk.get("#{@path}/#{cp}") + next if arr[0].nil? + data = arr[0].bytes + return if data.length < 9 + status = data[0] + # https://stackoverflow.com/a/68855488/3846548 + t = data[1..8].inject(0) {|m, b| (m << 8) + b } + time = Time.at(t/1000) + payload=data[9..].pack('c*') + begin + json = JSON.parse(payload) + json['queueNode'] = 'ingest' + json['id'] = cp + json['date'] = time + json['status'] = @@status_vals[status] + puts json.to_json + @jobs.add_job(QueueEntry.new(json)) + #puts JSON.pretty_generate(json) + rescue => exception + #puts exception + end + end + convert_json_to_table('') + end + + def table_rows(_body) + @jobs.to_table + end + + def get_zookeeper_conn + @config.fetch('zookeeper', '').split(',').first + end +end diff --git a/src-colladmin/config/database.ssm.yml b/src-colladmin/config/database.ssm.yml index 19f54f7d..b226be93 100644 --- a/src-colladmin/config/database.ssm.yml +++ b/src-colladmin/config/database.ssm.yml @@ -194,6 +194,7 @@ default: keylist: 7001:scanlist/9502.log disable-scan-nodenums: "{!SSM: colladmin/disable-scan-nodenums !DEFAULT: 0}" uc3inv_home: "{!SSM: admintool/uc3inv_home !DEFAULT: ''}" + zookeeper: "{!SSM: inventory/zoo/queueService}" # use personal credentials when configuring a local environment # set up a script to set these variables docker: @@ -233,4 +234,5 @@ docker: # Provide state endpoints for specific subservices - not applicable for docker endpoints: uc3inv_home: "" + zookeeper: "zoo:2181" diff --git a/src-colladmin/lib/queue.rb b/src-colladmin/lib/queue.rb index e6e73fe8..9d9a709b 100644 --- a/src-colladmin/lib/queue.rb +++ b/src-colladmin/lib/queue.rb @@ -11,51 +11,61 @@ def self.placeholder end def initialize(json) + # not yet used + #{ + # "queuePriority": "03", + # "responseForm": "xml", + # "localID": "2024_03_04_1717_v1file", + # "update": true + #} + super() # until July 2023, Merritt had 3 separate queues identified as a queue node add_property( :queueNode, - MerrittJsonProperty.new('Ingest Worker').lookup_value(json, 'que', 'queueNode') + MerrittJsonProperty.new('Ingest Worker').lookup_value(json, '', 'queueNode') ) add_property( :bid, - MerrittJsonProperty.new('Batch').lookup_value(json, 'que', 'batchID') + MerrittJsonProperty.new('Batch').lookup_value(json, '', 'batchID') ) add_property( :job, - MerrittJsonProperty.new('Job').lookup_value(json, 'que', 'jobID') + MerrittJsonProperty.new('Job').lookup_value(json, '', 'jobID') ) add_property( :profile, - MerrittJsonProperty.new('Profile').lookup_value(json, 'que', 'profile') + MerrittJsonProperty.new('Profile').lookup_value(json, '', 'profile') ) + # insert binary time field add_property( :date, - MerrittJsonProperty.new('Date').lookup_time_value(json, 'que', 'date') + MerrittJsonProperty.new('Date').lookup_time_value(json, '', 'date') ) add_property( :user, - MerrittJsonProperty.new('User').lookup_value(json, 'que', 'user') + MerrittJsonProperty.new('User').lookup_value(json, '', 'submitter') ) add_property( :title, - MerrittJsonProperty.new('Title').lookup_value(json, 'que', 'objectTitle') + MerrittJsonProperty.new('Title').lookup_value(json, '', 'title') ) add_property( :file_type, - MerrittJsonProperty.new('File Type').lookup_value(json, 'que', 'file_type') + MerrittJsonProperty.new('File Type').lookup_value(json, '', 'type') ) + # insert status from binary field add_property( :qstatus, - MerrittJsonProperty.new('QStatus').lookup_value(json, 'que', 'status') + MerrittJsonProperty.new('QStatus').lookup_value(json, '', 'status') ) add_property( :queue, - MerrittJsonProperty.new('Name').lookup_value(json, 'que', 'name') + MerrittJsonProperty.new('Name').lookup_value(json, '', 'filename') ) add_property( :queueId, - MerrittJsonProperty.new('Queue ID').lookup_value(json, 'que', 'iD') + MerrittJsonProperty.new('Queue ID').lookup_value(json, '', 'id') ) # extract the ingest worker node from the queue id string qid = get_value(:queueId, '') diff --git a/src-common/.dockerignore b/src-common/.dockerignore index 139884ec..dbc4dd99 100644 --- a/src-common/.dockerignore +++ b/src-common/.dockerignore @@ -1,4 +1,5 @@ Dockerfile README.md docker-compose.yml -vendor/ \ No newline at end of file +vendor/ +Gemfile.lock \ No newline at end of file From b1b380753297c8aaf10507813a448218af3d30b0 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Fri, 8 Mar 2024 17:41:45 -0800 Subject: [PATCH 23/24] specialize zklists --- src-colladmin/actions/ingest_queue_action.rb | 2 +- src-colladmin/actions/zookeeper_action.rb | 120 ++++++++++++++----- 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/src-colladmin/actions/ingest_queue_action.rb b/src-colladmin/actions/ingest_queue_action.rb index 43cdbf2e..0fe595fd 100644 --- a/src-colladmin/actions/ingest_queue_action.rb +++ b/src-colladmin/actions/ingest_queue_action.rb @@ -3,7 +3,7 @@ require_relative 'zookeeper_action' # Collection Admin Task class - see config/actions.yml for description -class IngestQueueAction < ZookeeperAction +class IngestQueueAction < IngestQueueZookeeperAction def initialize(config, action, path, myparams) @batch = myparams.fetch('batch', '') @profile = myparams.fetch('profile', '') diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb index da85cc50..36a2a380 100644 --- a/src-colladmin/actions/zookeeper_action.rb +++ b/src-colladmin/actions/zookeeper_action.rb @@ -7,16 +7,16 @@ # Collection Admin Task class - see config/actions.yml for description class ZkList def initialize - @jobs = [] + @items = [] end - def add_job(job) - @jobs.push(job) + def add_item(item) + @items.push(item) end def to_table table = [] - js = @jobs.sort do |a, b| + js = @items.sort do |a, b| if a.status == b.status b.date <=> a.date else @@ -30,49 +30,109 @@ def to_table end end +class QueueItemReader + @@na = "NA" + def initialize(zk_action, id, payload) + @bytes = payload.nil? ? [] : payload.bytes + @is_json = zk_action.is_json + @status_vals = zk_action.status_vals + @queue_node = zk_action.zk_path + @id = id + end + + def status_byte + @bytes.empty? ? 0 : @bytes[0] + end + + def status + return @@na if status_byte > @status_vals.length + @status_vals[status_byte] + end + + def time + return nil if @bytes.length < 9 + # https://stackoverflow.com/a/68855488/3846548 + t = @bytes[1..8].inject(0) {|m, b| (m << 8) + b } + Time.at(t/1000) + end + + def payload_text + return "" if @bytes.length < 10 + @bytes[9..].pack('c*') + end + + def payload_object + if @is_json + json = JSON.parse(payload_text) + else + json = { + payload: payload_text + } + end + json['queueNode'] = @queue_node + json['id'] = @id + json['date'] = time + json['status'] = status + json + end + +end + + class ZookeeperAction < AdminAction - @@status_vals = ['Pending', 'Consumed', 'Deleted', 'Completed', 'Failed', 'Resolved'] def initialize(config, action, path, myparams, filters) super(config, action, path, myparams) @filters = {} - @path = '/ingest' @zk = ZK.new(get_zookeeper_conn) - @jobs = ZkList.new + @items = ZkList.new + end + + def zk_path + '/tbd' + end + + def status_vals + [] + end + + def is_json + false + end + + def items + @items end def perform_action - @zk.children(@path).each do |cp| + @zk.children(zk_path).each do |cp| puts cp - arr = @zk.get("#{@path}/#{cp}") - next if arr[0].nil? - data = arr[0].bytes - return if data.length < 9 - status = data[0] - # https://stackoverflow.com/a/68855488/3846548 - t = data[1..8].inject(0) {|m, b| (m << 8) + b } - time = Time.at(t/1000) - payload=data[9..].pack('c*') - begin - json = JSON.parse(payload) - json['queueNode'] = 'ingest' - json['id'] = cp - json['date'] = time - json['status'] = @@status_vals[status] - puts json.to_json - @jobs.add_job(QueueEntry.new(json)) - #puts JSON.pretty_generate(json) - rescue => exception - #puts exception - end + arr = @zk.get("#{zk_path}/#{cp}") + po = QueueItemReader.new(self, cp, arr[0]).payload_object + puts po.to_json + @items.add_item(QueueEntry.new(po)) end convert_json_to_table('') end def table_rows(_body) - @jobs.to_table + items.to_table end def get_zookeeper_conn @config.fetch('zookeeper', '').split(',').first end end + +class IngestQueueZookeeperAction < ZookeeperAction + def zk_path + '/ingest' + end + + def status_vals + ['Pending', 'Consumed', 'Deleted', 'Completed', 'Failed', 'Resolved'] + end + + def is_json + true + end +end \ No newline at end of file From 75ace763afb3906e81b70a5a88889333422ee898 Mon Sep 17 00:00:00 2001 From: Terry Brady Date: Mon, 11 Mar 2024 13:28:26 -0700 Subject: [PATCH 24/24] refine queue job filter --- src-colladmin/actions/ingest_queue_action.rb | 21 +++++++++++++++++++ .../actions/ingest_queue_profile_action.rb | 17 ++++++++++----- src-colladmin/actions/zookeeper_action.rb | 6 +++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src-colladmin/actions/ingest_queue_action.rb b/src-colladmin/actions/ingest_queue_action.rb index 0fe595fd..d760b468 100644 --- a/src-colladmin/actions/ingest_queue_action.rb +++ b/src-colladmin/actions/ingest_queue_action.rb @@ -17,6 +17,27 @@ def initialize(config, action, path, myparams) @batches = {} end + def filter_batch + @filter.fetch(:batch, '') + end + + def filter_profile + @filter.fetch(:profile, '') + end + + def filter_qstatus + @filter.fetch(:qstatus, '') + end + + def register_item(item) + return unless filter_batch.empty? || filter_batch == item.bid + return unless filter_profile.empty? || filter_profile == item.profile + return unless filter_qstatus.empty? || filter_qstatus == item.qstatus + super(item) + end + + + def get_title 'List Ingest Queues' end diff --git a/src-colladmin/actions/ingest_queue_profile_action.rb b/src-colladmin/actions/ingest_queue_profile_action.rb index e4a2c7b4..a67813b8 100644 --- a/src-colladmin/actions/ingest_queue_profile_action.rb +++ b/src-colladmin/actions/ingest_queue_profile_action.rb @@ -3,9 +3,10 @@ require_relative 'forward_to_ingest_action' # Collection Admin Task class - see config/actions.yml for description -class IngestQueueProfileCountAction < ForwardToIngestAction +class IngestQueueProfileCountAction < IngestQueueZookeeperAction def initialize(config, action, path, myparams) - super(config, action, path, myparams, 'admin/queues') + super(config, action, path, myparams, {}) + @profiles = {} end def get_title @@ -32,14 +33,20 @@ def table_types ] end + def register_item(item) + super(item) + k = "#{item.profile},#{item.qstatus}" + @profiles[k] = @profiles.fetch(k, []) + @profiles[k].append(item) + end + def table_rows(body) - queue_list = QueueList.new(get_ingest_server, body) arr = [] - queue_list.profiles.keys.sort.each do |k| + @profiles.keys.sort.each do |k| ka = k.split(',') qs = ka[1] profile = ka[0] - list = queue_list.profiles[k] + list = @profiles[k] count = list.length status = 'PASS' status = 'FAIL' if qs == 'Failed' diff --git a/src-colladmin/actions/zookeeper_action.rb b/src-colladmin/actions/zookeeper_action.rb index 36a2a380..40780447 100644 --- a/src-colladmin/actions/zookeeper_action.rb +++ b/src-colladmin/actions/zookeeper_action.rb @@ -103,13 +103,17 @@ def items @items end + def register_item(item) + @items.add_item(item) + end + def perform_action @zk.children(zk_path).each do |cp| puts cp arr = @zk.get("#{zk_path}/#{cp}") po = QueueItemReader.new(self, cp, arr[0]).payload_object puts po.to_json - @items.add_item(QueueEntry.new(po)) + register_item(QueueEntry.new(po)) end convert_json_to_table('') end