Read Queue Data from ZK rather than Ingest

cognito-lambda-nonvpc/.dockerignore

@@ -1,4 +1,5 @@
 .bundle/
 vendor/
 README.md
-Dockerfile
+Dockerfile
+Gemfile.lock

cognito-lambda-nonvpc/Dockerfile

@@ -5,10 +5,12 @@
 # This image is based on an architecture that matches the Lambda Ruby runtime
 # docker build -t ${ECR_REGISTRY}/cognito-lambda-nonvpc .
 
-FROM public.ecr.aws/lambda/ruby:3
+FROM public.ecr.aws/lambda/ruby:3.2
+
+RUN yum -y update && yum -y upgrade
 
-RUN yum -y update
 RUN yum -y install gcc make git
+
 RUN gem update bundler 
 
 # Start with a Gemfile containing only MySQL

cognito-lambda-nonvpc/Gemfile.lock

@@ -3,11 +3,11 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.881.0)
-    aws-sdk-cognitoidentityprovider (1.85.0)
-      aws-sdk-core (~> 3, >= 3.188.0)
+    aws-partitions (1.895.0)
+    aws-sdk-cognitoidentityprovider (1.87.0)
+      aws-sdk-core (~> 3, >= 3.191.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.190.3)
+    aws-sdk-core (3.191.3)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
@@ -18,26 +18,26 @@ GEM
     json (2.7.1)
     language_server-protocol (3.17.0.3)
     parallel (1.24.0)
-    parser (3.3.0.4)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
     racc (1.7.3)
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.60.1)
+    rubocop (1.62.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
     ruby-progressbar (1.13.0)
     unicode-display_width (2.5.0)
 

merrit-billing/schema.sql

@@ -108,19 +108,6 @@ CREATE TABLE ingests_completed (
   INDEX ingest_date(ingest_date)
 );
 
-/*
-DROP TABLE IF EXISTS node_counts;
-*/
-CREATE TABLE node_counts (
-  inv_node_id int,
-  number int,
-  object_count bigint,
-  object_count_primary bigint,
-  object_count_secondary bigint,
-  file_count bigint,
-  billable_size bigint,
-  index node_id(inv_node_id)
-);
 
 /*
 DROP TABLE IF EXISTS daily_node_counts;

mysql-ruby-lambda/.dockerignore

@@ -1,4 +1,5 @@
 .bundle/
 vendor/
 README.md
-Dockerfile
+Dockerfile
+Gemfile.lock

mysql-ruby-lambda/Dockerfile

@@ -5,9 +5,9 @@
 # This image is based on an architecture that matches the Lambda Ruby runtime
 # docker build -t ${ECR_REGISTRY}/mysql-ruby-lambda .
 
-FROM public.ecr.aws/lambda/ruby:3
+FROM public.ecr.aws/lambda/ruby:3.2
 
-RUN yum -y update
+RUN yum -y update && yum -y upgrade
 
 # Install an os-specific MySQL installation.  
 # gcc and make are required to build mysql.

mysql-ruby-lambda/Gemfile.lock

@@ -13,43 +13,43 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.881.0)
-    aws-sdk-core (3.190.3)
+    aws-partitions (1.895.0)
+    aws-sdk-core (3.191.3)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-ssm (1.162.0)
-      aws-sdk-core (~> 3, >= 3.188.0)
+    aws-sdk-ssm (1.165.0)
+      aws-sdk-core (~> 3, >= 3.191.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.8.0)
       aws-eventstream (~> 1, >= 1.0.2)
     jmespath (1.6.2)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
     logger (1.6.0)
-    mysql2 (0.5.5)
+    mysql2 (0.5.6)
     parallel (1.24.0)
-    parser (3.3.0.4)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
     racc (1.7.3)
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.60.1)
+    rubocop (1.62.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
     ruby-progressbar (1.13.0)
     unicode-display_width (2.5.0)
     yaml (0.2.1)

simulate-lambda-alb/.dockerignore

@@ -1,4 +1,5 @@
 .bundle/
 vendor/
 README.md
-Dockerfile
+Dockerfile
+Gemfile.lock

simulate-lambda-alb/Dockerfile

@@ -9,7 +9,7 @@
 
 FROM ruby:3
 
-RUN apt-get update -y && apt-get -y upgrade
+RUN apt-get -y update && apt-get -y upgrade
 
 RUN gem install bundler
 

simulate-lambda-alb/Gemfile.lock

@@ -9,36 +9,39 @@ GEM
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     parallel (1.24.0)
-    parser (3.3.0.4)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
     racc (1.7.3)
-    rack (2.2.8)
-    rack-protection (3.2.0)
+    rack (3.0.9.1)
+    rack-protection (4.0.0)
       base64 (>= 0.1.0)
-      rack (~> 2.2, >= 2.2.4)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.60.1)
+    rubocop (1.62.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sinatra (3.2.0)
+    sinatra (4.0.0)
       mustermann (~> 3.0)
-      rack (~> 2.2, >= 2.2.4)
-      rack-protection (= 3.2.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.0.0)
+      rack-session (>= 2.0.0, < 3)
       tilt (~> 2.0)
     tilt (2.3.0)
     unicode-display_width (2.5.0)

src-admintool/.dockerignore

@@ -1,2 +1,3 @@
 Dockerfile
-vendor/
+vendor/
+Gemfile.lock

src-admintool/Gemfile.lock

@@ -13,50 +13,50 @@ GEM
   specs:
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.881.0)
-    aws-sdk-core (3.190.3)
+    aws-partitions (1.895.0)
+    aws-sdk-core (3.191.3)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-kms (1.76.0)
-      aws-sdk-core (~> 3, >= 3.188.0)
+    aws-sdk-kms (1.77.0)
+      aws-sdk-core (~> 3, >= 3.191.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.142.0)
-      aws-sdk-core (~> 3, >= 3.189.0)
+    aws-sdk-s3 (1.143.0)
+      aws-sdk-core (~> 3, >= 3.191.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.8)
-    aws-sdk-ssm (1.162.0)
-      aws-sdk-core (~> 3, >= 3.188.0)
+    aws-sdk-ssm (1.165.0)
+      aws-sdk-core (~> 3, >= 3.191.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.8.0)
       aws-eventstream (~> 1, >= 1.0.2)
     jmespath (1.6.2)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
     logger (1.6.0)
-    mysql2 (0.5.5)
+    mysql2 (0.5.6)
     parallel (1.24.0)
-    parser (3.3.0.4)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
     racc (1.7.3)
     rainbow (3.1.1)
     regexp_parser (2.9.0)
     rexml (3.2.6)
-    rubocop (1.60.1)
+    rubocop (1.62.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
     ruby-progressbar (1.13.0)
     unicode-display_width (2.5.0)
     yaml (0.2.1)

src-admintool/config/reports.yml

@@ -120,6 +120,13 @@ producer_files:
   category: files
   description: |
     Producer Files for a Mnemonic.
+prune_candidates:
+  link-title: Prune Candidate Files for a Mnemonic
+  breadcrumb: bp_content_projects
+  class: PruneCandidateFilesQuery
+  category: files
+  description: |
+    Prune Candidate Files for a Mnemonic.
 daily-build-s3-list:
   link-title: Daily Build Links
   breadcrumb: bp_content_projects

src-admintool/queries/objectid_files_query.rb

@@ -16,6 +16,17 @@ def get_sql
       select
         o.ark,
         v.number,
+        (
+          select max(vv.number)
+          from inv.inv_files ff
+          inner join inv.inv_versions vv
+            on ff.inv_version_id = vv.id
+          where
+            ff.inv_object_id = o.id
+          and exists (
+            select 1 where ff.pathname=f.pathname
+          )
+        ) as maxv,
         f.source,
         binary f.pathname,
         f.full_size,
@@ -69,11 +80,11 @@ def get_params
   end
 
   def get_headers(_results)
-    ['Ark', 'Version', 'Source', 'Path', 'File Size', 'Created', 'Nodes', 'Unverified']
+    ['Ark', 'Version', 'MaxVer', 'Source', 'Path', 'File Size', 'Created', 'Nodes', 'Unverified']
   end
 
   def get_types(_results)
-    ['ark', '', '', 'name', 'bytes', 'datetime', '', '']
+    ['ark', '', '', '', 'name', 'bytes', 'datetime', '', '']
   end
 
   def get_alternative_queries