From c49833fd909bba3ac80567c6460833b0fd47ce1c Mon Sep 17 00:00:00 2001 From: Jeremy Friesen Date: Wed, 7 Feb 2024 08:46:46 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20Adding=20dry-monads=20?= =?UTF-8?q?as=20development=20dependency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without the previously committed `require 'dry-monads'` we encounter the following error: ``` bundle exec rspec NameError: uninitialized constant Dry::Monads::Result::Transformer ./spec/test_app/config/application.rb:15:in `' ./spec/test_app/config/environment.rb:4:in `require_relative' ./spec/test_app/config/environment.rb:4:in `' ./spec/rails_helper.rb:8:in `' ./spec/parsers/bulkrax/xml_parser_spec.rb:3:in `' ``` The error repeats everywhere. When I add the development dependency and remove the require, I see the same error. So, to be clear about this development dependency, I've added it to the gemspec. See: - https://github.com/samvera/bulkrax/pull/900#discussion_r1480995354 - https://github.com/samvera/bulkrax/pull/900 --- bulkrax.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/bulkrax.gemspec b/bulkrax.gemspec index db56602c..3da1cb15 100644 --- a/bulkrax.gemspec +++ b/bulkrax.gemspec @@ -32,6 +32,7 @@ Gem::Specification.new do |s| s.add_dependency 'rubyzip' s.add_dependency 'simple_form' + s.add_development_dependency 'dry-monads' s.add_development_dependency 'sqlite3', '~> 1.3.13' s.add_development_dependency 'simplecov' s.add_development_dependency 'redis', '~> 4.2' From 8ce15d9604ace1a68236016dbbeb3ae0373e7de4 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 8 Feb 2024 06:32:06 -0800 Subject: [PATCH 2/2] Denormalize Status Message to that Entry Look Up Can Be Fast (#913) * Thats it, Ive had enough. we need status_message as a column on entry. Today is the day * call the cops * call the cops * Add guard condition for column existence --------- Co-authored-by: Jeremy Friesen --- app/models/bulkrax/importer.rb | 12 ++++-------- app/models/bulkrax/status.rb | 6 +++++- app/models/concerns/bulkrax/status_info.rb | 3 +++ bulkrax.gemspec | 1 + .../20240208005801_denormalize_status_message.rb | 7 +++++++ lib/bulkrax.rb | 1 + lib/tasks/bulkrax_tasks.rake | 12 ++++++++++++ 7 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 db/migrate/20240208005801_denormalize_status_message.rb diff --git a/app/models/bulkrax/importer.rb b/app/models/bulkrax/importer.rb index 76be42c0..94c400f3 100644 --- a/app/models/bulkrax/importer.rb +++ b/app/models/bulkrax/importer.rb @@ -123,16 +123,16 @@ def last_run @last_run ||= self.importer_runs.last end + def failed_entries? + entries.failed.any? + end + def failed_statuses @failed_statuses ||= Bulkrax::Status.latest_by_statusable .includes(:statusable) .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Failed') end - def failed_entries - @failed_entries ||= failed_statuses.map(&:statusable) - end - def failed_messages failed_statuses.each_with_object({}) do |e, i| i[e.error_message] ||= [] @@ -146,10 +146,6 @@ def completed_statuses .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Complete') end - def completed_entries - @completed_entries ||= completed_statuses.map(&:statusable) - end - def seen @seen ||= {} end diff --git a/app/models/bulkrax/status.rb b/app/models/bulkrax/status.rb index fbcb5a87..534f176f 100644 --- a/app/models/bulkrax/status.rb +++ b/app/models/bulkrax/status.rb @@ -2,7 +2,7 @@ module Bulkrax class Status < ApplicationRecord - belongs_to :statusable, polymorphic: true + belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message], if: :latest? } belongs_to :runnable, polymorphic: true serialize :error_backtrace, Array @@ -21,5 +21,9 @@ def self.latest_by_statusable_subtable status_table.join(latest_status_query.as(latest_status_table.name.to_s), Arel::Nodes::InnerJoin) .on(status_table[:id].eq(latest_status_table[:latest_status_id])) end + + def latest? + self.id == self.class.where(statusable_id: self.statusable_id, statusable_type: self.statusable_type).order('id desc').pick(:id) + end end end diff --git a/app/models/concerns/bulkrax/status_info.rb b/app/models/concerns/bulkrax/status_info.rb index 5edc2238..02ce1883 100644 --- a/app/models/concerns/bulkrax/status_info.rb +++ b/app/models/concerns/bulkrax/status_info.rb @@ -10,6 +10,9 @@ module StatusInfo as: :statusable, class_name: "Bulkrax::Status", inverse_of: :statusable + scope :failed, -> { where(status_message: 'Failed') } + scope :complete, -> { where(status_message: 'Complete') } + scope :pending, -> { where(status_message: 'Pending') } end def current_status diff --git a/bulkrax.gemspec b/bulkrax.gemspec index 3da1cb15..adf49b70 100644 --- a/bulkrax.gemspec +++ b/bulkrax.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |s| s.add_dependency 'rails', '>= 5.1.6' s.add_dependency 'bagit', '~> 0.4' s.add_dependency 'coderay' + s.add_dependency 'denormalize_fields' s.add_dependency 'iso8601', '~> 0.9.0' s.add_dependency 'kaminari' s.add_dependency 'language_list', '~> 1.2', '>= 1.2.1' diff --git a/db/migrate/20240208005801_denormalize_status_message.rb b/db/migrate/20240208005801_denormalize_status_message.rb new file mode 100644 index 00000000..314f2e62 --- /dev/null +++ b/db/migrate/20240208005801_denormalize_status_message.rb @@ -0,0 +1,7 @@ +class DenormalizeStatusMessage < ActiveRecord::Migration[5.2] + def change + add_column :bulkrax_entries, :status_message, :string, default: 'Pending' unless column_exists?(:bulkrax_entries, :status_message) + add_column :bulkrax_importers, :status_message, :string, default: 'Pending' unless column_exists?(:bulkrax_importers, :status_message) + add_column :bulkrax_exporters, :status_message, :string, default: 'Pending' unless column_exists?(:bulkrax_exporters, :status_message) + end +end diff --git a/lib/bulkrax.rb b/lib/bulkrax.rb index e88f608b..f485a592 100644 --- a/lib/bulkrax.rb +++ b/lib/bulkrax.rb @@ -3,6 +3,7 @@ require "bulkrax/version" require "bulkrax/engine" require 'active_support/all' +require 'denormalize_fields' # rubocop:disable Metrics/ModuleLength module Bulkrax diff --git a/lib/tasks/bulkrax_tasks.rake b/lib/tasks/bulkrax_tasks.rake index f0a741cb..f3da8dda 100644 --- a/lib/tasks/bulkrax_tasks.rake +++ b/lib/tasks/bulkrax_tasks.rake @@ -1,6 +1,18 @@ # frozen_string_literal: true namespace :bulkrax do + desc 'Update all status messages from the latest status. This is to refresh the denormalized field' + task update_status_messages: :environment do + @progress = ProgressBar.create(total: Bulkrax::Status.latest_by_statusable.count, + format: "%a %b\u{15E7}%i %c/%C %p%% %t", + progress_mark: ' ', + remainder_mark: "\u{FF65}") + Bulkrax::Status.latest_by_statusable.includes(:statusable).find_each do |status| + status.statusable.update(status_message: status.status_message) + @progress.increment + end + end + # Usage example: rails bulkrax:generate_test_csvs['5','100','GenericWork'] desc 'Generate CSVs with fake data for testing purposes' task :generate_test_csvs, [:num_of_csvs, :csv_rows, :record_type] => :environment do |_t, args|