From 2e4c9b79c37451fc2d147c0ae12cc4c10ceefa3c Mon Sep 17 00:00:00 2001 From: Kirk Wang Date: Fri, 15 Sep 2023 14:14:22 -0700 Subject: [PATCH 01/24] =?UTF-8?q?=F0=9F=8E=81=20Add=20`Institution`=20to?= =?UTF-8?q?=20importer=20visibility=20(#860)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit will add the option to set the `Institution` visibility when creating an importer. Both `Public` and `Private` are options are already implemented. --- .../bulkrax/importers/_bagit_fields.html.erb | 11 +++++----- .../bulkrax/importers/_csv_fields.html.erb | 3 ++- .../bulkrax/importers/_oai_fields.html.erb | 9 ++++---- .../bulkrax/importers/_xml_fields.html.erb | 21 ++++++++++--------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/app/views/bulkrax/importers/_bagit_fields.html.erb b/app/views/bulkrax/importers/_bagit_fields.html.erb index e9349d7d2..6e7bbd41a 100644 --- a/app/views/bulkrax/importers/_bagit_fields.html.erb +++ b/app/views/bulkrax/importers/_bagit_fields.html.erb @@ -1,15 +1,15 @@
- <%#= fi.input :metadata_type, - collection: importer.import_metadata_type, + <%#= fi.input :metadata_type, + collection: importer.import_metadata_type, selected: importer.parser_fields['metadata_type'], include_blank: true, input_html: { class: 'form-control' } %> <%= fi.input :metadata_file_name, as: :string, input_html: { value: importer.parser_fields['metadata_file_name'] } %> - <%= fi.input :metadata_format, - collection: importer.import_metadata_format, + <%= fi.input :metadata_format, + collection: importer.import_metadata_format, selected: importer.parser_fields['metadata_format'], include_blank: true, input_html: { class: 'form-control' } @@ -18,7 +18,8 @@ <%= fi.input :visibility, collection: [ ['Public', 'open'], - ['Private', 'restricted'] + ['Private', 'restricted'], + ['Institution', 'authenticated'] ], selected: importer.parser_fields['visibility'] || 'open', input_html: { class: 'form-control' } diff --git a/app/views/bulkrax/importers/_csv_fields.html.erb b/app/views/bulkrax/importers/_csv_fields.html.erb index 852b33f9c..758766e73 100644 --- a/app/views/bulkrax/importers/_csv_fields.html.erb +++ b/app/views/bulkrax/importers/_csv_fields.html.erb @@ -3,7 +3,8 @@ <%= fi.input :visibility, collection: [ ['Public', 'open'], - ['Private', 'restricted'] + ['Private', 'restricted'], + ['Institution', 'authenticated'] ], selected: importer.parser_fields['visibility'] || 'open', input_html: { class: 'form-control' } diff --git a/app/views/bulkrax/importers/_oai_fields.html.erb b/app/views/bulkrax/importers/_oai_fields.html.erb index 3f4488052..d3eb3a3e1 100644 --- a/app/views/bulkrax/importers/_oai_fields.html.erb +++ b/app/views/bulkrax/importers/_oai_fields.html.erb @@ -1,16 +1,17 @@
<%= fi.input :base_url, as: :string, input_html: { value: importer.parser_fields['base_url'] } %> - + <%= fi.input :metadata_prefix, as: :string, hint: 'Such as oai_dc, dcterms or oai_qdc', input_html: { value: importer.parser_fields['metadata_prefix'] } %> - + <%= fi.input :set, collection: [importer.parser_fields['set']], label: 'Set (source)', selected: importer.parser_fields['set'] %> <%= fi.input :visibility, collection: [ ['Public', 'open'], - ['Private', 'restricted'] - ], + ['Private', 'restricted'], + ['Institution', 'authenticated'] + ],x selected: importer.parser_fields['visibility'] || 'open', input_html: { class: 'form-control' } %> diff --git a/app/views/bulkrax/importers/_xml_fields.html.erb b/app/views/bulkrax/importers/_xml_fields.html.erb index 64e25e661..e77fb4d5c 100644 --- a/app/views/bulkrax/importers/_xml_fields.html.erb +++ b/app/views/bulkrax/importers/_xml_fields.html.erb @@ -1,31 +1,32 @@
- <%# @todo improve on this implementation. - As it stands, it's a hostage to namespaces, - eg. dc:title + <%# @todo improve on this implementation. + As it stands, it's a hostage to namespaces, + eg. dc:title if namespaces aren't in the xml, we would have to specify dc:title but if the namespaces ARE present, we remove them so we would need title %> - <%= fi.input :record_element, - hint: 'Provide the xml element name to use to identify the record, or records, eg. ROW - each record in the attached XML is wrapped in a tag.', + <%= fi.input :record_element, + hint: 'Provide the xml element name to use to identify the record, or records, eg. ROW - each record in the attached XML is wrapped in a tag.', input_html: { value: importer.parser_fields['record_element'] } %> - <%= fi.input :import_type, + <%= fi.input :import_type, collection: [ ['Single Work per Metadata File', 'single'], - ['Multiple Works per Metadata File', 'multiple'] - ], + ['Multiple Works per Metadata File', 'multiple'] + ], selected: importer.parser_fields['import_type'], input_html: { class: 'form-control' } %> - +

Visiblity

<%= fi.input :visibility, collection: [ ['Public', 'open'], - ['Private', 'restricted'] + ['Private', 'restricted'], + ['Institution', 'authenticated'] ], selected: importer.parser_fields['visibility'] || 'open', input_html: { class: 'form-control' } From 71a35297eb1480e974d9d32e3be5a9a2fffe3daa Mon Sep 17 00:00:00 2001 From: Matthew Critchlow Date: Wed, 27 Sep 2023 10:21:50 -0700 Subject: [PATCH 02/24] Support additional view i18n labels (#858) Trying to sync up with existing support for changing these labels in different views --- app/views/bulkrax/entries/show.html.erb | 10 +++++----- config/locales/bulkrax.en.yml | 17 +++++++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/app/views/bulkrax/entries/show.html.erb b/app/views/bulkrax/entries/show.html.erb index 0869bc998..13a875248 100644 --- a/app/views/bulkrax/entries/show.html.erb +++ b/app/views/bulkrax/entries/show.html.erb @@ -2,17 +2,17 @@

- Identifier: + <%= t('bulkrax.importer.labels.identifier') %>: <%= @entry.identifier %>

- Entry ID: + <%= t('bulkrax.importer.labels.entry_id') %>: <%= @entry.id %>

- Type: + <%= t('bulkrax.importer.labels.type') %>: <%= @entry.factory_class || 'Unknown' %>

<%= render partial: 'raw_metadata'%> @@ -23,10 +23,10 @@

<% if @importer.present? %> - Importer: + <%= t('bulkrax.importer.labels.importer') %>: <%= link_to @importer.name, importer_path(@importer) %> <% elsif @exporter.present? %> - Exporter: + <%= t('bulkrax.importer.labels.exporter') %>: <%= link_to @exporter.name, exporter_path(@exporter) %> <% end %>

diff --git a/config/locales/bulkrax.en.yml b/config/locales/bulkrax.en.yml index e73d3c18a..573cc78be 100644 --- a/config/locales/bulkrax.en.yml +++ b/config/locales/bulkrax.en.yml @@ -46,18 +46,23 @@ en: generated_metadata: "These exported fields currently cannot be imported." importer: labels: - name: Name - user: User admin_set: Admin set + collection_entries: Collection Entries + entry_id: Entry ID + exporter: Exporter + file_set_entries: File Set Entries frequency: Frequency - parser_klass: Parser klass + identifier: Identifier + importer: Importer limit: Limit - total_work_entries: Total Works + name: Name + parser_klass: Parser klass total_collections: Total Collections total_file_sets: Total File Sets + total_work_entries: Total Works + type: Type + user: User work_entries: Work Entries - collection_entries: Collection Entries - file_set_entries: File Set Entries table_header: labels: identifier: Identifier From 1fc65e405dee38cba8b363b0837df28f2ef6518d Mon Sep 17 00:00:00 2001 From: Shana Moore Date: Mon, 2 Oct 2023 12:51:06 -0700 Subject: [PATCH 03/24] :bug: add guard clauses to add_indices_to_bulkrax migration (#862) add guard clauses to add_indices_to_bulkrax migration Not doing so produced the following error in Adventist, when trying to run migrations: "ArgumentError: Index name 'index_bulkrax_entries_on_identifier' on table 'bulkrax_entries' already exists" --- .../20230608153601_add_indices_to_bulkrax.rb | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/db/migrate/20230608153601_add_indices_to_bulkrax.rb b/db/migrate/20230608153601_add_indices_to_bulkrax.rb index d0fd6c023..d503060d5 100644 --- a/db/migrate/20230608153601_add_indices_to_bulkrax.rb +++ b/db/migrate/20230608153601_add_indices_to_bulkrax.rb @@ -1,16 +1,14 @@ class AddIndicesToBulkrax < ActiveRecord::Migration[5.1] def change - check_and_add_index :bulkrax_entries, :identifier - check_and_add_index :bulkrax_entries, :type - check_and_add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx' - check_and_add_index :bulkrax_pending_relationships, :child_id - check_and_add_index :bulkrax_pending_relationships, :parent_id - check_and_add_index :bulkrax_statuses, :error_class - check_and_add_index :bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx' - check_and_add_index :bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx' - end + add_index :bulkrax_entries, :identifier unless index_exists?(:bulkrax_entries, :identifier) + add_index :bulkrax_entries, :type unless index_exists?(:bulkrax_entries, :type) + add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx' unless index_exists?(:bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx') + + add_index :bulkrax_pending_relationships, :parent_id unless index_exists?(:bulkrax_pending_relationships, :parent_id) + add_index :bulkrax_pending_relationships, :child_id unless index_exists?(:bulkrax_pending_relationships, :child_id) - def check_and_add_index(table_name, column_name, options = {}) - add_index(table_name, column_name, options) unless index_exists?(table_name, column_name, options) + add_index :bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx' unless index_exists?(:bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx') + add_index :bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx' unless index_exists?(:bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx') + add_index :bulkrax_statuses, :error_class unless index_exists?(:bulkrax_statuses, :error_class) end end From 9ea4de68709cf27258b71c0a8815bb51b8bc4385 Mon Sep 17 00:00:00 2001 From: Shana Moore Date: Mon, 2 Oct 2023 14:52:38 -0700 Subject: [PATCH 04/24] =?UTF-8?q?=F0=9F=92=B8=20Minor=20bump=20to=20v5.4.0?= =?UTF-8?q?=20(#864)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mostly a small collection of bug fixes. Ref: - https://playbook-staging.notch8.com/en/samvera/bulkrax/release-a-new-version#set-the-version-number --- lib/bulkrax/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bulkrax/version.rb b/lib/bulkrax/version.rb index ac10a184b..3354df1a3 100644 --- a/lib/bulkrax/version.rb +++ b/lib/bulkrax/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Bulkrax - VERSION = '5.3.0' + VERSION = '5.4.0' end From 0e68a5ea81307dc9e06f9a1e7c0c4cc8cd3de873 Mon Sep 17 00:00:00 2001 From: Alisha Evans Date: Wed, 4 Oct 2023 09:25:05 -0500 Subject: [PATCH 05/24] 5.4.0-bug-fixes (#865) * restore the ability to load the "/importers" page. there was an extra "x" in the app/views/bulkrax/importers/_oai_fields.html.erb file. it threw a syntax error when trying to create a new importer. * restore our ability to properly look up works by their `work_identifier` - ref: https://github.com/samvera-labs/bulkrax/issues/866 - ref:https://github.com/samvera-labs/bulkrax/issues/867 --- app/factories/bulkrax/object_factory.rb | 6 +++++- app/views/bulkrax/importers/_oai_fields.html.erb | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/factories/bulkrax/object_factory.rb b/app/factories/bulkrax/object_factory.rb index df742308e..9e9e8ee69 100644 --- a/app/factories/bulkrax/object_factory.rb +++ b/app/factories/bulkrax/object_factory.rb @@ -103,7 +103,11 @@ def find_or_create end def search_by_identifier - work_index = ::ActiveFedora.index_field_mapper.solr_name(work_identifier, :facetable) + # TODO(alishaevn): return the proper `work_index` value below + # ref: https://github.com/samvera-labs/bulkrax/issues/866 + # ref:https://github.com/samvera-labs/bulkrax/issues/867 + # work_index = ::ActiveFedora.index_field_mapper.solr_name(work_identifier, :facetable) + work_index = work_identifier query = { work_index => source_identifier_value } # Query can return partial matches (something6 matches both something6 and something68) diff --git a/app/views/bulkrax/importers/_oai_fields.html.erb b/app/views/bulkrax/importers/_oai_fields.html.erb index d3eb3a3e1..b89f37550 100644 --- a/app/views/bulkrax/importers/_oai_fields.html.erb +++ b/app/views/bulkrax/importers/_oai_fields.html.erb @@ -11,7 +11,7 @@ ['Public', 'open'], ['Private', 'restricted'], ['Institution', 'authenticated'] - ],x + ], selected: importer.parser_fields['visibility'] || 'open', input_html: { class: 'form-control' } %> From 91c76639559adb37ba857bb79a8cb42bacb42e83 Mon Sep 17 00:00:00 2001 From: Alisha Evans Date: Wed, 4 Oct 2023 09:30:15 -0500 Subject: [PATCH 06/24] preparing to deploy v5.4.1 (#868) --- lib/bulkrax/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bulkrax/version.rb b/lib/bulkrax/version.rb index 3354df1a3..52997a1db 100644 --- a/lib/bulkrax/version.rb +++ b/lib/bulkrax/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Bulkrax - VERSION = '5.4.0' + VERSION = '5.4.1' end From 9968414eaabcb6530d39866f6adf861fd775f862 Mon Sep 17 00:00:00 2001 From: lsitu Date: Tue, 10 Oct 2023 16:31:48 -0700 Subject: [PATCH 07/24] Avoid NoMethodError in Bulkrax::Importers::Controller#create. (#870) --- app/parsers/bulkrax/csv_parser.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/parsers/bulkrax/csv_parser.rb b/app/parsers/bulkrax/csv_parser.rb index d859390dd..f7e34ddac 100644 --- a/app/parsers/bulkrax/csv_parser.rb +++ b/app/parsers/bulkrax/csv_parser.rb @@ -33,9 +33,10 @@ def build_records model_field_mappings.map(&:to_sym).each do |model_mapping| next unless r.key?(model_mapping) - if r[model_mapping].strip.casecmp('collection').zero? + model = r[model_mapping].nil? ? "" : r[model_mapping].strip + if model.casecmp('collection').zero? @collections << r - elsif r[model_mapping].strip.casecmp('fileset').zero? + elsif model.casecmp('fileset').zero? @file_sets << r else @works << r From 75e3d2633389420bbf57723588e87844d8ce37ad Mon Sep 17 00:00:00 2001 From: tamsin johnson Date: Tue, 17 Oct 2023 07:20:30 -0700 Subject: [PATCH 08/24] use the `GlobalID` library tooling to determine global id (#869) the old `#gid` method was weird and breaking for users expecting things to work with `GlobalID` tools: https://github.com/samvera/hyrax/issues/4696 Hyrax has provided a mechanism for updating "gid" references since the 3.0 line, and will drop `#gid` in 5.0.0. use `#to_global_id` to get global ids instead. --- app/factories/bulkrax/object_factory.rb | 2 +- app/models/bulkrax/csv_collection_entry.rb | 2 +- app/parsers/bulkrax/oai_dc_parser.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/factories/bulkrax/object_factory.rb b/app/factories/bulkrax/object_factory.rb index 9e9e8ee69..dd5d37705 100644 --- a/app/factories/bulkrax/object_factory.rb +++ b/app/factories/bulkrax/object_factory.rb @@ -253,7 +253,7 @@ def clean_attrs(attrs) def collection_type(attrs) return attrs if attrs['collection_type_gid'].present? - attrs['collection_type_gid'] = Hyrax::CollectionType.find_or_create_default_collection_type.gid + attrs['collection_type_gid'] = Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s attrs end diff --git a/app/models/bulkrax/csv_collection_entry.rb b/app/models/bulkrax/csv_collection_entry.rb index c62f46dd9..ea6df6325 100644 --- a/app/models/bulkrax/csv_collection_entry.rb +++ b/app/models/bulkrax/csv_collection_entry.rb @@ -15,7 +15,7 @@ def add_identifier def add_collection_type_gid return if self.parsed_metadata['collection_type_gid'].present? - self.parsed_metadata['collection_type_gid'] = ::Hyrax::CollectionType.find_or_create_default_collection_type.gid + self.parsed_metadata['collection_type_gid'] = ::Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s end end end diff --git a/app/parsers/bulkrax/oai_dc_parser.rb b/app/parsers/bulkrax/oai_dc_parser.rb index 4116ac529..4319ab3f1 100644 --- a/app/parsers/bulkrax/oai_dc_parser.rb +++ b/app/parsers/bulkrax/oai_dc_parser.rb @@ -67,7 +67,7 @@ def create_collections metadata = { visibility: 'open' } - metadata[:collection_type_gid] = Hyrax::CollectionType.find_or_create_default_collection_type.gid if defined?(::Hyrax) + metadata[:collection_type_gid] = Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s if defined?(::Hyrax) collections.each_with_index do |set, index| next unless collection_name == 'all' || collection_name == set.spec From 3e86ccd3c2d115666391176b663bae03eae015e5 Mon Sep 17 00:00:00 2001 From: Jeremy Friesen Date: Wed, 18 Oct 2023 10:26:09 -0400 Subject: [PATCH 09/24] =?UTF-8?q?=F0=9F=93=9A=20Update=20docs=20in=20prepa?= =?UTF-8?q?ration=20for=20best=20practices=20seminar=20(#873)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior to this commit, I was not including documented examples of how to use the spec helper methods. Also there were private API methods that I accidentally called in a demonstration. So, I updated the documentation and privatized the private API methods to help folks leverage this function. --- lib/bulkrax/entry_spec_helper.rb | 76 +++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/lib/bulkrax/entry_spec_helper.rb b/lib/bulkrax/entry_spec_helper.rb index 3a40325ec..7e0da64e6 100644 --- a/lib/bulkrax/entry_spec_helper.rb +++ b/lib/bulkrax/entry_spec_helper.rb @@ -10,13 +10,17 @@ module Bulkrax # # This module came about through a desire to expose a quick means of vetting the accuracy of the # different parsers. + # + # @see .entry_for module EntrySpecHelper ## # @api public # @since v5.0.1 # - # The purpose of this method is encapsulate the logic of creating the appropriate Bulkrax::Entry - # object based on the given data, identifier, and parser_class_name. + # The purpose of this method is encapsulate the logic of creating the appropriate + # {Bulkrax::Entry} object based on the given data, identifier, and parser_class_name. Due to + # the different means of instantiation of {Bulkrax::Entry} subclasses, there are several + # optional parameters. # # From that entry, you should be able to test how {Bulkrax::Entry#build_metadata} populates the # {Bulkrax::Entry#parsed_metadata} variable. Other uses may emerge. @@ -29,9 +33,47 @@ module EntrySpecHelper # @param parser_class_name [String] The name of the parser class you're wanting to test. # @param type [Sybmol] The type of entry (e.g. :entry, :collection, :file_set) for testing. # @param options [Hash] these are to be passed along into the instantiation of - # the various classes. See implementation details. + # the various classes. + # @option options [String] importer_name (Optional) The name of the test importer. One will be + # auto-assigned if unprovided. + # @option options [String] importer_admin_set_id (Optional) The ID of an admin set to deposit + # into. One will be auto-assigned if unprovided. And this admin set does not need to + # be persisted nor exist. It is simply a required parameter for instantiating an + # importer. + # @option options [User] user (Optional) The user who is performing the import. One will be + # auto-assigned if unprovided. The user does not need to be persisted. It is simply a + # required parameter for instantiating an importer + # @option options [Integer] limit (Optional) You really shouldn't need to set this, but for + # completeness it is provided. + # @option options [Hash] importer_field_mappings Each parser class may require + # different field mappings. See the given examples for more details. + # + # @return [Bulkrax::Entry] a subclass of {Bulkrax::Entry} based on the application's + # configuration. It would behoove you to write a spec regarding the returned entry's + # class. + # + # @example + # entry = Bulkrax::EntrySpecHelper.entry_for( + # data: { source_identifier: "123", title: "Hello World" }, + # parser_class_name: "Bulkrax::CsvParser", + # importer_field_mappings: { 'import_file_path' => "path/to/file.csv" } + # ) + # + # @note In the case of the Bulkrax::CsvParser, the :data keyword is a Hash, where the keys are + # the column name of the CSV you're importing. The 'import_file_path' is a path to a CSV + # file. That CSV's columns does not need to match the :data's keys, though there may be + # required headers on that CSV based on the parser implementation. # - # @return [Bulkrax::Entry] + # @example + # entry = Bulkrax::EntrySpecHelper.entry_for( + # identifier: identifier, + # data: File.read("/path/to/some/file.xml"), + # parser_class_name: "Bulkrax::OaiDcParser", + # parser_fields: { "base_url" => "http://oai.adventistdigitallibrary.org/OAI-script" } + # ) + # + # @note In the case of an OaiParser, the :data keyword should be a String. And you'll need to + # provide a :parser_fields with a "base_url". def self.entry_for(data:, identifier:, parser_class_name:, type: :entry, **options) importer = importer_for(parser_class_name: parser_class_name, **options) entry_type_method_name = ENTRY_TYPE_TO_METHOD_NAME_MAP.fetch(type) @@ -58,6 +100,7 @@ def self.entry_for(data:, identifier:, parser_class_name:, type: :entry, **optio **options) end + ## # @api public # # @param parser_class_name [String] @@ -121,16 +164,6 @@ def self.importer_for(parser_class_name:, parser_fields: {}, **options) end private_class_method :importer_for - ## - # @api private - # - # @param data [Hash] we're expecting a hash with keys that are symbols and then - # values that are strings. - # - # @return [Bulkrax::CsvEntry] - # - # @note As a foible of this implementation, you'll need to include along a CSV to establish the - # columns that you'll parse (e.g. the first row def self.build_csv_entry_for(importer:, data:, identifier:, entry_class:, **_options) entry_class.new( importerexporter: importer, @@ -138,13 +171,8 @@ def self.build_csv_entry_for(importer:, data:, identifier:, entry_class:, **_opt raw_metadata: data ) end + private_class_method :build_csv_entry_for - ## - # @api private - # - # @param data [String] we're expecting a string that is well-formed XML for OAI parsing. - # - # @return [Bulkrax::OaiEntry] def self.build_oai_entry_for(importer:, data:, identifier:, entry_class:, **options) # The raw record assumes we take the XML data, parse it and then send that to the # OAI::GetRecordResponse object. @@ -165,13 +193,8 @@ def self.build_oai_entry_for(importer:, data:, identifier:, entry_class:, **opti raw_metadata: raw_metadata ) end + private_class_method :build_oai_entry_for - ## - # @api private - # - # @param data [String] we're expecting a string that is well-formed XML. - # - # @return [Bulkrax::XmlEntry] def self.build_xml_entry_for(importer:, data:, identifier:, entry_class:, **options) raw_metadata = { importer.parser.source_identifier.to_s => identifier, @@ -186,5 +209,6 @@ def self.build_xml_entry_for(importer:, data:, identifier:, entry_class:, **opti raw_metadata: raw_metadata ) end + private_class_method :build_xml_entry_for end end From bc82a6f6b46a5f7292911f3795efcd785fdded61 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Thu, 19 Oct 2023 16:37:01 -0400 Subject: [PATCH 10/24] Nav-tabs event scoping (#874) Scope the tab showing behavior to bulkrax views, so that it does not impact unrelated pages --- app/assets/javascripts/bulkrax/navtabs.js.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/bulkrax/navtabs.js.erb b/app/assets/javascripts/bulkrax/navtabs.js.erb index 76c8725df..fed8d52d2 100644 --- a/app/assets/javascripts/bulkrax/navtabs.js.erb +++ b/app/assets/javascripts/bulkrax/navtabs.js.erb @@ -1,7 +1,7 @@ <% unless defined?(::Hyku) %> // enables the tabs in the importers/exporters pages. $(document).ready(function() { - $('.nav-tabs a').click(function (e) { + $('.bulkrax-nav-tab-top-margin.nav-tabs a').click(function (e) { e.preventDefault(); $(this).tab('show'); }); From b689afa637142437d802d2c137fcf2386bcb21c7 Mon Sep 17 00:00:00 2001 From: Kirk Wang Date: Wed, 25 Oct 2023 10:10:08 -0700 Subject: [PATCH 11/24] =?UTF-8?q?=F0=9F=90=9B=20Fix=20tabs=20for=20Hydra?= =?UTF-8?q?=20application=20(#875)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit will add logic for showing the correct tab in a Hydra application like WVU's. We found that if we add the bootstrap require in their application.js then Blacklight's dropdown menu's stop working. However this then breaks Bulkrax tabs. Because we're not adding bootstrap into the Hydra appilication, we mimic the behavior of the tabs function instead. --- app/assets/javascripts/bulkrax/navtabs.js.erb | 21 +++++++++++++++++-- .../bulkrax/shared/_bulkrax_errors.html.erb | 4 ++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/assets/javascripts/bulkrax/navtabs.js.erb b/app/assets/javascripts/bulkrax/navtabs.js.erb index fed8d52d2..c314850d5 100644 --- a/app/assets/javascripts/bulkrax/navtabs.js.erb +++ b/app/assets/javascripts/bulkrax/navtabs.js.erb @@ -1,9 +1,26 @@ <% unless defined?(::Hyku) %> // enables the tabs in the importers/exporters pages. $(document).ready(function() { - $('.bulkrax-nav-tab-top-margin.nav-tabs a').click(function (e) { + $('.bulkrax-nav-tab-top-margin.nav-tabs a').click(function(e) { e.preventDefault(); - $(this).tab('show'); + + // Remove active class from all tabs and hide all tab content + $('.bulkrax-nav-tab-top-margin.nav-tabs a').parent().removeClass('active'); + $('.tab-content .tab-pane').removeClass('active'); + + // Add active class to clicked tab and show its content + $(this).parent().addClass('active'); + $($(this).attr('href')).addClass('active'); }); + + $('#full-errors-tab, #full-errors-tab a').click(function(e) { + $('#raw-errors-tab, #bulkrax-raw-toggle-1').removeClass('active'); + $('#full-errors-tab, #bulkrax-full-toggle-1').addClass('active'); + }) + + $('#raw-errors-tab, #raw-errors-tab a').click(function(e) { + $('#full-errors-tab, #bulkrax-full-toggle-1').removeClass('active'); + $('#raw-errors-tab, #bulkrax-raw-toggle-1').addClass('active'); + }) }); <% end %> diff --git a/app/views/bulkrax/shared/_bulkrax_errors.html.erb b/app/views/bulkrax/shared/_bulkrax_errors.html.erb index 456e255c7..c9a6b4128 100644 --- a/app/views/bulkrax/shared/_bulkrax_errors.html.erb +++ b/app/views/bulkrax/shared/_bulkrax_errors.html.erb @@ -13,8 +13,8 @@
- - + +
From 5dee67c94d681a6c45f9a6418c1e666b90880cd8 Mon Sep 17 00:00:00 2001 From: Adam Ploshay Date: Thu, 26 Oct 2023 08:36:53 -0700 Subject: [PATCH 12/24] add Compatibility section to readme (#879) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index c5c220c60..b2cbe4239 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,11 @@ To delete an importer or exporter, select the delete (x) icon. ### Downloading an export Once your the exporter has run, a download icon will appear on the exporters menu page. +## Compatibility + +* Ruby 2.6 or 2.7 is required +* Hyrax 2.3 or newer is required + ## Contributing If you're working on a PR for this project, create a feature branch off of `main`. From f9973213ca40d43570cc10ecc7e2d5b352c411d4 Mon Sep 17 00:00:00 2001 From: "Brandon R. Straley" Date: Tue, 31 Oct 2023 11:05:44 -0700 Subject: [PATCH 13/24] Update readme to remove references to samvera-labs (#880) * Update readme to remove references to samvera-labs We promoted from samvera-labs to samvera so we need to change references to the repo to /samvera/ * Add Badges for tests and lint --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b2cbe4239..3a87a03b6 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![Test Suite](https://github.com/samvera/bulkrax/actions/workflows/test.yml/badge.svg) +![Test Suite](https://github.com/samvera/bulkrax/actions/workflows/lint.yml/badge.svg) # Bulkrax Bulkrax is a batteries included importer for Samvera applications. It currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the box. It is also designed to be extensible, allowing you to easily add new importers in to your application or to include them with other gems. Bulkrax provides a full admin interface including creating, editing, scheduling and reviewing imports. @@ -10,7 +12,7 @@ Add this line to your application's Gemfile: ```ruby gem 'bulkrax' # or if using from github -gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git', branch: 'main' +gem 'bulkrax', git: 'https://github.com/samvera/bulkrax.git', branch: 'main' ``` And then execute: @@ -72,7 +74,7 @@ Bulkrax.setup do |config| end ``` -The [configuration guide](https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax) provides detailed instructions on the various available configurations. +The [configuration guide](https://github.com/samvera/bulkrax/wiki/Configuring-Bulkrax) provides detailed instructions on the various available configurations. Example: @@ -122,7 +124,7 @@ It's unlikely that the incoming import data has fields that exactly match those By default, a mapping for the OAI parser has been added to map standard oai_dc fields to Hyrax basic_metadata. The other parsers have no default mapping, and will map any incoming fields to Hyrax properties with the same name. Configurations can be added in `config/initializers/bulkrax.rb` -Configuring field mappings is documented in the [Bulkrax Configuration Guide](https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax). +Configuring field mappings is documented in the [Bulkrax Configuration Guide](https://github.com/samvera/bulkrax/wiki/Configuring-Bulkrax). ## Importing Files @@ -153,7 +155,7 @@ end ## Customizing Bulkrax -For further information on how to extend and customize Bulkrax, please see the [Bulkrax Customization Guide](https://github.com/samvera-labs/bulkrax/wiki/Customizing-Bulkrax). +For further information on how to extend and customize Bulkrax, please see the [Bulkrax Customization Guide](https://github.com/samvera/bulkrax/wiki/Customizing-Bulkrax). ## How it Works Once you have Bulkrax installed, you will have access to an easy to use interface with which you are able to create, edit, delete, run, and re-run imports and exports. @@ -191,10 +193,10 @@ If you're working on a PR for this project, create a feature branch off of `main This repository follows the [Samvera Community Code of Conduct](https://samvera.atlassian.net/wiki/spaces/samvera/pages/405212316/Code+of+Conduct) and [language recommendations](https://github.com/samvera/maintenance/blob/master/templates/CONTRIBUTING.md#language). Please ***do not*** create a branch called `master` for this repository or as part of your pull request; the branch will either need to be removed or renamed before it can be considered for inclusion in the code base and history of this repository. See -[CONTRIBUTING.md](https://github.com/samvera-labs/bulkrax/blob/main/CONTRIBUTING.md) +[CONTRIBUTING.md](https://github.com/samvera/bulkrax/blob/main/CONTRIBUTING.md) for contributing guidelines. -We encourage everyone to help improve this project. Bug reports and pull requests are welcome on GitHub at https://github.com/samvera-labs/bulkrax. +We encourage everyone to help improve this project. Bug reports and pull requests are welcome on GitHub at https://github.com/samvera/bulkrax. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](https://contributor-covenant.org) code of conduct. From 1c39f9fffad79c6be0600140a04fe0ff91aef1e9 Mon Sep 17 00:00:00 2001 From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com> Date: Tue, 31 Oct 2023 13:50:41 -0700 Subject: [PATCH 14/24] Reduce SQL calls when incrementing/decrementing run counters (#881) * Reduce SQL calls when incrementing run counters SQL logs before changes (using `#increment!`): ```sql Bulkrax::ImporterRun Load (8.9ms) SELECT "bulkrax_importer_runs".* FROM "bulkrax_importer_runs" WHERE "bulkrax_importer_runs"."id" = $1 LIMIT $2 [["id", 1], ["LIMIT", 1]] Bulkrax::ImporterRun Update All (2.5ms) UPDATE "bulkrax_importer_runs" SET "processed_records" = COALESCE("processed_records", 0) + 1 WHERE "bulkrax_importer_runs"."id" = $1 [["id", 1]] ``` SQL logs after changes (using `#increment_counter`): ```sql Bulkrax::ImporterRun Update All (6.5ms) UPDATE "bulkrax_importer_runs" SET "processed_records" = COALESCE("processed_records", 0) + 1 WHERE "bulkrax_importer_runs"."id" = $1 [["id", 1]] ``` The `SELECT` statement serves no purpose since the `UPDATE` statement is atomic * replace #decrement! with #decrement_counter * put rubocop disable comments back * WIP: update counter-related specs * fix more specs * make some counter specs more specific * remove references to "children" ImporterRun columns These columns (`processed_children` and `failed_children`) were renamed in the RenameChildrenCountersToRelationships migration; they no longer exist * prefer avoiding raw SQL * rubocop --- app/jobs/bulkrax/create_relationships_job.rb | 4 +- app/jobs/bulkrax/delete_job.rb | 8 +- app/jobs/bulkrax/export_work_job.rb | 12 +- app/jobs/bulkrax/import_collection_job.rb | 12 +- app/jobs/bulkrax/import_file_set_job.rb | 12 +- app/jobs/bulkrax/import_work_job.rb | 10 +- app/parsers/bulkrax/application_parser.rb | 4 +- spec/factories/bulkrax_exporter_runs.rb | 8 +- spec/factories/bulkrax_importer_runs.rb | 6 +- spec/jobs/bulkrax/delete_work_job_spec.rb | 24 ++-- spec/jobs/bulkrax/export_work_job_spec.rb | 38 ++++-- spec/jobs/bulkrax/import_file_set_job_spec.rb | 108 ++++++++++++++--- spec/jobs/bulkrax/import_work_job_spec.rb | 113 +++++++++++++++--- spec/test_app/db/schema.rb | 2 - 14 files changed, 269 insertions(+), 92 deletions(-) diff --git a/app/jobs/bulkrax/create_relationships_job.rb b/app/jobs/bulkrax/create_relationships_job.rb index 6e9eb77b8..b6399dabd 100644 --- a/app/jobs/bulkrax/create_relationships_job.rb +++ b/app/jobs/bulkrax/create_relationships_job.rb @@ -98,7 +98,7 @@ def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcS if errors.present? # rubocop:disable Rails/SkipsModelValidations - importer_run.increment!(:failed_relationships, number_of_failures) + ImporterRun.update_counters(importer_run_id, failed_relationships: number_of_failures) # rubocop:enable Rails/SkipsModelValidations parent_entry&.set_status_info(errors.last, importer_run) @@ -108,7 +108,7 @@ def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcS return false # stop current job from continuing to run after rescheduling else # rubocop:disable Rails/SkipsModelValidations - Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships, number_of_successes) + ImporterRun.update_counters(importer_run_id, processed_relationships: number_of_successes) # rubocop:enable Rails/SkipsModelValidations end end diff --git a/app/jobs/bulkrax/delete_job.rb b/app/jobs/bulkrax/delete_job.rb index 764cd8a72..1fcd04cca 100644 --- a/app/jobs/bulkrax/delete_job.rb +++ b/app/jobs/bulkrax/delete_job.rb @@ -4,17 +4,17 @@ module Bulkrax class DeleteJob < ApplicationJob queue_as :import - # rubocop:disable Rails/SkipsModelValidations def perform(entry, importer_run) obj = entry.factory.find obj&.delete - ImporterRun.find(importer_run.id).increment!(:deleted_records) - ImporterRun.find(importer_run.id).decrement!(:enqueued_records) + # rubocop:disable Rails/SkipsModelValidations + ImporterRun.increment_counter(:deleted_records, importer_run.id) + ImporterRun.decrement_counter(:enqueued_records, importer_run.id) + # rubocop:enable Rails/SkipsModelValidations entry.save! entry.importer.current_run = ImporterRun.find(importer_run.id) entry.importer.record_status entry.set_status_info("Deleted", ImporterRun.find(importer_run.id)) end - # rubocop:enable Rails/SkipsModelValidations end end diff --git a/app/jobs/bulkrax/export_work_job.rb b/app/jobs/bulkrax/export_work_job.rb index bc307f3d7..842a35c25 100644 --- a/app/jobs/bulkrax/export_work_job.rb +++ b/app/jobs/bulkrax/export_work_job.rb @@ -12,17 +12,17 @@ def perform(*args) entry.save rescue StandardError # rubocop:disable Rails/SkipsModelValidations - exporter_run.increment!(:failed_records) - exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0 + ExporterRun.increment_counter(:failed_records, args[1]) + ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0 raise else if entry.failed? - exporter_run.increment!(:failed_records) - exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0 + ExporterRun.increment_counter(:failed_records, args[1]) + ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0 raise entry.reload.current_status.error_class.constantize else - exporter_run.increment!(:processed_records) - exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0 + ExporterRun.increment_counter(:processed_records, args[1]) + ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0 end # rubocop:enable Rails/SkipsModelValidations end diff --git a/app/jobs/bulkrax/import_collection_job.rb b/app/jobs/bulkrax/import_collection_job.rb index 163ecd6f9..03405180c 100644 --- a/app/jobs/bulkrax/import_collection_job.rb +++ b/app/jobs/bulkrax/import_collection_job.rb @@ -10,13 +10,13 @@ def perform(*args) begin entry.build entry.save! - ImporterRun.find(args[1]).increment!(:processed_records) - ImporterRun.find(args[1]).increment!(:processed_collections) - ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches + ImporterRun.increment_counter(:processed_records, args[1]) + ImporterRun.increment_counter(:processed_collections, args[1]) + ImporterRun.decrement_counter(:enqueued_records, args[1]) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches rescue => e - ImporterRun.find(args[1]).increment!(:failed_records) - ImporterRun.find(args[1]).increment!(:failed_collections) - ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches + ImporterRun.increment_counter(:failed_records, args[1]) + ImporterRun.increment_counter(:failed_collections, args[1]) + ImporterRun.decrement_counter(:enqueued_records, args[1]) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches raise e end entry.importer.current_run = ImporterRun.find(args[1]) diff --git a/app/jobs/bulkrax/import_file_set_job.rb b/app/jobs/bulkrax/import_file_set_job.rb index c74ab45c0..07fc6a388 100644 --- a/app/jobs/bulkrax/import_file_set_job.rb +++ b/app/jobs/bulkrax/import_file_set_job.rb @@ -21,14 +21,14 @@ def perform(entry_id, importer_run_id) entry.build if entry.succeeded? # rubocop:disable Rails/SkipsModelValidations - ImporterRun.find(importer_run_id).increment!(:processed_records) - ImporterRun.find(importer_run_id).increment!(:processed_file_sets) + ImporterRun.increment_counter(:processed_records, importer_run_id) + ImporterRun.increment_counter(:processed_file_sets, importer_run_id) else - ImporterRun.find(importer_run_id).increment!(:failed_records) - ImporterRun.find(importer_run_id).increment!(:failed_file_sets) + ImporterRun.increment_counter(:failed_records, importer_run_id) + ImporterRun.increment_counter(:failed_file_sets, importer_run_id) # rubocop:enable Rails/SkipsModelValidations end - ImporterRun.find(importer_run_id).decrement!(:enqueued_records) unless ImporterRun.find(importer_run_id).enqueued_records <= 0 # rubocop:disable Rails/SkipsModelValidations + ImporterRun.decrement_counter(:enqueued_records, importer_run_id) unless ImporterRun.find(importer_run_id).enqueued_records <= 0 # rubocop:disable Rails/SkipsModelValidations entry.save! entry.importer.current_run = ImporterRun.find(importer_run_id) entry.importer.record_status @@ -40,7 +40,7 @@ def perform(entry_id, importer_run_id) if entry.import_attempts < 5 ImportFileSetJob.set(wait: (entry.import_attempts + 1).minutes).perform_later(entry_id, importer_run_id) else - ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations + ImporterRun.decrement_counter(:enqueued_records, importer_run_id) # rubocop:disable Rails/SkipsModelValidations entry.set_status_info(e) end end diff --git a/app/jobs/bulkrax/import_work_job.rb b/app/jobs/bulkrax/import_work_job.rb index 95258049f..a3a22ea86 100644 --- a/app/jobs/bulkrax/import_work_job.rb +++ b/app/jobs/bulkrax/import_work_job.rb @@ -23,16 +23,16 @@ def perform(entry_id, run_id, time_to_live = 3, *) entry = Entry.find(entry_id) entry.build if entry.status == "Complete" - ImporterRun.find(run_id).increment!(:processed_records) - ImporterRun.find(run_id).increment!(:processed_works) + ImporterRun.increment_counter(:processed_records, run_id) + ImporterRun.increment_counter(:processed_works, run_id) else # do not retry here because whatever parse error kept you from creating a work will likely # keep preventing you from doing so. - ImporterRun.find(run_id).increment!(:failed_records) - ImporterRun.find(run_id).increment!(:failed_works) + ImporterRun.increment_counter(:failed_records, run_id) + ImporterRun.increment_counter(:failed_works, run_id) end # Regardless of completion or not, we want to decrement the enqueued records. - ImporterRun.find(run_id).decrement!(:enqueued_records) unless ImporterRun.find(run_id).enqueued_records <= 0 + ImporterRun.decrement_counter(:enqueued_records, run_id) unless ImporterRun.find(run_id).enqueued_records <= 0 entry.save! entry.importer.current_run = ImporterRun.find(run_id) diff --git a/app/parsers/bulkrax/application_parser.rb b/app/parsers/bulkrax/application_parser.rb index 97c5c1ed3..9c5a1280f 100644 --- a/app/parsers/bulkrax/application_parser.rb +++ b/app/parsers/bulkrax/application_parser.rb @@ -270,8 +270,8 @@ def invalid_record(message) current_run.invalid_records ||= "" current_run.invalid_records += message current_run.save - ImporterRun.find(current_run.id).increment!(:failed_records) - ImporterRun.find(current_run.id).decrement!(:enqueued_records) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches + ImporterRun.increment_counter(:failed_records, current_run.id) + ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches end # rubocop:enable Rails/SkipsModelValidations diff --git a/spec/factories/bulkrax_exporter_runs.rb b/spec/factories/bulkrax_exporter_runs.rb index 92a748c2d..e7088a4c4 100644 --- a/spec/factories/bulkrax_exporter_runs.rb +++ b/spec/factories/bulkrax_exporter_runs.rb @@ -2,11 +2,11 @@ FactoryBot.define do factory :bulkrax_exporter_run, class: 'Bulkrax::ExporterRun' do - exporter { nil } + exporter { FactoryBot.build(:bulkrax_exporter) } total_work_entries { 1 } enqueued_records { 1 } - processed_records { 1 } - deleted_records { 1 } - failed_records { 1 } + processed_records { 0 } + deleted_records { 0 } + failed_records { 0 } end end diff --git a/spec/factories/bulkrax_importer_runs.rb b/spec/factories/bulkrax_importer_runs.rb index f51f2d93f..f3cf45140 100644 --- a/spec/factories/bulkrax_importer_runs.rb +++ b/spec/factories/bulkrax_importer_runs.rb @@ -5,8 +5,8 @@ importer { FactoryBot.build(:bulkrax_importer) } total_work_entries { 1 } enqueued_records { 1 } - processed_records { 1 } - deleted_records { 1 } - failed_records { 1 } + processed_records { 0 } + deleted_records { 0 } + failed_records { 0 } end end diff --git a/spec/jobs/bulkrax/delete_work_job_spec.rb b/spec/jobs/bulkrax/delete_work_job_spec.rb index 73599ae16..a40c0ba47 100644 --- a/spec/jobs/bulkrax/delete_work_job_spec.rb +++ b/spec/jobs/bulkrax/delete_work_job_spec.rb @@ -5,8 +5,8 @@ module Bulkrax RSpec.describe DeleteWorkJob, type: :job do subject(:delete_work_job) { described_class.new } - let(:entry) { FactoryBot.build(:bulkrax_entry) } - let(:importer_run) { FactoryBot.build(:bulkrax_importer_run) } + let(:entry) { create(:bulkrax_entry) } + let(:importer_run) { create(:bulkrax_importer_run) } describe 'successful job object removed' do before do @@ -18,12 +18,14 @@ module Bulkrax end it 'increments :deleted_records' do - entry.save - importer_run.save + expect(importer_run.enqueued_records).to eq(1) + expect(importer_run.deleted_records).to eq(0) + delete_work_job.perform(entry, importer_run) importer_run.reload - expect(importer_run.enqueued_records).to equal(0) - expect(importer_run.deleted_records).to equal(2) + + expect(importer_run.enqueued_records).to eq(0) + expect(importer_run.deleted_records).to eq(1) end end @@ -35,12 +37,14 @@ module Bulkrax end it 'increments :deleted_records' do - entry.save - importer_run.save + expect(importer_run.enqueued_records).to eq(1) + expect(importer_run.deleted_records).to eq(0) + delete_work_job.perform(entry, importer_run) importer_run.reload - expect(importer_run.enqueued_records).to equal(0) - expect(importer_run.deleted_records).to equal(2) + + expect(importer_run.enqueued_records).to eq(0) + expect(importer_run.deleted_records).to eq(1) end end end diff --git a/spec/jobs/bulkrax/export_work_job_spec.rb b/spec/jobs/bulkrax/export_work_job_spec.rb index 183bda396..c4d3176b4 100644 --- a/spec/jobs/bulkrax/export_work_job_spec.rb +++ b/spec/jobs/bulkrax/export_work_job_spec.rb @@ -5,23 +5,43 @@ module Bulkrax RSpec.describe ExportWorkJob, type: :job do subject(:export_work_job) { described_class.new } - let(:entry) { FactoryBot.build(:bulkrax_entry) } - let(:exporter_run) { FactoryBot.build(:bulkrax_exporter_run) } + let(:exporter) { create(:bulkrax_exporter, :all) } + let(:entry) { create(:bulkrax_entry, importerexporter: exporter) } + let(:exporter_run) { create(:bulkrax_exporter_run, exporter: exporter) } before do allow(Bulkrax::Entry).to receive(:find).with(1).and_return(entry) - allow(Bulkrax::ExporterRun).to receive(:find).with(1).and_return(exporter_run) allow(entry).to receive(:build) end describe 'successful job' do - before do - allow(entry).to receive(:save).and_return(true) + it 'increments :processed_records' do + expect(exporter_run.processed_records).to eq(0) + + export_work_job.perform(entry.id, exporter_run.id) + exporter_run.reload + + expect(exporter_run.processed_records).to eq(1) end - it 'increments :processed_records and decrements enqueued record' do - expect(exporter_run).to receive(:increment!).with(:processed_records) - expect(exporter_run).to receive(:decrement!).with(:enqueued_records) - export_work_job.perform(1, 1) + + it 'decrements :enqueued_records' do + expect(exporter_run.enqueued_records).to eq(1) + + export_work_job.perform(entry.id, exporter_run.id) + exporter_run.reload + + expect(exporter_run.enqueued_records).to eq(0) + end + + it "doesn't change unrelated counters" do + expect(exporter_run.failed_records).to eq(0) + expect(exporter_run.deleted_records).to eq(0) + + export_work_job.perform(1, exporter_run.id) + exporter_run.reload + + expect(exporter_run.failed_records).to eq(0) + expect(exporter_run.deleted_records).to eq(0) end end end diff --git a/spec/jobs/bulkrax/import_file_set_job_spec.rb b/spec/jobs/bulkrax/import_file_set_job_spec.rb index 092baa81d..962c7c0af 100644 --- a/spec/jobs/bulkrax/import_file_set_job_spec.rb +++ b/spec/jobs/bulkrax/import_file_set_job_spec.rb @@ -5,8 +5,8 @@ module Bulkrax RSpec.describe ImportFileSetJob, type: :job do subject(:import_file_set_job) { described_class.new } - let(:importer) { FactoryBot.create(:bulkrax_importer_csv_complex) } - let(:importer_run) { importer.current_run } + let(:importer) { create(:bulkrax_importer_csv_complex) } + let(:importer_run) { create(:bulkrax_importer_run, importer: importer) } let(:entry) { create(:bulkrax_csv_entry_file_set, :with_file_set_metadata, importerexporter: importer) } let(:factory) { instance_double(ObjectFactory) } @@ -16,7 +16,6 @@ module Bulkrax before do allow(Entry).to receive(:find).with(entry.id).and_return(entry) - allow(ImporterRun).to receive(:find).with(importer_run.id).and_return(importer_run) allow(::Hyrax.config).to receive(:curation_concerns).and_return([Work]) allow(::Work).to receive(:where).and_return([]) allow(importer.parser).to receive(:path_to_files).with(filename: 'removed.png').and_return('spec/fixtures/removed.png') @@ -51,15 +50,49 @@ module Bulkrax import_file_set_job.perform(entry.id, importer_run.id) end - it "updates the importer run's counters" do - expect(importer_run).to receive(:increment!).with(:processed_records).once - expect(importer_run).to receive(:increment!).with(:processed_file_sets).once + it 'increments :processed_records and :processed_file_sets' do + expect(importer_run.processed_records).to eq(0) + expect(importer_run.processed_file_sets).to eq(0) - expect(importer_run).not_to receive(:decrement!).with(:enqueued_records) - expect(importer_run).not_to receive(:increment!).with(:failed_records) - expect(importer_run).not_to receive(:increment!).with(:failed_file_sets) + import_file_set_job.perform(entry.id, importer_run.id) + importer_run.reload + + expect(importer_run.processed_records).to eq(1) + expect(importer_run.processed_file_sets).to eq(1) + end + + it 'decrements :enqueued_records' do + expect(importer_run.enqueued_records).to eq(1) + + import_file_set_job.perform(entry.id, importer_run.id) + importer_run.reload + + expect(importer_run.enqueued_records).to eq(0) + end + + it "doesn't change unrelated counters" do + expect(importer_run.failed_records).to eq(0) + expect(importer_run.deleted_records).to eq(0) + expect(importer_run.processed_collections).to eq(0) + expect(importer_run.failed_collections).to eq(0) + expect(importer_run.processed_relationships).to eq(0) + expect(importer_run.failed_relationships).to eq(0) + expect(importer_run.failed_file_sets).to eq(0) + expect(importer_run.processed_works).to eq(0) + expect(importer_run.failed_works).to eq(0) import_file_set_job.perform(entry.id, importer_run.id) + importer_run.reload + + expect(importer_run.failed_records).to eq(0) + expect(importer_run.deleted_records).to eq(0) + expect(importer_run.processed_collections).to eq(0) + expect(importer_run.failed_collections).to eq(0) + expect(importer_run.processed_relationships).to eq(0) + expect(importer_run.failed_relationships).to eq(0) + expect(importer_run.failed_file_sets).to eq(0) + expect(importer_run.processed_works).to eq(0) + expect(importer_run.failed_works).to eq(0) end end @@ -79,8 +112,8 @@ module Bulkrax end it "does not update any of importer run's counters" do - expect(importer_run).not_to receive(:increment!) - expect(importer_run).not_to receive(:decrement!) + expect(ImporterRun).not_to receive(:increment_counter) + expect(ImporterRun).not_to receive(:decrement_counter) end it 'reschedules the job to try again after a couple minutes' do @@ -111,8 +144,11 @@ module Bulkrax end it "only decrements the importer run's :enqueued_records counter" do - expect(importer_run).not_to receive(:increment!) - expect(importer_run).to receive(:decrement!).with(:enqueued_records).once + expect(ImporterRun).not_to receive(:increment_counter) + expect(ImporterRun) + .to receive(:decrement_counter) + .with(:enqueued_records, importer_run.id) + .once import_file_set_job.perform(entry.id, importer_run.id) end @@ -165,15 +201,49 @@ module Bulkrax import_file_set_job.perform(entry.id, importer_run.id) end - it "updates the importer run's counters" do - expect(importer_run).to receive(:increment!).with(:failed_records).once - expect(importer_run).to receive(:increment!).with(:failed_file_sets).once + it 'increments :failed_records and :failed_file_sets' do + expect(importer_run.failed_records).to eq(0) + expect(importer_run.failed_file_sets).to eq(0) + + import_file_set_job.perform(entry.id, importer_run.id) + importer_run.reload + + expect(importer_run.failed_records).to eq(1) + expect(importer_run.failed_file_sets).to eq(1) + end + + it 'decrements :enqueued_records' do + expect(importer_run.enqueued_records).to eq(1) + + import_file_set_job.perform(entry.id, importer_run.id) + importer_run.reload + + expect(importer_run.enqueued_records).to eq(0) + end - expect(importer_run).not_to receive(:decrement!).with(:enqueued_records) - expect(importer_run).not_to receive(:increment!).with(:processed_records) - expect(importer_run).not_to receive(:increment!).with(:processed_file_sets) + it "doesn't change unrelated counters" do + expect(importer_run.processed_records).to eq(0) + expect(importer_run.deleted_records).to eq(0) + expect(importer_run.processed_collections).to eq(0) + expect(importer_run.failed_collections).to eq(0) + expect(importer_run.processed_relationships).to eq(0) + expect(importer_run.failed_relationships).to eq(0) + expect(importer_run.processed_file_sets).to eq(0) + expect(importer_run.processed_works).to eq(0) + expect(importer_run.failed_works).to eq(0) import_file_set_job.perform(entry.id, importer_run.id) + importer_run.reload + + expect(importer_run.processed_records).to eq(0) + expect(importer_run.deleted_records).to eq(0) + expect(importer_run.processed_collections).to eq(0) + expect(importer_run.failed_collections).to eq(0) + expect(importer_run.processed_relationships).to eq(0) + expect(importer_run.failed_relationships).to eq(0) + expect(importer_run.processed_file_sets).to eq(0) + expect(importer_run.processed_works).to eq(0) + expect(importer_run.failed_works).to eq(0) end end end diff --git a/spec/jobs/bulkrax/import_work_job_spec.rb b/spec/jobs/bulkrax/import_work_job_spec.rb index 14652c4c6..f55eb50a4 100644 --- a/spec/jobs/bulkrax/import_work_job_spec.rb +++ b/spec/jobs/bulkrax/import_work_job_spec.rb @@ -11,7 +11,6 @@ module Bulkrax before do allow(Bulkrax::Entry).to receive(:find).with(1).and_return(entry) - allow(Bulkrax::ImporterRun).to receive(:find).with(importer_run_id).and_return(importer_run) end describe 'successful job' do @@ -20,11 +19,57 @@ module Bulkrax allow(entry).to receive(:build).and_return(instance_of(Work)) allow(entry).to receive(:status).and_return('Complete') end + it 'increments :processed_records' do - expect(importer_run).to receive(:increment!).with(:processed_records) - expect(importer_run).to receive(:increment!).with(:processed_works) - expect(importer_run).to receive(:decrement!).with(:enqueued_records) + expect(importer_run.processed_records).to eq(0) + + import_work_job.perform(1, importer_run_id) + importer_run.reload + + expect(importer_run.processed_records).to eq(1) + end + + it 'increments :processed_works' do + expect(importer_run.processed_works).to eq(0) + import_work_job.perform(1, importer_run_id) + importer_run.reload + + expect(importer_run.processed_works).to eq(1) + end + + it 'decrements :enqueued_records' do + expect(importer_run.enqueued_records).to eq(1) + + import_work_job.perform(1, importer_run_id) + importer_run.reload + + expect(importer_run.enqueued_records).to eq(0) + end + + it "doesn't change unrelated counters" do + expect(importer_run.failed_records).to eq(0) + expect(importer_run.deleted_records).to eq(0) + expect(importer_run.processed_collections).to eq(0) + expect(importer_run.failed_collections).to eq(0) + expect(importer_run.processed_relationships).to eq(0) + expect(importer_run.failed_relationships).to eq(0) + expect(importer_run.processed_file_sets).to eq(0) + expect(importer_run.failed_file_sets).to eq(0) + expect(importer_run.failed_works).to eq(0) + + import_work_job.perform(1, importer_run_id) + importer_run.reload + + expect(importer_run.failed_records).to eq(0) + expect(importer_run.deleted_records).to eq(0) + expect(importer_run.processed_collections).to eq(0) + expect(importer_run.failed_collections).to eq(0) + expect(importer_run.processed_relationships).to eq(0) + expect(importer_run.failed_relationships).to eq(0) + expect(importer_run.processed_file_sets).to eq(0) + expect(importer_run.failed_file_sets).to eq(0) + expect(importer_run.failed_works).to eq(0) end end @@ -32,11 +77,22 @@ module Bulkrax before do allow(entry).to receive(:build_for_importer).and_raise(CollectionsCreatedError) end - it 'does not call increment' do - expect(importer_run).not_to receive(:increment!) - expect(importer_run).not_to receive(:decrement!) + + it 'does not change counters' do + expect(importer_run.processed_records).to eq(0) + expect(importer_run.processed_works).to eq(0) + expect(importer_run.failed_records).to eq(0) + expect(importer_run.failed_works).to eq(0) + import_work_job.perform(1, importer_run_id) + importer_run.reload + + expect(importer_run.processed_records).to eq(0) + expect(importer_run.processed_works).to eq(0) + expect(importer_run.failed_records).to eq(0) + expect(importer_run.failed_works).to eq(0) end + it 'reschedules the job' do expect(import_work_job).to receive(:reschedule) # rubocop:disable RSpec/SubjectStub import_work_job.perform(1, importer_run_id) @@ -47,11 +103,25 @@ module Bulkrax before do allow(entry).to receive(:build).and_return(nil) end - it 'increments :failed_records' do - expect(importer_run).to receive(:increment!).with(:failed_records) - expect(importer_run).to receive(:increment!).with(:failed_works) - expect(importer_run).to receive(:decrement!).with(:enqueued_records) + + it 'increments :failed_records and :failed_works' do + expect(importer_run.failed_records).to eq(0) + expect(importer_run.failed_works).to eq(0) + import_work_job.perform(1, importer_run_id) + importer_run.reload + + expect(importer_run.failed_records).to eq(1) + expect(importer_run.failed_works).to eq(1) + end + + it 'decrements :enqueued_records' do + expect(importer_run.enqueued_records).to eq(1) + + import_work_job.perform(1, importer_run_id) + importer_run.reload + + expect(importer_run.enqueued_records).to eq(0) end end @@ -59,10 +129,25 @@ module Bulkrax before do allow(entry).to receive(:build).and_raise(OAIError) end - it 'increments :failed_records' do + + it 'does not increment :failed_records or :failed_works' do + expect(importer_run.failed_records).to eq(0) + expect(importer_run.failed_works).to eq(0) + expect { import_work_job.perform(1, importer_run_id) }.to raise_error(OAIError) - expect(importer_run).not_to receive(:increment!).with(:failed_records) - expect(importer_run).not_to receive(:decrement!).with(:enqueued_records) + importer_run.reload + + expect(importer_run.failed_records).to eq(0) + expect(importer_run.failed_works).to eq(0) + end + + it 'does not decrement :enqueued_records' do + expect(importer_run.enqueued_records).to eq(1) + + expect { import_work_job.perform(1, importer_run_id) }.to raise_error(OAIError) + importer_run.reload + + expect(importer_run.enqueued_records).to eq(1) end end end diff --git a/spec/test_app/db/schema.rb b/spec/test_app/db/schema.rb index 054b476fc..70580d6ec 100644 --- a/spec/test_app/db/schema.rb +++ b/spec/test_app/db/schema.rb @@ -99,8 +99,6 @@ t.integer "total_file_set_entries", default: 0 t.integer "processed_works", default: 0 t.integer "failed_works", default: 0 - t.integer "processed_children", default: 0 - t.integer "failed_children", default: 0 t.index ["importer_id"], name: "index_bulkrax_importer_runs_on_importer_id" end From 657c6feaae8d81fe9487dbd16dc42730637b9313 Mon Sep 17 00:00:00 2001 From: tamsin johnson Date: Tue, 31 Oct 2023 19:16:27 -0400 Subject: [PATCH 15/24] add support for Rails 6, Hyrax 4, and Blacklight 7 (#782) * add support for Rails 6, Hyrax 4, and Blacklight 7 this makes no attempt to ensure UI-layer support for BL7 and whatever bootstrap versions it requires (excepting whatever is tested by RSpec). the scope here is simply to make it *possible* for a Hyrax 4 application to install and use Bulkrax. * include current Hyrax CSS class and onclick in sidebar additions * change `.hidden` css classes to `.d-none` * bootstrap upgrade nav-link classes * update dry-monads to ~> 1.5.0 * re-run rubocop * begin testing on ruby 3, drop support for 2.6 * no more auto foreign key * update schema in test app * add support for Rails 6, Hyrax 4, and Blacklight 7 this makes no attempt to ensure UI-layer support for BL7 and whatever bootstrap versions it requires (excepting whatever is tested by RSpec). the scope here is simply to make it *possible* for a Hyrax 4 application to install and use Bulkrax. * include current Hyrax CSS class and onclick in sidebar additions * change `.hidden` css classes to `.d-none` * bootstrap upgrade nav-link classes * update dry-monads to ~> 1.5.0 * re-run rubocop * begin testing on ruby 3, drop support for 2.6 * no more auto foreign key * update schema in test app * trying to talk ci in to passing * add support for Rails 6, Hyrax 4, and Blacklight 7 this makes no attempt to ensure UI-layer support for BL7 and whatever bootstrap versions it requires (excepting whatever is tested by RSpec). the scope here is simply to make it *possible* for a Hyrax 4 application to install and use Bulkrax. * include current Hyrax CSS class and onclick in sidebar additions * change `.hidden` css classes to `.d-none` * bootstrap upgrade nav-link classes * update dry-monads to ~> 1.5.0 * re-run rubocop * begin testing on ruby 3, drop support for 2.6 * no more auto foreign key * update schema in test app * trying to talk ci in to passing * deal with differences in rails spec db handling for rails 6 --------- Co-authored-by: Alexandra Dunn Co-authored-by: Rob Kaufman --- .github/workflows/lint.yml | 6 +- .github/workflows/test.yml | 8 +- .rubocop_todo.yml | 98 ++----------------- Gemfile | 5 +- README.md | 2 +- app/assets/javascripts/bulkrax/exporters.js | 10 +- app/factories/bulkrax/object_factory.rb | 2 +- app/jobs/bulkrax/create_relationships_job.rb | 3 +- app/models/bulkrax/csv_entry.rb | 6 +- .../remove_relationships_for_importer.rb | 3 +- app/views/bulkrax/exporters/_form.html.erb | 20 ++-- app/views/bulkrax/exporters/show.html.erb | 6 +- app/views/bulkrax/importers/show.html.erb | 6 +- .../_bulkrax_sidebar_additions.html.erb | 16 ++- bulkrax.gemspec | 2 +- ...ge_importer_and_exporter_to_polymorphic.rb | 10 +- .../templates/config/initializers/bulkrax.rb | 1 + .../app/controllers/application_controller.rb | 4 +- spec/test_app/config/application.rb | 1 - 19 files changed, 71 insertions(+), 138 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7b159b4f7..06aed9f9a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,12 +18,12 @@ jobs: uses: actions/cache@v2.1.3 with: path: vendor/bundle - key: 2.6.5 + key: 2.7.2 - name: Setup Ruby - uses: ruby/setup-ruby@v1.59.1 + uses: ruby/setup-ruby@v1.159.0 with: - ruby-version: 2.6.5 + ruby-version: 2.7.2 - name: Install dependencies run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3cac1e93..38896fc21 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby: ['2.6', '2.7'] # TODO add 3.0 compat , '3.0'] + ruby: ['2.7', '3.0'] name: Run specs with ruby ${{ matrix.ruby }} steps: - uses: actions/checkout@v2 @@ -24,7 +24,7 @@ jobs: key: ${{ matrix.ruby }} - name: Setup Ruby - uses: ruby/setup-ruby@v1.59.1 + uses: ruby/setup-ruby@v1.159.0 with: ruby-version: ${{ matrix.ruby }} @@ -37,10 +37,10 @@ jobs: bundle install - name: Migrate test database - run: bin/rails db:migrate RAILS_ENV=test + run: bundle exec rake db:migrate db:test:prepare - name: Run rspec - run: bin/rspec + run: bundle exec rake - name: Upload coverage results uses: actions/upload-artifact@v2 diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index ad19a7fd2..3e90e131c 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -6,73 +6,13 @@ # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: EmptyLineBetweenMethodDefs, EmptyLineBetweenClassDefs, EmptyLineBetweenModuleDefs, AllowAdjacentOneLineDefs, NumberOfEmptyLines. -Layout/EmptyLineBetweenDefs: - Exclude: - - 'app/jobs/bulkrax/import_file_set_job.rb' - - 'app/models/bulkrax/entry.rb' - -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: EnforcedStyle. -# SupportedStyles: empty_lines, no_empty_lines -Layout/EmptyLinesAroundBlockBody: - Exclude: - - 'spec/rails_helper.rb' - -# Offense count: 3 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle. -# SupportedHashRocketStyles: key, separator, table -# SupportedColonStyles: key, separator, table -# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit -Layout/HashAlignment: - Exclude: - - 'app/parsers/bulkrax/csv_parser.rb' - - 'spec/models/bulkrax/rdf_entry_spec.rb' - - 'spec/models/bulkrax/xml_entry_spec.rb' - -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: Width, AllowedPatterns, IgnoredPatterns. -Layout/IndentationWidth: - Exclude: - - 'spec/rails_helper.rb' - -# Offense count: 8 +# Offense count: 5 # This cop supports safe auto-correction (--auto-correct). # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, IgnoredPatterns. # URISchemes: http, https Layout/LineLength: Max: 301 -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: EnforcedStyle. -# SupportedStyles: symmetrical, new_line, same_line -Layout/MultilineMethodCallBraceLayout: - Exclude: - - 'app/parsers/bulkrax/csv_parser.rb' - -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -Layout/RescueEnsureAlignment: - Exclude: - - 'spec/rails_helper.rb' - -# Offense count: 7 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: AllowInHeredoc. -Layout/TrailingWhitespace: - Exclude: - - 'app/models/bulkrax/csv_entry.rb' - - 'app/parsers/bulkrax/csv_parser.rb' - - 'spec/models/bulkrax/rdf_entry_spec.rb' - - 'spec/models/bulkrax/xml_entry_spec.rb' - - 'spec/rails_helper.rb' - # Offense count: 16 # Configuration parameters: IgnoredMethods, CountRepeatedAttributes. Metrics/AbcSize: @@ -81,14 +21,14 @@ Metrics/AbcSize: # Offense count: 4 # Configuration parameters: CountComments, CountAsOne. Metrics/ClassLength: - Max: 201 + Max: 140 -# Offense count: 13 +# Offense count: 12 # Configuration parameters: IgnoredMethods. Metrics/CyclomaticComplexity: Max: 19 -# Offense count: 32 +# Offense count: 28 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods. Metrics/MethodLength: Max: 26 @@ -96,9 +36,9 @@ Metrics/MethodLength: # Offense count: 2 # Configuration parameters: CountComments, CountAsOne. Metrics/ModuleLength: - Max: 131 + Max: 130 -# Offense count: 9 +# Offense count: 10 # Configuration parameters: IgnoredMethods. Metrics/PerceivedComplexity: Max: 19 @@ -110,30 +50,8 @@ Rails/HasManyOrHasOneDependent: Exclude: - 'app/models/concerns/bulkrax/status_info.rb' -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: Keywords, RequireColon. -# Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW, NOTE -Style/CommentAnnotation: - Exclude: - - 'app/models/bulkrax/xml_entry.rb' - - 'spec/models/bulkrax/oai_entry_spec.rb' - -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -Style/IfUnlessModifier: - Exclude: - - 'app/models/bulkrax/csv_entry.rb' - - 'lib/generators/bulkrax/templates/config/initializers/bulkrax.rb' - # Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -Style/MultilineIfModifier: +# Configuration parameters: MinBodyLength, AllowConsecutiveConditionals. +Style/GuardClause: Exclude: - 'app/models/bulkrax/csv_entry.rb' - -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -Style/RedundantBegin: - Exclude: - - 'spec/rails_helper.rb' diff --git a/Gemfile b/Gemfile index 30071446e..d8f99f28f 100644 --- a/Gemfile +++ b/Gemfile @@ -12,11 +12,11 @@ gemspec # Git. Remember to move these dependencies to your gemspec before releasing # your gem to rubygems.org. -gem 'blacklight', '~> 6.25.0' +gem 'blacklight' gem 'bootstrap-sass', '~> 3.4.1' gem 'coderay' gem 'factory_bot_rails' -gem 'hyrax', '>= 2.3' +gem 'hyrax', '>= 2.3', '< 4.999' gem 'oai' gem 'rsolr', '>= 1.0' gem 'rspec-rails' @@ -29,6 +29,7 @@ group :development, :test do gem 'pry-byebug' gem 'solargraph' gem 'solr_wrapper', '>= 0.3' + gem 'sqlite3', '~> 1.4' end group :lint do diff --git a/README.md b/README.md index 3a87a03b6..bb9ec1ef9 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ Once your the exporter has run, a download icon will appear on the exporters men ## Compatibility -* Ruby 2.6 or 2.7 is required +* Ruby 2.7 or newer is required * Hyrax 2.3 or newer is required ## Contributing diff --git a/app/assets/javascripts/bulkrax/exporters.js b/app/assets/javascripts/bulkrax/exporters.js index 829ceb47b..67944d41a 100644 --- a/app/assets/javascripts/bulkrax/exporters.js +++ b/app/assets/javascripts/bulkrax/exporters.js @@ -14,7 +14,7 @@ function hideUnhide(field) { } }; -function addRequired(selectedSource) { +function addRequired(selectedSource) { selectedSource.addClass('required').attr('required', 'required'); selectedSource.parent().addClass('required'); } @@ -26,14 +26,14 @@ function removeRequired(allSources) { // hide all export_source function hide(allSources) { - allSources.addClass('hidden'); - allSources.find('#exporter_export_source').addClass('hidden').attr('type', 'hidden'); + allSources.addClass('d-none'); + allSources.find('#exporter_export_source').addClass('.d-none').attr('type', 'd-none'); } // unhide selected export_source function unhideSelected(selectedSource) { - selectedSource.removeClass('hidden').removeAttr('type'); - selectedSource.parent().removeClass('hidden').removeAttr('type'); + selectedSource.removeClass('d-none').removeAttr('type'); + selectedSource.parent().removeClass('d-none').removeAttr('type'); }; // add the autocomplete javascript diff --git a/app/factories/bulkrax/object_factory.rb b/app/factories/bulkrax/object_factory.rb index dd5d37705..ca0ba791a 100644 --- a/app/factories/bulkrax/object_factory.rb +++ b/app/factories/bulkrax/object_factory.rb @@ -123,7 +123,7 @@ def search_by_identifier def create attrs = transform_attributes @object = klass.new - object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent) + object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if defined?(Hyrax::Adapters::NestingIndexAdapter) && object.respond_to?(:reindex_extent) run_callbacks :save do run_callbacks :create do if klass == Collection diff --git a/app/jobs/bulkrax/create_relationships_job.rb b/app/jobs/bulkrax/create_relationships_job.rb index b6399dabd..a0dfc44ca 100644 --- a/app/jobs/bulkrax/create_relationships_job.rb +++ b/app/jobs/bulkrax/create_relationships_job.rb @@ -158,7 +158,8 @@ def process(relationship:, importer_run_id:, parent_record:, ability:) end def add_to_collection(child_record, parent_record) - parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) + parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if + defined?(Hyrax::Adapters::NestingIndexAdapter) child_record.member_of_collections << parent_record child_record.save! end diff --git a/app/models/bulkrax/csv_entry.rb b/app/models/bulkrax/csv_entry.rb index aaa0bc24b..a106a74bf 100644 --- a/app/models/bulkrax/csv_entry.rb +++ b/app/models/bulkrax/csv_entry.rb @@ -88,8 +88,10 @@ def build_metadata def validate_record raise StandardError, 'Record not found' if record.nil? - raise StandardError, "Missing required elements, missing element(s) are: "\ -"#{importerexporter.parser.missing_elements(record).join(', ')}" unless importerexporter.parser.required_elements?(record) + unless importerexporter.parser.required_elements?(record) + raise StandardError, "Missing required elements, missing element(s) are: "\ +"#{importerexporter.parser.missing_elements(record).join(', ')}" + end end def add_identifier diff --git a/app/services/bulkrax/remove_relationships_for_importer.rb b/app/services/bulkrax/remove_relationships_for_importer.rb index 784b5ed07..10fa92e40 100644 --- a/app/services/bulkrax/remove_relationships_for_importer.rb +++ b/app/services/bulkrax/remove_relationships_for_importer.rb @@ -63,7 +63,8 @@ def break_relationships! remove_relationships_from_work(obj) end - obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if defined?(Hyrax) + obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if + defined?(Hyrax::Adapters::NestingIndexAdapter) obj.save! end end diff --git a/app/views/bulkrax/exporters/_form.html.erb b/app/views/bulkrax/exporters/_form.html.erb index 4e54215b2..df3af0b38 100644 --- a/app/views/bulkrax/exporters/_form.html.erb +++ b/app/views/bulkrax/exporters/_form.html.erb @@ -33,8 +33,8 @@ label: t('bulkrax.exporter.labels.importer'), required: true, prompt: 'Select from the list', - label_html: { class: 'importer export-source-option hidden' }, - input_html: { class: 'importer export-source-option hidden form-control' }, + label_html: { class: 'importer export-source-option d-none' }, + input_html: { class: 'importer export-source-option d-none form-control' }, collection: form.object.importers_list.sort %> <%= form.input :export_source_collection, @@ -42,9 +42,9 @@ label: t('bulkrax.exporter.labels.collection'), required: true, placeholder: @collection&.title&.first, - label_html: { class: 'collection export-source-option hidden' }, + label_html: { class: 'collection export-source-option d-none' }, input_html: { - class: 'collection export-source-option hidden form-control', + class: 'collection export-source-option d-none form-control', data: { 'autocomplete-url' => '/authorities/search/collections', 'autocomplete' => 'collection' @@ -56,8 +56,8 @@ label: t('bulkrax.exporter.labels.worktype'), required: true, prompt: 'Select from the list', - label_html: { class: 'worktype export-source-option hidden' }, - input_html: { class: 'worktype export-source-option hidden form-control' }, + label_html: { class: 'worktype export-source-option d-none' }, + input_html: { class: 'worktype export-source-option d-none form-control' }, collection: Bulkrax.curation_concerns.map { |cc| [cc.to_s, cc.to_s] } %> <%= form.input :limit, @@ -80,7 +80,7 @@ as: :boolean, label: t('bulkrax.exporter.labels.filter_by_date') %> -