diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7b159b4f7..06aed9f9a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,12 +18,12 @@ jobs: uses: actions/cache@v2.1.3 with: path: vendor/bundle - key: 2.6.5 + key: 2.7.2 - name: Setup Ruby - uses: ruby/setup-ruby@v1.59.1 + uses: ruby/setup-ruby@v1.159.0 with: - ruby-version: 2.6.5 + ruby-version: 2.7.2 - name: Install dependencies run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3cac1e93..38896fc21 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby: ['2.6', '2.7'] # TODO add 3.0 compat , '3.0'] + ruby: ['2.7', '3.0'] name: Run specs with ruby ${{ matrix.ruby }} steps: - uses: actions/checkout@v2 @@ -24,7 +24,7 @@ jobs: key: ${{ matrix.ruby }} - name: Setup Ruby - uses: ruby/setup-ruby@v1.59.1 + uses: ruby/setup-ruby@v1.159.0 with: ruby-version: ${{ matrix.ruby }} @@ -37,10 +37,10 @@ jobs: bundle install - name: Migrate test database - run: bin/rails db:migrate RAILS_ENV=test + run: bundle exec rake db:migrate db:test:prepare - name: Run rspec - run: bin/rspec + run: bundle exec rake - name: Upload coverage results uses: actions/upload-artifact@v2 diff --git a/.rubocop.yml b/.rubocop.yml index f08dee44a..7c702ccbf 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,7 +1,9 @@ # Bixby coding conventions +require: rubocop-factory_bot + # Added AllCops config for further modification AllCops: - TargetRubyVersion: 2.6 + TargetRubyVersion: 2.7 DisabledByDefault: true DisplayCopNames: true Exclude: diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index ad19a7fd2..3e90e131c 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -6,73 +6,13 @@ # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: EmptyLineBetweenMethodDefs, EmptyLineBetweenClassDefs, EmptyLineBetweenModuleDefs, AllowAdjacentOneLineDefs, NumberOfEmptyLines. -Layout/EmptyLineBetweenDefs: - Exclude: - - 'app/jobs/bulkrax/import_file_set_job.rb' - - 'app/models/bulkrax/entry.rb' - -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: EnforcedStyle. -# SupportedStyles: empty_lines, no_empty_lines -Layout/EmptyLinesAroundBlockBody: - Exclude: - - 'spec/rails_helper.rb' - -# Offense count: 3 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle. -# SupportedHashRocketStyles: key, separator, table -# SupportedColonStyles: key, separator, table -# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit -Layout/HashAlignment: - Exclude: - - 'app/parsers/bulkrax/csv_parser.rb' - - 'spec/models/bulkrax/rdf_entry_spec.rb' - - 'spec/models/bulkrax/xml_entry_spec.rb' - -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: Width, AllowedPatterns, IgnoredPatterns. -Layout/IndentationWidth: - Exclude: - - 'spec/rails_helper.rb' - -# Offense count: 8 +# Offense count: 5 # This cop supports safe auto-correction (--auto-correct). # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, IgnoredPatterns. # URISchemes: http, https Layout/LineLength: Max: 301 -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: EnforcedStyle. -# SupportedStyles: symmetrical, new_line, same_line -Layout/MultilineMethodCallBraceLayout: - Exclude: - - 'app/parsers/bulkrax/csv_parser.rb' - -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -Layout/RescueEnsureAlignment: - Exclude: - - 'spec/rails_helper.rb' - -# Offense count: 7 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: AllowInHeredoc. -Layout/TrailingWhitespace: - Exclude: - - 'app/models/bulkrax/csv_entry.rb' - - 'app/parsers/bulkrax/csv_parser.rb' - - 'spec/models/bulkrax/rdf_entry_spec.rb' - - 'spec/models/bulkrax/xml_entry_spec.rb' - - 'spec/rails_helper.rb' - # Offense count: 16 # Configuration parameters: IgnoredMethods, CountRepeatedAttributes. Metrics/AbcSize: @@ -81,14 +21,14 @@ Metrics/AbcSize: # Offense count: 4 # Configuration parameters: CountComments, CountAsOne. Metrics/ClassLength: - Max: 201 + Max: 140 -# Offense count: 13 +# Offense count: 12 # Configuration parameters: IgnoredMethods. Metrics/CyclomaticComplexity: Max: 19 -# Offense count: 32 +# Offense count: 28 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods. Metrics/MethodLength: Max: 26 @@ -96,9 +36,9 @@ Metrics/MethodLength: # Offense count: 2 # Configuration parameters: CountComments, CountAsOne. Metrics/ModuleLength: - Max: 131 + Max: 130 -# Offense count: 9 +# Offense count: 10 # Configuration parameters: IgnoredMethods. Metrics/PerceivedComplexity: Max: 19 @@ -110,30 +50,8 @@ Rails/HasManyOrHasOneDependent: Exclude: - 'app/models/concerns/bulkrax/status_info.rb' -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -# Configuration parameters: Keywords, RequireColon. -# Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW, NOTE -Style/CommentAnnotation: - Exclude: - - 'app/models/bulkrax/xml_entry.rb' - - 'spec/models/bulkrax/oai_entry_spec.rb' - -# Offense count: 2 -# This cop supports safe auto-correction (--auto-correct). -Style/IfUnlessModifier: - Exclude: - - 'app/models/bulkrax/csv_entry.rb' - - 'lib/generators/bulkrax/templates/config/initializers/bulkrax.rb' - # Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -Style/MultilineIfModifier: +# Configuration parameters: MinBodyLength, AllowConsecutiveConditionals. +Style/GuardClause: Exclude: - 'app/models/bulkrax/csv_entry.rb' - -# Offense count: 1 -# This cop supports safe auto-correction (--auto-correct). -Style/RedundantBegin: - Exclude: - - 'spec/rails_helper.rb' diff --git a/Gemfile b/Gemfile index 30071446e..ebfe5fb1c 100644 --- a/Gemfile +++ b/Gemfile @@ -12,11 +12,11 @@ gemspec # Git. Remember to move these dependencies to your gemspec before releasing # your gem to rubygems.org. -gem 'blacklight', '~> 6.25.0' +gem 'blacklight' gem 'bootstrap-sass', '~> 3.4.1' gem 'coderay' gem 'factory_bot_rails' -gem 'hyrax', '>= 2.3' +gem 'hyrax', '>= 2.3', '< 4.999' gem 'oai' gem 'rsolr', '>= 1.0' gem 'rspec-rails' @@ -29,8 +29,10 @@ group :development, :test do gem 'pry-byebug' gem 'solargraph' gem 'solr_wrapper', '>= 0.3' + gem 'sqlite3', '~> 1.4' end group :lint do gem 'bixby' + gem 'rubocop-factory_bot', require: false end diff --git a/README.md b/README.md index c5c220c60..bb9ec1ef9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![Test Suite](https://github.com/samvera/bulkrax/actions/workflows/test.yml/badge.svg) +![Test Suite](https://github.com/samvera/bulkrax/actions/workflows/lint.yml/badge.svg) # Bulkrax Bulkrax is a batteries included importer for Samvera applications. It currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the box. It is also designed to be extensible, allowing you to easily add new importers in to your application or to include them with other gems. Bulkrax provides a full admin interface including creating, editing, scheduling and reviewing imports. @@ -10,7 +12,7 @@ Add this line to your application's Gemfile: ```ruby gem 'bulkrax' # or if using from github -gem 'bulkrax', git: 'https://github.com/samvera-labs/bulkrax.git', branch: 'main' +gem 'bulkrax', git: 'https://github.com/samvera/bulkrax.git', branch: 'main' ``` And then execute: @@ -72,7 +74,7 @@ Bulkrax.setup do |config| end ``` -The [configuration guide](https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax) provides detailed instructions on the various available configurations. +The [configuration guide](https://github.com/samvera/bulkrax/wiki/Configuring-Bulkrax) provides detailed instructions on the various available configurations. Example: @@ -122,7 +124,7 @@ It's unlikely that the incoming import data has fields that exactly match those By default, a mapping for the OAI parser has been added to map standard oai_dc fields to Hyrax basic_metadata. The other parsers have no default mapping, and will map any incoming fields to Hyrax properties with the same name. Configurations can be added in `config/initializers/bulkrax.rb` -Configuring field mappings is documented in the [Bulkrax Configuration Guide](https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax). +Configuring field mappings is documented in the [Bulkrax Configuration Guide](https://github.com/samvera/bulkrax/wiki/Configuring-Bulkrax). ## Importing Files @@ -153,7 +155,7 @@ end ## Customizing Bulkrax -For further information on how to extend and customize Bulkrax, please see the [Bulkrax Customization Guide](https://github.com/samvera-labs/bulkrax/wiki/Customizing-Bulkrax). +For further information on how to extend and customize Bulkrax, please see the [Bulkrax Customization Guide](https://github.com/samvera/bulkrax/wiki/Customizing-Bulkrax). ## How it Works Once you have Bulkrax installed, you will have access to an easy to use interface with which you are able to create, edit, delete, run, and re-run imports and exports. @@ -180,16 +182,21 @@ To delete an importer or exporter, select the delete (x) icon. ### Downloading an export Once your the exporter has run, a download icon will appear on the exporters menu page. +## Compatibility + +* Ruby 2.7 or newer is required +* Hyrax 2.3 or newer is required + ## Contributing If you're working on a PR for this project, create a feature branch off of `main`. This repository follows the [Samvera Community Code of Conduct](https://samvera.atlassian.net/wiki/spaces/samvera/pages/405212316/Code+of+Conduct) and [language recommendations](https://github.com/samvera/maintenance/blob/master/templates/CONTRIBUTING.md#language). Please ***do not*** create a branch called `master` for this repository or as part of your pull request; the branch will either need to be removed or renamed before it can be considered for inclusion in the code base and history of this repository. See -[CONTRIBUTING.md](https://github.com/samvera-labs/bulkrax/blob/main/CONTRIBUTING.md) +[CONTRIBUTING.md](https://github.com/samvera/bulkrax/blob/main/CONTRIBUTING.md) for contributing guidelines. -We encourage everyone to help improve this project. Bug reports and pull requests are welcome on GitHub at https://github.com/samvera-labs/bulkrax. +We encourage everyone to help improve this project. Bug reports and pull requests are welcome on GitHub at https://github.com/samvera/bulkrax. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](https://contributor-covenant.org) code of conduct. diff --git a/app/assets/javascripts/bulkrax/bulkrax.js b/app/assets/javascripts/bulkrax/bulkrax.js index c918def53..006ce8205 100644 --- a/app/assets/javascripts/bulkrax/bulkrax.js +++ b/app/assets/javascripts/bulkrax/bulkrax.js @@ -8,4 +8,15 @@ $(document).on('turbolinks:load ready', function() { $('button#fm_toggle').click(function() { $('#field_mapping').toggle(); }); + $('#bulkraxItemModal').on('show.bs.modal', function (event) { + var button = $(event.relatedTarget) // Button that triggered the modal + var recipient = button.data('entry-id') // Extract info from data-* attributes + // If necessary, you could initiate an AJAX request here (and then do the updating in a callback). + // Update the modal's content. We'll use jQuery here, but you could use a data binding library or other methods instead. + var modal = $(this) + modal.find('a').each(function() { + this.href = this.href.replace(/\d+\?/, recipient + '?') + }) + return true + }) }); diff --git a/app/assets/javascripts/bulkrax/exporters.js b/app/assets/javascripts/bulkrax/exporters.js index 829ceb47b..67944d41a 100644 --- a/app/assets/javascripts/bulkrax/exporters.js +++ b/app/assets/javascripts/bulkrax/exporters.js @@ -14,7 +14,7 @@ function hideUnhide(field) { } }; -function addRequired(selectedSource) { +function addRequired(selectedSource) { selectedSource.addClass('required').attr('required', 'required'); selectedSource.parent().addClass('required'); } @@ -26,14 +26,14 @@ function removeRequired(allSources) { // hide all export_source function hide(allSources) { - allSources.addClass('hidden'); - allSources.find('#exporter_export_source').addClass('hidden').attr('type', 'hidden'); + allSources.addClass('d-none'); + allSources.find('#exporter_export_source').addClass('.d-none').attr('type', 'd-none'); } // unhide selected export_source function unhideSelected(selectedSource) { - selectedSource.removeClass('hidden').removeAttr('type'); - selectedSource.parent().removeClass('hidden').removeAttr('type'); + selectedSource.removeClass('d-none').removeAttr('type'); + selectedSource.parent().removeClass('d-none').removeAttr('type'); }; // add the autocomplete javascript diff --git a/app/assets/javascripts/bulkrax/navtabs.js.erb b/app/assets/javascripts/bulkrax/navtabs.js.erb index 76c8725df..c314850d5 100644 --- a/app/assets/javascripts/bulkrax/navtabs.js.erb +++ b/app/assets/javascripts/bulkrax/navtabs.js.erb @@ -1,9 +1,26 @@ <% unless defined?(::Hyku) %> // enables the tabs in the importers/exporters pages. $(document).ready(function() { - $('.nav-tabs a').click(function (e) { + $('.bulkrax-nav-tab-top-margin.nav-tabs a').click(function(e) { e.preventDefault(); - $(this).tab('show'); + + // Remove active class from all tabs and hide all tab content + $('.bulkrax-nav-tab-top-margin.nav-tabs a').parent().removeClass('active'); + $('.tab-content .tab-pane').removeClass('active'); + + // Add active class to clicked tab and show its content + $(this).parent().addClass('active'); + $($(this).attr('href')).addClass('active'); }); + + $('#full-errors-tab, #full-errors-tab a').click(function(e) { + $('#raw-errors-tab, #bulkrax-raw-toggle-1').removeClass('active'); + $('#full-errors-tab, #bulkrax-full-toggle-1').addClass('active'); + }) + + $('#raw-errors-tab, #raw-errors-tab a').click(function(e) { + $('#full-errors-tab, #bulkrax-full-toggle-1').removeClass('active'); + $('#raw-errors-tab, #bulkrax-raw-toggle-1').addClass('active'); + }) }); <% end %> diff --git a/app/controllers/bulkrax/entries_controller.rb b/app/controllers/bulkrax/entries_controller.rb index 57c0fd6a3..4328ed0bf 100644 --- a/app/controllers/bulkrax/entries_controller.rb +++ b/app/controllers/bulkrax/entries_controller.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require_dependency "bulkrax/application_controller" -require_dependency "oai" - module Bulkrax class EntriesController < ApplicationController include Hyrax::ThemedLayoutController if defined?(::Hyrax) @@ -18,6 +15,42 @@ def show end end + def update + @entry = Entry.find(params[:id]) + @entry.factory&.find&.destroy if params[:destroy_first] + @entry.build + @entry.save + item = @entry.importerexporter + entry_path = item.class.to_s.include?('Importer') ? bulkrax.importer_entry_path(item.id, @entry.id) : bulkrax.exporter_entry_path(item.id, @entry.id) + + redirect_back fallback_location: entry_path, notice: "Entry update ran, new status is #{@entry.status}" + end + + def destroy + @entry = Entry.find(params[:id]) + @status = "" + begin + work = @entry.factory&.find + if work.present? + work.destroy + @entry.destroy + @status = "Entry and work deleted" + else + @entry.destroy + @status = "Entry deleted" + end + rescue StandardError => e + @status = "Error: #{e.message}" + end + + item = @entry.importerexporter + entry_path = item.class.to_s.include?('Importer') ? bulkrax.importer_entry_path(item.id, @entry.id) : bulkrax.exporter_entry_path(item.id, @entry.id) + + redirect_back fallback_location: entry_path, notice: @status + end + + protected + # GET /importers/1/entries/1 def show_importer @importer = Importer.find(params[:importer_id]) diff --git a/app/controllers/bulkrax/exporters_controller.rb b/app/controllers/bulkrax/exporters_controller.rb index 22efe039a..02ab2b95f 100644 --- a/app/controllers/bulkrax/exporters_controller.rb +++ b/app/controllers/bulkrax/exporters_controller.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_dependency "bulkrax/application_controller" - module Bulkrax class ExportersController < ApplicationController include Hyrax::ThemedLayoutController if defined?(::Hyrax) diff --git a/app/controllers/bulkrax/importers_controller.rb b/app/controllers/bulkrax/importers_controller.rb index cb762e0a6..8248975cd 100644 --- a/app/controllers/bulkrax/importers_controller.rb +++ b/app/controllers/bulkrax/importers_controller.rb @@ -1,11 +1,8 @@ # frozen_string_literal: true -require_dependency 'bulkrax/application_controller' -require_dependency 'oai' - module Bulkrax # rubocop:disable Metrics/ClassLength - class ImportersController < ApplicationController + class ImportersController < ::Bulkrax::ApplicationController include Hyrax::ThemedLayoutController if defined?(::Hyrax) include Bulkrax::DownloadBehavior include Bulkrax::API @@ -241,7 +238,7 @@ def importer_params end def list_external_sets - url = params[:base_url] || (@harvester ? @harvester.base_url : nil) + url = params[:base_url] || @harvester&.base_url setup_client(url) if url.present? @sets = [['All', 'all']] @@ -310,10 +307,16 @@ def update_harvest end def set_files_parser_fields + @importer.parser_fields['update_files'] = + @importer.parser_fields['replace_files'] = + @importer.parser_fields['remove_and_rerun'] = + @importer.parser_fields['metadata_only'] = false if params[:commit] == 'Update Metadata and Files' @importer.parser_fields['update_files'] = true elsif params[:commit] == ('Update and Replace Files' || 'Update and Re-Harvest All Items') @importer.parser_fields['replace_files'] = true + elsif params[:commit] == 'Remove and Rerun' + @importer.parser_fields['remove_and_rerun'] = true elsif params[:commit] == 'Update and Harvest Updated Items' return else diff --git a/app/factories/bulkrax/object_factory.rb b/app/factories/bulkrax/object_factory.rb index 0034df2a7..34e9b07a0 100644 --- a/app/factories/bulkrax/object_factory.rb +++ b/app/factories/bulkrax/object_factory.rb @@ -28,15 +28,16 @@ class ObjectFactory # rubocop:disable Metrics/ClassLength class_attribute :transformation_removes_blank_hash_values, default: false define_model_callbacks :save, :create - attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier, :related_parents_parsed_mapping, :importer_run_id + attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier, :work_identifier_search_field, :related_parents_parsed_mapping, :importer_run_id # rubocop:disable Metrics/ParameterLists - def initialize(attributes:, source_identifier_value:, work_identifier:, related_parents_parsed_mapping: nil, replace_files: false, user: nil, klass: nil, importer_run_id: nil, update_files: false) + def initialize(attributes:, source_identifier_value:, work_identifier:, work_identifier_search_field:, related_parents_parsed_mapping: nil, replace_files: false, user: nil, klass: nil, importer_run_id: nil, update_files: false) @attributes = ActiveSupport::HashWithIndifferentAccess.new(attributes) @replace_files = replace_files @update_files = update_files @user = user || User.batch_user @work_identifier = work_identifier + @work_identifier_search_field = work_identifier_search_field @related_parents_parsed_mapping = related_parents_parsed_mapping @source_identifier_value = source_identifier_value @klass = klass || Bulkrax.default_work_type.constantize @@ -106,8 +107,7 @@ def find_or_create end def search_by_identifier - work_index = ::ActiveFedora.index_field_mapper.solr_name(work_identifier, :facetable) - query = { work_index => + query = { work_identifier_search_field => source_identifier_value } # Query can return partial matches (something6 matches both something6 and something68) # so we need to weed out any that are not the correct full match. But other items might be @@ -122,7 +122,7 @@ def search_by_identifier def create attrs = transform_attributes @object = klass.new - object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent) + object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if defined?(Hyrax::Adapters::NestingIndexAdapter) && object.respond_to?(:reindex_extent) run_callbacks :save do run_callbacks :create do if klass == Collection @@ -252,7 +252,7 @@ def clean_attrs(attrs) def collection_type(attrs) return attrs if attrs['collection_type_gid'].present? - attrs['collection_type_gid'] = Hyrax::CollectionType.find_or_create_default_collection_type.gid + attrs['collection_type_gid'] = Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s attrs end diff --git a/app/helpers/bulkrax/application_helper.rb b/app/helpers/bulkrax/application_helper.rb index 0242aa5d8..9418043c1 100644 --- a/app/helpers/bulkrax/application_helper.rb +++ b/app/helpers/bulkrax/application_helper.rb @@ -1,9 +1,13 @@ # frozen_string_literal: true -require 'coderay' - module Bulkrax module ApplicationHelper - include ::Hyrax::HyraxHelperBehavior if defined?(::Hyrax) + def item_entry_path(item, e, opts = {}) + an_importer?(item) ? bulkrax.importer_entry_path(item.id, e.id, opts) : bulkrax.exporter_entry_path(item.id, e.id, opts) + end + + def an_importer?(item) + item.class.to_s.include?('Importer') + end def coderay(value, opts) CodeRay diff --git a/app/jobs/bulkrax/create_relationships_job.rb b/app/jobs/bulkrax/create_relationships_job.rb index 6e9eb77b8..a0dfc44ca 100644 --- a/app/jobs/bulkrax/create_relationships_job.rb +++ b/app/jobs/bulkrax/create_relationships_job.rb @@ -98,7 +98,7 @@ def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcS if errors.present? # rubocop:disable Rails/SkipsModelValidations - importer_run.increment!(:failed_relationships, number_of_failures) + ImporterRun.update_counters(importer_run_id, failed_relationships: number_of_failures) # rubocop:enable Rails/SkipsModelValidations parent_entry&.set_status_info(errors.last, importer_run) @@ -108,7 +108,7 @@ def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcS return false # stop current job from continuing to run after rescheduling else # rubocop:disable Rails/SkipsModelValidations - Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships, number_of_successes) + ImporterRun.update_counters(importer_run_id, processed_relationships: number_of_successes) # rubocop:enable Rails/SkipsModelValidations end end @@ -158,7 +158,8 @@ def process(relationship:, importer_run_id:, parent_record:, ability:) end def add_to_collection(child_record, parent_record) - parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) + parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if + defined?(Hyrax::Adapters::NestingIndexAdapter) child_record.member_of_collections << parent_record child_record.save! end diff --git a/app/jobs/bulkrax/delete_job.rb b/app/jobs/bulkrax/delete_job.rb index 764cd8a72..1fcd04cca 100644 --- a/app/jobs/bulkrax/delete_job.rb +++ b/app/jobs/bulkrax/delete_job.rb @@ -4,17 +4,17 @@ module Bulkrax class DeleteJob < ApplicationJob queue_as :import - # rubocop:disable Rails/SkipsModelValidations def perform(entry, importer_run) obj = entry.factory.find obj&.delete - ImporterRun.find(importer_run.id).increment!(:deleted_records) - ImporterRun.find(importer_run.id).decrement!(:enqueued_records) + # rubocop:disable Rails/SkipsModelValidations + ImporterRun.increment_counter(:deleted_records, importer_run.id) + ImporterRun.decrement_counter(:enqueued_records, importer_run.id) + # rubocop:enable Rails/SkipsModelValidations entry.save! entry.importer.current_run = ImporterRun.find(importer_run.id) entry.importer.record_status entry.set_status_info("Deleted", ImporterRun.find(importer_run.id)) end - # rubocop:enable Rails/SkipsModelValidations end end diff --git a/app/jobs/bulkrax/export_work_job.rb b/app/jobs/bulkrax/export_work_job.rb index bc307f3d7..842a35c25 100644 --- a/app/jobs/bulkrax/export_work_job.rb +++ b/app/jobs/bulkrax/export_work_job.rb @@ -12,17 +12,17 @@ def perform(*args) entry.save rescue StandardError # rubocop:disable Rails/SkipsModelValidations - exporter_run.increment!(:failed_records) - exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0 + ExporterRun.increment_counter(:failed_records, args[1]) + ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0 raise else if entry.failed? - exporter_run.increment!(:failed_records) - exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0 + ExporterRun.increment_counter(:failed_records, args[1]) + ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0 raise entry.reload.current_status.error_class.constantize else - exporter_run.increment!(:processed_records) - exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0 + ExporterRun.increment_counter(:processed_records, args[1]) + ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0 end # rubocop:enable Rails/SkipsModelValidations end diff --git a/app/jobs/bulkrax/import_collection_job.rb b/app/jobs/bulkrax/import_collection_job.rb index 163ecd6f9..03405180c 100644 --- a/app/jobs/bulkrax/import_collection_job.rb +++ b/app/jobs/bulkrax/import_collection_job.rb @@ -10,13 +10,13 @@ def perform(*args) begin entry.build entry.save! - ImporterRun.find(args[1]).increment!(:processed_records) - ImporterRun.find(args[1]).increment!(:processed_collections) - ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches + ImporterRun.increment_counter(:processed_records, args[1]) + ImporterRun.increment_counter(:processed_collections, args[1]) + ImporterRun.decrement_counter(:enqueued_records, args[1]) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches rescue => e - ImporterRun.find(args[1]).increment!(:failed_records) - ImporterRun.find(args[1]).increment!(:failed_collections) - ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches + ImporterRun.increment_counter(:failed_records, args[1]) + ImporterRun.increment_counter(:failed_collections, args[1]) + ImporterRun.decrement_counter(:enqueued_records, args[1]) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches raise e end entry.importer.current_run = ImporterRun.find(args[1]) diff --git a/app/jobs/bulkrax/import_file_set_job.rb b/app/jobs/bulkrax/import_file_set_job.rb index c74ab45c0..07fc6a388 100644 --- a/app/jobs/bulkrax/import_file_set_job.rb +++ b/app/jobs/bulkrax/import_file_set_job.rb @@ -21,14 +21,14 @@ def perform(entry_id, importer_run_id) entry.build if entry.succeeded? # rubocop:disable Rails/SkipsModelValidations - ImporterRun.find(importer_run_id).increment!(:processed_records) - ImporterRun.find(importer_run_id).increment!(:processed_file_sets) + ImporterRun.increment_counter(:processed_records, importer_run_id) + ImporterRun.increment_counter(:processed_file_sets, importer_run_id) else - ImporterRun.find(importer_run_id).increment!(:failed_records) - ImporterRun.find(importer_run_id).increment!(:failed_file_sets) + ImporterRun.increment_counter(:failed_records, importer_run_id) + ImporterRun.increment_counter(:failed_file_sets, importer_run_id) # rubocop:enable Rails/SkipsModelValidations end - ImporterRun.find(importer_run_id).decrement!(:enqueued_records) unless ImporterRun.find(importer_run_id).enqueued_records <= 0 # rubocop:disable Rails/SkipsModelValidations + ImporterRun.decrement_counter(:enqueued_records, importer_run_id) unless ImporterRun.find(importer_run_id).enqueued_records <= 0 # rubocop:disable Rails/SkipsModelValidations entry.save! entry.importer.current_run = ImporterRun.find(importer_run_id) entry.importer.record_status @@ -40,7 +40,7 @@ def perform(entry_id, importer_run_id) if entry.import_attempts < 5 ImportFileSetJob.set(wait: (entry.import_attempts + 1).minutes).perform_later(entry_id, importer_run_id) else - ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations + ImporterRun.decrement_counter(:enqueued_records, importer_run_id) # rubocop:disable Rails/SkipsModelValidations entry.set_status_info(e) end end diff --git a/app/jobs/bulkrax/import_work_job.rb b/app/jobs/bulkrax/import_work_job.rb index 95258049f..a3a22ea86 100644 --- a/app/jobs/bulkrax/import_work_job.rb +++ b/app/jobs/bulkrax/import_work_job.rb @@ -23,16 +23,16 @@ def perform(entry_id, run_id, time_to_live = 3, *) entry = Entry.find(entry_id) entry.build if entry.status == "Complete" - ImporterRun.find(run_id).increment!(:processed_records) - ImporterRun.find(run_id).increment!(:processed_works) + ImporterRun.increment_counter(:processed_records, run_id) + ImporterRun.increment_counter(:processed_works, run_id) else # do not retry here because whatever parse error kept you from creating a work will likely # keep preventing you from doing so. - ImporterRun.find(run_id).increment!(:failed_records) - ImporterRun.find(run_id).increment!(:failed_works) + ImporterRun.increment_counter(:failed_records, run_id) + ImporterRun.increment_counter(:failed_works, run_id) end # Regardless of completion or not, we want to decrement the enqueued records. - ImporterRun.find(run_id).decrement!(:enqueued_records) unless ImporterRun.find(run_id).enqueued_records <= 0 + ImporterRun.decrement_counter(:enqueued_records, run_id) unless ImporterRun.find(run_id).enqueued_records <= 0 entry.save! entry.importer.current_run = ImporterRun.find(run_id) diff --git a/app/models/bulkrax/csv_collection_entry.rb b/app/models/bulkrax/csv_collection_entry.rb index c62f46dd9..ea6df6325 100644 --- a/app/models/bulkrax/csv_collection_entry.rb +++ b/app/models/bulkrax/csv_collection_entry.rb @@ -15,7 +15,7 @@ def add_identifier def add_collection_type_gid return if self.parsed_metadata['collection_type_gid'].present? - self.parsed_metadata['collection_type_gid'] = ::Hyrax::CollectionType.find_or_create_default_collection_type.gid + self.parsed_metadata['collection_type_gid'] = ::Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s end end end diff --git a/app/models/bulkrax/csv_entry.rb b/app/models/bulkrax/csv_entry.rb index aaa0bc24b..a106a74bf 100644 --- a/app/models/bulkrax/csv_entry.rb +++ b/app/models/bulkrax/csv_entry.rb @@ -88,8 +88,10 @@ def build_metadata def validate_record raise StandardError, 'Record not found' if record.nil? - raise StandardError, "Missing required elements, missing element(s) are: "\ -"#{importerexporter.parser.missing_elements(record).join(', ')}" unless importerexporter.parser.required_elements?(record) + unless importerexporter.parser.required_elements?(record) + raise StandardError, "Missing required elements, missing element(s) are: "\ +"#{importerexporter.parser.missing_elements(record).join(', ')}" + end end def add_identifier diff --git a/app/models/bulkrax/importer.rb b/app/models/bulkrax/importer.rb index 69601053e..9d88f552d 100644 --- a/app/models/bulkrax/importer.rb +++ b/app/models/bulkrax/importer.rb @@ -3,7 +3,7 @@ require 'iso8601' module Bulkrax - class Importer < ApplicationRecord + class Importer < ApplicationRecord # rubocop:disable Metrics/ClassLength include Bulkrax::ImporterExporterBehavior include Bulkrax::StatusInfo @@ -163,6 +163,14 @@ def update_files self.parser_fields['update_files'] end + def remove_and_rerun + self.parser_fields['remove_and_rerun'] + end + + def metadata_only? + parser.parser_fields['metadata_only'] == true + end + def import_works import_objects(['work']) end @@ -185,6 +193,12 @@ def import_objects(types_array = nil) self.only_updates ||= false self.save if self.new_record? # Object needs to be saved for statuses types = types_array || DEFAULT_OBJECT_TYPES + if remove_and_rerun + self.entries.find_each do |e| + e.factory.find&.destroy! + e.destroy! + end + end parser.create_objects(types) rescue StandardError => e set_status_info(e) @@ -220,9 +234,5 @@ def path_string rescue "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}" end - - def metadata_only? - parser.parser_fields['metadata_only'] == true - end end end diff --git a/app/models/concerns/bulkrax/import_behavior.rb b/app/models/concerns/bulkrax/import_behavior.rb index c095ab568..eea56afac 100644 --- a/app/models/concerns/bulkrax/import_behavior.rb +++ b/app/models/concerns/bulkrax/import_behavior.rb @@ -179,6 +179,7 @@ def factory @factory ||= of.new(attributes: self.parsed_metadata, source_identifier_value: identifier, work_identifier: parser.work_identifier, + work_identifier_search_field: parser.work_identifier_search_field, related_parents_parsed_mapping: parser.related_parents_parsed_mapping, replace_files: replace_files, user: user, diff --git a/app/parsers/bulkrax/application_parser.rb b/app/parsers/bulkrax/application_parser.rb index 97c5c1ed3..2a46d124d 100644 --- a/app/parsers/bulkrax/application_parser.rb +++ b/app/parsers/bulkrax/application_parser.rb @@ -81,6 +81,13 @@ def work_identifier @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source end + # @return [Symbol] the solr property of the source_identifier. Used for searching. + # defaults to work_identifier value + "_sim" + # @see #work_identifier + def work_identifier_search_field + @work_identifier_search_field ||= Array.wrap(get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field'))&.first&.to_s || "#{work_identifier}_sim" + end + # @return [String] def generated_metadata_mapping @generated_metadata_mapping ||= 'generated' @@ -95,7 +102,7 @@ def related_parents_raw_mapping # @return [String] # @see #related_parents_field_mapping def related_parents_parsed_mapping - @related_parents_parsed_mapping ||= (get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents') + @related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents' end # @return [String, NilClass] @@ -107,7 +114,7 @@ def related_children_raw_mapping # @return [String] # @see #related_children_raw_mapping def related_children_parsed_mapping - @related_children_parsed_mapping ||= (get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children') + @related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children' end # @api private @@ -270,8 +277,8 @@ def invalid_record(message) current_run.invalid_records ||= "" current_run.invalid_records += message current_run.save - ImporterRun.find(current_run.id).increment!(:failed_records) - ImporterRun.find(current_run.id).decrement!(:enqueued_records) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches + ImporterRun.increment_counter(:failed_records, current_run.id) + ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches end # rubocop:enable Rails/SkipsModelValidations diff --git a/app/parsers/bulkrax/csv_parser.rb b/app/parsers/bulkrax/csv_parser.rb index d859390dd..f7e34ddac 100644 --- a/app/parsers/bulkrax/csv_parser.rb +++ b/app/parsers/bulkrax/csv_parser.rb @@ -33,9 +33,10 @@ def build_records model_field_mappings.map(&:to_sym).each do |model_mapping| next unless r.key?(model_mapping) - if r[model_mapping].strip.casecmp('collection').zero? + model = r[model_mapping].nil? ? "" : r[model_mapping].strip + if model.casecmp('collection').zero? @collections << r - elsif r[model_mapping].strip.casecmp('fileset').zero? + elsif model.casecmp('fileset').zero? @file_sets << r else @works << r diff --git a/app/parsers/bulkrax/oai_dc_parser.rb b/app/parsers/bulkrax/oai_dc_parser.rb index 4116ac529..4319ab3f1 100644 --- a/app/parsers/bulkrax/oai_dc_parser.rb +++ b/app/parsers/bulkrax/oai_dc_parser.rb @@ -67,7 +67,7 @@ def create_collections metadata = { visibility: 'open' } - metadata[:collection_type_gid] = Hyrax::CollectionType.find_or_create_default_collection_type.gid if defined?(::Hyrax) + metadata[:collection_type_gid] = Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s if defined?(::Hyrax) collections.each_with_index do |set, index| next unless collection_name == 'all' || collection_name == set.spec diff --git a/app/services/bulkrax/remove_relationships_for_importer.rb b/app/services/bulkrax/remove_relationships_for_importer.rb index 784b5ed07..10fa92e40 100644 --- a/app/services/bulkrax/remove_relationships_for_importer.rb +++ b/app/services/bulkrax/remove_relationships_for_importer.rb @@ -63,7 +63,8 @@ def break_relationships! remove_relationships_from_work(obj) end - obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if defined?(Hyrax) + obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if + defined?(Hyrax::Adapters::NestingIndexAdapter) obj.save! end end diff --git a/app/views/bulkrax/entries/show.html.erb b/app/views/bulkrax/entries/show.html.erb index 0869bc998..13a875248 100644 --- a/app/views/bulkrax/entries/show.html.erb +++ b/app/views/bulkrax/entries/show.html.erb @@ -2,17 +2,17 @@

- Identifier: + <%= t('bulkrax.importer.labels.identifier') %>: <%= @entry.identifier %>

- Entry ID: + <%= t('bulkrax.importer.labels.entry_id') %>: <%= @entry.id %>

- Type: + <%= t('bulkrax.importer.labels.type') %>: <%= @entry.factory_class || 'Unknown' %>

<%= render partial: 'raw_metadata'%> @@ -23,10 +23,10 @@

<% if @importer.present? %> - Importer: + <%= t('bulkrax.importer.labels.importer') %>: <%= link_to @importer.name, importer_path(@importer) %> <% elsif @exporter.present? %> - Exporter: + <%= t('bulkrax.importer.labels.exporter') %>: <%= link_to @exporter.name, exporter_path(@exporter) %> <% end %>

diff --git a/app/views/bulkrax/exporters/_form.html.erb b/app/views/bulkrax/exporters/_form.html.erb index 4e54215b2..df3af0b38 100644 --- a/app/views/bulkrax/exporters/_form.html.erb +++ b/app/views/bulkrax/exporters/_form.html.erb @@ -33,8 +33,8 @@ label: t('bulkrax.exporter.labels.importer'), required: true, prompt: 'Select from the list', - label_html: { class: 'importer export-source-option hidden' }, - input_html: { class: 'importer export-source-option hidden form-control' }, + label_html: { class: 'importer export-source-option d-none' }, + input_html: { class: 'importer export-source-option d-none form-control' }, collection: form.object.importers_list.sort %> <%= form.input :export_source_collection, @@ -42,9 +42,9 @@ label: t('bulkrax.exporter.labels.collection'), required: true, placeholder: @collection&.title&.first, - label_html: { class: 'collection export-source-option hidden' }, + label_html: { class: 'collection export-source-option d-none' }, input_html: { - class: 'collection export-source-option hidden form-control', + class: 'collection export-source-option d-none form-control', data: { 'autocomplete-url' => '/authorities/search/collections', 'autocomplete' => 'collection' @@ -56,8 +56,8 @@ label: t('bulkrax.exporter.labels.worktype'), required: true, prompt: 'Select from the list', - label_html: { class: 'worktype export-source-option hidden' }, - input_html: { class: 'worktype export-source-option hidden form-control' }, + label_html: { class: 'worktype export-source-option d-none' }, + input_html: { class: 'worktype export-source-option d-none form-control' }, collection: Bulkrax.curation_concerns.map { |cc| [cc.to_s, cc.to_s] } %> <%= form.input :limit, @@ -80,7 +80,7 @@ as: :boolean, label: t('bulkrax.exporter.labels.filter_by_date') %> -