From e7d9c250f242aefa706c4d81930fd6137bfaec87 Mon Sep 17 00:00:00 2001 From: Seth Shaw Date: Tue, 10 Apr 2018 08:43:15 -0700 Subject: [PATCH] Apollo's first working version - Updated migrations for the new test data - Updated readme - Added view block to list images related to subjects on their pages - Fixed preceding character in apollo.csv that was causing a migration error for that file --- README.md | 14 +- data/apollo.csv | 2 +- .../migrate_plus.migration.auth_complex.yml | 41 ++++ ...migrate_plus.migration.auth_geographic.yml | 41 ++++ .../migrate_plus.migration.auth_person.yml | 41 ++++ .../migrate_plus.migration.auth_topic.yml | 41 ++++ .../migrate_plus.migration.claw_file.yml | 10 +- .../migrate_plus.migration.claw_image.yml | 20 +- .../migrate_plus.migration.claw_media.yml | 10 +- migrate_apollo/migrate_apollo.info.yml | 11 + .../migrate_plus.migration.authorities.yml | 65 ----- migrate_cdm/migrate_cdm.info.yml | 11 - ...ck.views_block__subject_images_block_1.yml | 33 +++ .../install/views.view.subject_images.yml | 232 ++++++++++++++++++ 14 files changed, 468 insertions(+), 104 deletions(-) create mode 100644 migrate_apollo/config/install/migrate_plus.migration.auth_complex.yml create mode 100644 migrate_apollo/config/install/migrate_plus.migration.auth_geographic.yml create mode 100644 migrate_apollo/config/install/migrate_plus.migration.auth_person.yml create mode 100644 migrate_apollo/config/install/migrate_plus.migration.auth_topic.yml rename migrate_cdm/config/install/migrate_plus.migration.cdm_file.yml => migrate_apollo/config/install/migrate_plus.migration.claw_file.yml (86%) rename migrate_cdm/config/install/migrate_plus.migration.cdm_image.yml => migrate_apollo/config/install/migrate_plus.migration.claw_image.yml (88%) rename migrate_cdm/config/install/migrate_plus.migration.cdm_media.yml => migrate_apollo/config/install/migrate_plus.migration.claw_media.yml (80%) create mode 100644 migrate_apollo/migrate_apollo.info.yml delete mode 100644 migrate_cdm/config/install/migrate_plus.migration.authorities.yml delete mode 100644 migrate_cdm/migrate_cdm.info.yml create mode 100644 unlv_image/config/install/block.block.views_block__subject_images_block_1.yml create mode 100644 unlv_image/config/install/views.view.subject_images.yml diff --git a/README.md b/README.md index 5313531..aa13352 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Proof of Concept: CLAW Migrate Files (Apollo Edition) -**STILL UNDER CONSTRUCTION** - This repository consists of two modules: 1. unlv_image: A local implementation of the Islandora Image module to include additional metadata fields. @@ -15,10 +13,10 @@ The source data used for this proof of concept came from the [Project Apollo Arc # Running -Note: using drush with migrate_tools is optional, but the instructions assumes it is enabled +Note: using drush with migrate_tools is optional, but the instructions assume it is installed. -1. Copy the migrate_cdm and unlv_image directories to your modules directory -2. Copy the data directory to the directory containing your drupal root (drupal root and the data directory will be parallel.) -3. Enable the modules. E.g. `drush en -y migrate_cdm` -4. Migrate the authorities. E.g. `drush mim tematres_agents` -5. Migrate the images. E.g. `drush mim --all` (Migrate will automatically skip any authorities in tematres_agents we already migrated.) +1. Copy the data directory to your drupal web root (e.g. in my tests the drupal web root is `/var/www/drupalvm/drupal/web` and the data directory is `/var/www/drupalvm/drupal/web/data`). +2. Copy the migrate_cdm and unlv_image directories to your modules directory. +3. Enable the modules. E.g. `drush en -y migrate_tools migrate_apollo`. +4. Run the migration. E.g. `drush mim --all`. +5. See a wonderful list of the newly migrated images on your Drupal site's front page! diff --git a/data/apollo.csv b/data/apollo.csv index e2c4fae..38b67b1 100644 --- a/data/apollo.csv +++ b/data/apollo.csv @@ -1,4 +1,4 @@ -AS11-36-5390,Apollo 11 Hasselblad image from film magazine 36/N - Trans-Lunar,"Neil took this picture of Buzz during their initial inspection of the LM at about 057:03. Journal Contributor David Sander notes that ""Buzz is wearing his intravehicular suit, a specially made set of garments designed to be as flame retardant as the rest of the ship, and made from the same fabric as the outer layer of the spacesuits"". Paolo Attivissimo notes that Buzz's watch reads 5:35 (Houston time), which is 57:03 GET (Ground Elapsed Time)","Aldrin, Buzz", +AS11-36-5390,Apollo 11 Hasselblad image from film magazine 36/N - Trans-Lunar,"Neil took this picture of Buzz during their initial inspection of the LM at about 057:03. Journal Contributor David Sander notes that ""Buzz is wearing his intravehicular suit, a specially made set of garments designed to be as flame retardant as the rest of the ship, and made from the same fabric as the outer layer of the spacesuits"". Paolo Attivissimo notes that Buzz's watch reads 5:35 (Houston time), which is 57:03 GET (Ground Elapsed Time)","Aldrin, Buzz", AS11-37-5528,"Apollo 11 Hasselblad image from film magazine 37/R - Orbit, Post-Landing, Post-EVA",,"Armstrong, Neil", AS11-37-5545,"Apollo 11 Hasselblad image from film magazine 37/R - Orbit, Post-Landing, Post-EVA",,,Flags--United States AS11-40-5850,Apollo 11 Hasselblad image from film magazine 40/S - EVA,"First EVA picture. Neil's first frame in a pan taken west of the ladder. Jettison bag under the Descent Stage, south footpad, bent probe, strut supports. The view is more or less up-Sun, so we are seeing the shadowed faces of boulders. 20 July 1969.",,Lunar excursion module;Moonwalk diff --git a/migrate_apollo/config/install/migrate_plus.migration.auth_complex.yml b/migrate_apollo/config/install/migrate_plus.migration.auth_complex.yml new file mode 100644 index 0000000..62cc8cc --- /dev/null +++ b/migrate_apollo/config/install/migrate_plus.migration.auth_complex.yml @@ -0,0 +1,41 @@ +id: auth_complex +label: Import MADS ComplexSubject Authorities +migration_group: authorities + +source: + plugin: url + data_fetcher_plugin: file + data_parser_plugin: xml + urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root + item_selector: /rdf:RDF/madsrdf:ComplexSubject + fields: + - + name: uri + label: 'URI' + selector: '@rdf:about' + - + name: pref_label + label: 'MADS Auth Label' + selector: 'madsrdf:authoritativeLabel' + - + name: ref_uri + label: "External Authority" + selector: 'skos:exactMatch/@rdf:resource' + + ids: + uri: + type: string + +destination: + plugin: entity:node + +process: + type: + plugin: default_value + default_value: subject + title: pref_label + field_uris: + - # Skip this field if there are no skos:exactMatch + plugin: skip_on_empty + method: process + source: ref_uri diff --git a/migrate_apollo/config/install/migrate_plus.migration.auth_geographic.yml b/migrate_apollo/config/install/migrate_plus.migration.auth_geographic.yml new file mode 100644 index 0000000..aef1bcf --- /dev/null +++ b/migrate_apollo/config/install/migrate_plus.migration.auth_geographic.yml @@ -0,0 +1,41 @@ +id: auth_geographic +label: Import MADS Geographic Authorities +migration_group: authorities + +source: + plugin: url + data_fetcher_plugin: file + data_parser_plugin: xml + urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root + item_selector: /rdf:RDF/madsrdf:Geographic + fields: + - + name: uri + label: 'URI' + selector: '@rdf:about' + - + name: pref_label + label: 'MADS Auth Label' + selector: 'madsrdf:authoritativeLabel' + - + name: ref_uri + label: "External Authority" + selector: 'skos:exactMatch/@rdf:resource' + + ids: + uri: + type: string + +destination: + plugin: entity:node + +process: + type: + plugin: default_value + default_value: subject + title: pref_label + field_uris: + - # Skip this field if there are no skos:exactMatch + plugin: skip_on_empty + method: process + source: ref_uri diff --git a/migrate_apollo/config/install/migrate_plus.migration.auth_person.yml b/migrate_apollo/config/install/migrate_plus.migration.auth_person.yml new file mode 100644 index 0000000..89f5406 --- /dev/null +++ b/migrate_apollo/config/install/migrate_plus.migration.auth_person.yml @@ -0,0 +1,41 @@ +id: auth_person +label: Import MADS PersonalName Authorities +migration_group: authorities + +source: + plugin: url + data_fetcher_plugin: file + data_parser_plugin: xml + urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root + item_selector: /rdf:RDF/madsrdf:PersonalName + fields: + - + name: uri + label: 'URI' + selector: '@rdf:about' + - + name: pref_label + label: 'MADS Auth Label' + selector: 'madsrdf:authoritativeLabel' + - + name: ref_uri + label: "External Authority" + selector: 'skos:exactMatch/@rdf:resource' + + ids: + uri: + type: string + +destination: + plugin: entity:node + +process: + type: + plugin: default_value + default_value: person + title: pref_label + field_uris: + - # Skip this field if there are no skos:exactMatch + plugin: skip_on_empty + method: process + source: ref_uri diff --git a/migrate_apollo/config/install/migrate_plus.migration.auth_topic.yml b/migrate_apollo/config/install/migrate_plus.migration.auth_topic.yml new file mode 100644 index 0000000..5321a8e --- /dev/null +++ b/migrate_apollo/config/install/migrate_plus.migration.auth_topic.yml @@ -0,0 +1,41 @@ +id: auth_topic +label: Import MADS Topic Authorities +migration_group: authorities + +source: + plugin: url + data_fetcher_plugin: file + data_parser_plugin: xml + urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root + item_selector: /rdf:RDF/madsrdf:Topic + fields: + - + name: uri + label: 'URI' + selector: '@rdf:about' + - + name: pref_label + label: 'MADS Auth Label' + selector: 'madsrdf:authoritativeLabel' + - + name: ref_uri + label: "External Authority" + selector: 'skos:exactMatch/@rdf:resource' + + ids: + uri: + type: string + +destination: + plugin: entity:node + +process: + type: + plugin: default_value + default_value: subject + title: pref_label + field_uris: + - # Skip this field if there are no skos:exactMatch + plugin: skip_on_empty + method: process + source: ref_uri diff --git a/migrate_cdm/config/install/migrate_plus.migration.cdm_file.yml b/migrate_apollo/config/install/migrate_plus.migration.claw_file.yml similarity index 86% rename from migrate_cdm/config/install/migrate_plus.migration.cdm_file.yml rename to migrate_apollo/config/install/migrate_plus.migration.claw_file.yml index ce4c8fc..4deace4 100644 --- a/migrate_cdm/config/install/migrate_plus.migration.cdm_file.yml +++ b/migrate_apollo/config/install/migrate_plus.migration.claw_file.yml @@ -1,20 +1,20 @@ -id: cdm_file_import -label: Import CDM Image Files +id: claw_file +label: Import Image Files migration_group: test_digital source: plugin: csv - path: '../data/source.csv' # Path relative to Drupal site root + path: 'data/apollo.csv' # Path relative to Drupal site root delimiter: ',' header_row_count: 0 # No headers, 1 if there are headers keys: - digital_id constants: - source_base_dir: '../data/images' + source_base_dir: 'data/images' collection_alias: 'apollo' dest_base_dir: 'public://originals' extension: 'tiff' - column_names: + column_names: 0: digital_id: 'Digital ID' # basename of the file 1: diff --git a/migrate_cdm/config/install/migrate_plus.migration.cdm_image.yml b/migrate_apollo/config/install/migrate_plus.migration.claw_image.yml similarity index 88% rename from migrate_cdm/config/install/migrate_plus.migration.cdm_image.yml rename to migrate_apollo/config/install/migrate_plus.migration.claw_image.yml index c6e6665..1cfa622 100644 --- a/migrate_cdm/config/install/migrate_plus.migration.cdm_image.yml +++ b/migrate_apollo/config/install/migrate_plus.migration.claw_image.yml @@ -1,15 +1,19 @@ -id: cdm_image_import +id: claw_image label: Import CDM Images migration_group: test_digital migration_dependencies: required: - - cdm_media_import - - authorities # Loading authorities first allows us to look them up + - claw_media + # Loading authorities first allows us to look them up + - auth_person + - auth_complex + - auth_geographic + - auth_topic source: plugin: csv - path: '../data/source.csv' # Path relative to Drupal site root + path: 'data/apollo.csv' # Path relative to Drupal site root delimiter: ',' header_row_count: 0 # No headers, 1 if there are headers keys: @@ -62,12 +66,9 @@ process: type: person temp_subjects: # Temporary field of subjects - - - plugin: concat # Multiple CSV fields include subjects - delimiter: ';' - source: subjects - plugin: skip_on_empty + source: subjects method: process - plugin: explode @@ -80,6 +81,7 @@ process: entity_type: node default_values: type: subject + field_subjects: # Gather temp arrays into the destination field - plugin: get @@ -92,7 +94,7 @@ process: # Now the TIFF entity references field_tiff/target_id: plugin: migration_lookup - migration: cdm_media_import + migration: claw_media source: digital_id no_stub: true field_tiff/alt: title diff --git a/migrate_cdm/config/install/migrate_plus.migration.cdm_media.yml b/migrate_apollo/config/install/migrate_plus.migration.claw_media.yml similarity index 80% rename from migrate_cdm/config/install/migrate_plus.migration.cdm_media.yml rename to migrate_apollo/config/install/migrate_plus.migration.claw_media.yml index b24fd87..2dc72e1 100644 --- a/migrate_cdm/config/install/migrate_plus.migration.cdm_media.yml +++ b/migrate_apollo/config/install/migrate_plus.migration.claw_media.yml @@ -1,14 +1,14 @@ -id: cdm_media_import +id: claw_media label: Import CDM Image Media migration_group: test_digital migration_dependencies: required: - - cdm_file_import + - claw_file source: plugin: csv - path: '../data/source.csv' # Path relative to Drupal site root + path: 'data/apollo.csv' # Path relative to Drupal site root delimiter: ',' header_row_count: 0 # No headers, 1 if there are headers keys: @@ -20,13 +20,13 @@ source: process: mid: plugin: migration_lookup - migration: cdm_file_import + migration: claw_file source: digital_id no_stub: true field_file/target_id: plugin: migration_lookup - migration: cdm_file_import + migration: claw_file source: digital_id no_stub: true diff --git a/migrate_apollo/migrate_apollo.info.yml b/migrate_apollo/migrate_apollo.info.yml new file mode 100644 index 0000000..e72be36 --- /dev/null +++ b/migrate_apollo/migrate_apollo.info.yml @@ -0,0 +1,11 @@ +name: 'Migrate Islandora CLAW Images: Apollo Demo' +description: Migrates Master Tiff Images into an Islandora Claw Image derivative using Apollo 11 mission photos as an example. +package: custom +type: module +core: 8.x + +dependencies: + - drupal:migrate + - migrate_plus:migrate_plus + - migrate_source_csv + - unlv_image diff --git a/migrate_cdm/config/install/migrate_plus.migration.authorities.yml b/migrate_cdm/config/install/migrate_plus.migration.authorities.yml deleted file mode 100644 index f955ace..0000000 --- a/migrate_cdm/config/install/migrate_plus.migration.authorities.yml +++ /dev/null @@ -1,65 +0,0 @@ -id: authorities -label: Import MADS Authorities -migration_group: authorities - -source: - plugin: url - data_fetcher_plugin: file - data_parser_plugin: xml - urls: '../data/small-auth-set.xml' # Path relative to Drupal site root - item_selector: /rdf:RDF/madsrdf:PersonalName - fields: - - - name: uri - label: 'TemaTres URI' - selector: '@rdf:about' - - - name: pref_label - label: 'MADS Auth Label' - selector: 'madsrdf:authoritativeLabel' - - - name: type - label: 'Type URI' - selector: 'madsrdf:identifiesRWO/madsrdf:RWO/rdf:type' - ids: - uri: - type: string - -destination: - plugin: entity:node - -process: - type: - - - plugin: skip_on_empty - source: type - method: row - message: 'Type not supported' - - - plugin: static_map - default_value: '' - map: - 'http://id.loc.gov/ontologies/bibframe/Person': person - '5844': corporate_body - '5843': corporate_body - - # Mapping results in empty value (probably the collection type) - plugin: skip_on_empty - method: row - message: 'TemaTres Agent Type is not supported.' - title: pref_label - field_uris: - - # Skip this field if there are no skos:exactMatch - plugin: skip_on_empty - method: process - source: uri - field_relation: - - - plugin: skip_on_empty - method: process - source: related_uri - message: 'No Relations found for this Agent' - - - plugin: migration_lookup - migration: tematres_agents - no_stub: true # Stubbing doesn't work so just run a migration update to get the relations on the second pass - # source: related_uri diff --git a/migrate_cdm/migrate_cdm.info.yml b/migrate_cdm/migrate_cdm.info.yml deleted file mode 100644 index cbf33bb..0000000 --- a/migrate_cdm/migrate_cdm.info.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Migrate CONTENTdm TSV Module -description: Migrates a CONTENTdm collection TSV -package: custom -type: module -core: 8.x - -dependencies: - - drupal:migrate - - migrate_plus:migrate_plus - - migrate_source_csv - - unlv_image diff --git a/unlv_image/config/install/block.block.views_block__subject_images_block_1.yml b/unlv_image/config/install/block.block.views_block__subject_images_block_1.yml new file mode 100644 index 0000000..3d224bf --- /dev/null +++ b/unlv_image/config/install/block.block.views_block__subject_images_block_1.yml @@ -0,0 +1,33 @@ +langcode: en +status: true +dependencies: + config: + - views.view.subject_images + module: + - node + - views + theme: + - bartik +id: views_block__subject_images_block_1 +theme: bartik +region: content +weight: -2 +provider: null +plugin: 'views_block:subject_images-block_1' +settings: + id: 'views_block:subject_images-block_1' + label: Images + provider: views + label_display: visible + views_label: Images + items_per_page: none + context_mapping: { } +visibility: + node_type: + id: node_type + bundles: + person: person + subject: subject + negate: false + context_mapping: + node: '@node.node_route_context:node' diff --git a/unlv_image/config/install/views.view.subject_images.yml b/unlv_image/config/install/views.view.subject_images.yml new file mode 100644 index 0000000..d97bd26 --- /dev/null +++ b/unlv_image/config/install/views.view.subject_images.yml @@ -0,0 +1,232 @@ +langcode: en +status: true +dependencies: + config: + - node.type.unlv_image + module: + - node + - user +id: subject_images +label: subject_images +module: views +description: '' +tag: '' +base_table: node_field_data +base_field: nid +core: 8.x +display: + default: + display_plugin: default + id: default + display_title: Master + position: 0 + display_options: + access: + type: perm + options: + perm: 'access content' + cache: + type: tag + options: { } + query: + type: views_query + options: + disable_sql_rewrite: false + distinct: false + replica: false + query_comment: '' + query_tags: { } + exposed_form: + type: basic + options: + submit_button: Apply + reset_button: false + reset_button_label: Reset + exposed_sorts_label: 'Sort by' + expose_sort_order: true + sort_asc_label: Asc + sort_desc_label: Desc + pager: + type: some + options: + items_per_page: 10 + offset: 0 + style: + type: default + row: + type: fields + fields: + title: + id: title + table: node_field_data + field: title + settings: + link_to_entity: true + plugin_id: field + relationship: none + group_type: group + admin_label: '' + label: '' + exclude: false + alter: + alter_text: false + text: '' + make_link: false + path: '' + absolute: false + external: false + replace_spaces: false + path_case: none + trim_whitespace: false + alt: '' + rel: '' + link_class: '' + prefix: '' + suffix: '' + target: '' + nl2br: false + max_length: 0 + word_boundary: true + ellipsis: true + more_link: false + more_link_text: '' + more_link_path: '' + strip_tags: false + trim: false + preserve_tags: '' + html: false + element_type: '' + element_class: '' + element_label_type: '' + element_label_class: '' + element_label_colon: true + element_wrapper_type: '' + element_wrapper_class: '' + element_default_classes: true + empty: '' + hide_empty: false + empty_zero: false + hide_alter_empty: true + click_sort_column: value + type: string + group_column: value + group_columns: { } + group_rows: true + delta_limit: 0 + delta_offset: 0 + delta_reversed: false + delta_first_last: false + multi_type: separator + separator: ', ' + field_api_classes: false + filters: + status: + value: '1' + table: node_field_data + field: status + plugin_id: boolean + entity_type: node + entity_field: status + id: status + expose: + operator: '' + group: 1 + type: + id: type + table: node_field_data + field: type + value: + unlv_image: unlv_image + entity_type: node + entity_field: type + plugin_id: bundle + sorts: + title: + id: title + table: node_field_data + field: title + order: ASC + entity_type: node + entity_field: title + plugin_id: standard + relationship: none + group_type: group + admin_label: '' + exposed: false + expose: + label: '' + title: person_images + header: { } + footer: { } + empty: { } + relationships: + field_subjects: + id: field_subjects + table: node__field_subjects + field: field_subjects + relationship: none + group_type: group + admin_label: 'field_subjects: Content' + required: true + plugin_id: standard + arguments: + field_subjects_target_id: + id: field_subjects_target_id + table: node__field_subjects + field: field_subjects_target_id + relationship: none + group_type: group + admin_label: '' + default_action: default + exception: + value: all + title_enable: false + title: All + title_enable: false + title: '' + default_argument_type: node + default_argument_options: { } + default_argument_skip_url: false + summary_options: + base_path: '' + count: true + items_per_page: 25 + override: false + summary: + sort_order: asc + number_of_records: 0 + format: default_summary + specify_validation: false + validate: + type: none + fail: 'not found' + validate_options: { } + break_phrase: false + not: false + plugin_id: numeric + display_extenders: { } + cache_metadata: + max-age: -1 + contexts: + - 'languages:language_content' + - 'languages:language_interface' + - url + - 'user.node_grants:view' + - user.permissions + tags: { } + block_1: + display_plugin: block + id: block_1 + display_title: Block + position: 1 + display_options: + display_extenders: { } + cache_metadata: + max-age: -1 + contexts: + - 'languages:language_content' + - 'languages:language_interface' + - url + - 'user.node_grants:view' + - user.permissions + tags: { }