Update for Islandora 1.x (#7)

seth-shaw-unlv · Sep 18, 2019 · bcd8b97 · bcd8b97
1 parent 824baf4
commit bcd8b97
Show file tree

Hide file tree

Showing 70 changed files with 92 additions and 2,290 deletions.
diff --git a/README.md b/README.md
@@ -1,11 +1,9 @@
-# Proof of Concept: CLAW Migrate Files (Apollo Edition)
+# Proof of Concept: Islandora 8 Migrate Files (Apollo Edition)
 
 This repository consists of two modules:
 
-1. unlv_image: A local implementation of the Islandora Image module to include additional metadata fields.
-2. migrate_cdm: Uses the Migrate API to load Tiff masters, metadata from a CSV, and MADS authority records from the Library of Congress.
-
-*Note:* This proof-of-concept assumes that new content types will be created for each metadata profile/object type pair; ergo, this proof-of-concept includes a new content type called UNLV_image which parallel's islandora_image but adds some node entity references. However, CLAW is under active development which will cause this example to break from time to time. I intend to continue updating this example as the current dev-version of CLAW develops.
+1. migrate_apollo: Uses the Migrate API to load Tiff masters, metadata from a CSV, and MADS RDF XML authority records from the Library of Congress.
+2. data: the source data we will migrate (see below).
 
 # Source Data
 
@@ -15,11 +13,11 @@ The source data used for this proof of concept came from the [Project Apollo Arc
 
 Note: using drush with migrate_tools is optional, but the instructions assume it is installed.
 
-0. Install the prerequisite modules (islandora_image, migrate_plus, and migrate_source_csv) and their dependencies. E.g. `composer require islandora/islandora_image drupal/migrate_tools:^4.0 drupal/migrate_source_csv`.
-0. Copy the data directory to your drupal web root (e.g. in my tests the drupal web root is `/var/www/drupalvm/drupal/web` and the data directory is `/var/www/drupalvm/drupal/web/data`).
-0. Copy the migrate_cdm and unlv_image directories to your modules directory.
-0. Enable the modules. E.g. `drush en -y migrate_tools migrate_apollo`.
-0. Run the migration. E.g. `drush -l http://localhost:8000 mim --userid=1 --all`. *Note: drush must be run by the webserver user because the claw_file migration copies files to the "public://directory". E.g.* `sudo -u www-data drush -l http://localhost:8000 mim --userid=1 --all` *if you are using vagrant. Also, the userid flag is specific to the migrate:import command, it provides the necessary user information to the JWT authentication to enable derivatives.*
+0. Install Islandora 8, including islandora_defaults.
+0. Copy the data directory to your drupal web root (e.g. for islandora the default drupal web root is `/var/www/html/drupal/web` and the data directory is `/var/www/html/drupal/web/data`).
+0. Clone this git repo to your modules directory. (`git clone https://github.com/seth-shaw-unlv/claw-migrate-files-poc.git`)
+0. Enable the modules. E.g. `drush en -y migrate_apollo`.
+0. Run the migration. E.g. `drush -l http://localhost:8000 mim --userid=1 --all`. *Note: drush must be run by the webserver user because the claw_file migration copies files to the "public://directory". E.g.* `sudo -u www-data drush -l http://localhost:8000 mim --userid=1 --all` *if you are using Ubuntu. Also, the userid flag is specific to the migrate:import command, it provides the necessary user information to the JWT authentication to enable derivatives.*
 0. See a wonderful list of the newly migrated images on your Drupal site's front page!
 
 # Combining People and Subject entities in a single column

diff --git a/...l/migrate_plus.migration.auth_complex.yml → ...l/migrate_plus.migration.auth_complex.yml b/...l/migrate_plus.migration.auth_complex.yml → ...l/migrate_plus.migration.auth_complex.yml
@@ -4,6 +4,8 @@ migration_group: authorities
 
 source:
   plugin: url
+  constants:
+    source: lcsh
   data_fetcher_plugin: file
   data_parser_plugin: xml
   urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root
@@ -27,15 +29,12 @@ source:
       type: string
 
 destination:
-  plugin: entity:node
+  plugin: entity:taxonomy_term
 
 process:
-  type:
+  vid:
     plugin: default_value
     default_value: subject
-  title: pref_label
-  field_uris:
-    - # Skip this field if there are no skos:exactMatch
-      plugin: skip_on_empty
-      method: process
-      source: ref_uri
+  name: pref_label
+  field_authority_link/uri: uri
+  field_authority_link/source: constants/source 
diff --git a/...igrate_plus.migration.auth_geographic.yml → ...igrate_plus.migration.auth_geographic.yml b/...igrate_plus.migration.auth_geographic.yml → ...igrate_plus.migration.auth_geographic.yml
@@ -4,6 +4,8 @@ migration_group: authorities
 
 source:
   plugin: url
+  constants:
+    source: lcnaf
   data_fetcher_plugin: file
   data_parser_plugin: xml
   urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root
@@ -27,15 +29,29 @@ source:
       type: string
 
 destination:
-  plugin: entity:node
+  plugin: entity:taxonomy_term
 
 process:
-  type:
+  vid:
     plugin: default_value
-    default_value: subject
-  title: pref_label
-  field_uris:
-    - # Skip this field if there are no skos:exactMatch
-      plugin: skip_on_empty
-      method: process
-      source: ref_uri
+    default_value: geo_location
+  name: pref_label
+  temp_authority_source:
+    plugin: str_replace
+    regex: true
+    source: uri
+    search: '#http://id\.loc\.gov/authorities/(\w+)/.*#'
+    replace: "$1"
+  field_authority_link/uri: uri
+  field_authority_link/source:
+    plugin: static_map
+    source: '@temp_authority_source'
+    map:
+      names: lcnaf
+      subjects: lcsh
+  field_authority_link/title:
+    plugin: static_map
+    source: '@temp_authority_source'
+    map:
+      names: 'Library of Congress Name Authority File'
+      subjects: 'Library of Congress Subject Headings'
diff --git a/...ll/migrate_plus.migration.auth_person.yml → ...ll/migrate_plus.migration.auth_person.yml b/...ll/migrate_plus.migration.auth_person.yml → ...ll/migrate_plus.migration.auth_person.yml
@@ -4,6 +4,8 @@ migration_group: authorities
 
 source:
   plugin: url
+  constants:
+    source: lcnaf
   data_fetcher_plugin: file
   data_parser_plugin: xml
   urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root
@@ -27,15 +29,12 @@ source:
       type: string
 
 destination:
-  plugin: entity:node
+  plugin: entity:taxonomy_term
 
 process:
-  type:
+  vid:
     plugin: default_value
     default_value: person
-  title: pref_label
-  field_uris:
-    - # Skip this field if there are no skos:exactMatch
-      plugin: skip_on_empty
-      method: process
-      source: ref_uri
+  name: pref_label
+  field_authority_link/uri: uri
+  field_authority_link/source: constants/source
diff --git a/...all/migrate_plus.migration.auth_topic.yml → ...all/migrate_plus.migration.auth_topic.yml b/...all/migrate_plus.migration.auth_topic.yml → ...all/migrate_plus.migration.auth_topic.yml
@@ -4,6 +4,8 @@ migration_group: authorities
 
 source:
   plugin: url
+  constants:
+    source: lcsh
   data_fetcher_plugin: file
   data_parser_plugin: xml
   urls: 'data/apollo-auths.rdf' # Path relative to Drupal site root
@@ -27,15 +29,12 @@ source:
       type: string
 
 destination:
-  plugin: entity:node
+  plugin: entity:taxonomy_term
 
 process:
-  type:
+  vid:
     plugin: default_value
     default_value: subject
-  title: pref_label
-  field_uris:
-    - # Skip this field if there are no skos:exactMatch
-      plugin: skip_on_empty
-      method: process
-      source: ref_uri
+  name: pref_label
+  field_authority_link/uri: uri
+  field_authority_link/source: constants/source
diff --git a/...tall/migrate_plus.migration.claw_file.yml → ...igrate_plus.migration.islandora8_file.yml b/...tall/migrate_plus.migration.claw_file.yml → ...igrate_plus.migration.islandora8_file.yml
@@ -1,7 +1,6 @@
-uuid: 98014998-7099-496d-8387-4136fc85de7f
-id: claw_file
+id: islandora8_file
 label: Import Image Files
-migration_group: test_digital
+migration_group: apollo
 
 source:
   plugin: csv

diff --git a/...all/migrate_plus.migration.claw_media.yml → ...grate_plus.migration.islandora8_media.yml b/...all/migrate_plus.migration.claw_media.yml → ...grate_plus.migration.islandora8_media.yml
@@ -1,11 +1,11 @@
-id: claw_media
-label: Import CDM Image Media
-migration_group: test_digital
+id: islandora8_media
+label: Import Media
+migration_group: apollo
 
 migration_dependencies:
   required:
-    - claw_file
-    - claw_image # so we can lookup the value for field_media_of
+    - islandora8_files
+    - islandora8_metadata # so we can lookup the value for field_media_of
 
 source:
   plugin: csv
@@ -15,7 +15,7 @@ source:
   keys:
     - digital_id
   constants:
-    preservation_master: 'Preservation Master'
+    media_use: 'Original File'
     uid: 1 # UID of Admin user, may be changed to uid of someone with permission to create items
   column_names:
     0:
@@ -33,7 +33,7 @@ process:
   # Lookup the Tiff we just migrated
   field_media_file/target_id:
     plugin: migration_lookup
-    migration: claw_file
+    migration: islandora8_file
     source: digital_id
     no_stub: true
   field_media_file/display:
@@ -43,20 +43,20 @@ process:
     plugin: default_value
     default_value: ''
 
-  # Lookup the UNLV_Image we just created
+  # Lookup the metadata record we just created
   field_media_of:
     plugin: migration_lookup
-    migration: claw_image
+    migration: islandora8_metadata
     source: digital_id
     no_stub: true
 
   # Set as Preservation Master
-  field_tags:
+  field_media_use:
     plugin: entity_lookup
-    source: constants/preservation_master
+    source: constants/media_use
     value_key: name
     bundle_key: vid
-    bundle: tags
+    bundle: islandora_media_use
     entity_type: taxonomy_term
     ignore_case: true
 

diff --git a/...all/migrate_plus.migration.claw_image.yml → ...te_plus.migration.islandora8_metadata.yml b/...all/migrate_plus.migration.claw_image.yml → ...te_plus.migration.islandora8_metadata.yml
@@ -1,6 +1,6 @@
-id: claw_image
-label: Import CDM Images
-migration_group: test_digital
+id: islandora8_metadata
+label: 'Import Metadata'
+migration_group: apollo
 
 migration_dependencies:
   required:
@@ -22,15 +22,15 @@ source:
     image: 'Image'
     uid: 1 # UID of Admin user, may be changed to uid of someone with permission to create items
   column_names:
-    0:
+    -
       digital_id: 'Digital ID'
-    1:
+    -
       title: 'Title'
-    2:
+    -
       description: 'Description'
-    3:
+    -
       subject_person: 'Identified Individual'
-    4:
+    -
       subjects: 'Subjects'
 
 destination: # We're creating nodes, ya'll.
@@ -39,11 +39,11 @@ destination: # We're creating nodes, ya'll.
 process:
   type: # The content type of the nodes we are creating
     plugin: default_value
-    default_value: unlv_image
+    default_value: islandora_object
 
   # One-to-One mappings
   uid: constants/uid
-  field_digital_id: digital_id
+  field_identifier: digital_id
   title: title
   field_description: description
 
@@ -56,17 +56,17 @@ process:
       - digital_id
 
   # Type Tags
-  field_tags:
+  field_model:
     plugin: entity_lookup
     source: constants/image
     value_key: name
     bundle_key: vid
-    bundle: tags
+    bundle: islandora_models
     entity_type: taxonomy_term
     ignore_case: true
 
 # SUBJECTS
-  # Since subjects can be of multiple content types we need to perform
+  # Since subjects can be in multiple vocabularies we need to perform
   # lookups for each type, assign them to a temp array, and recombine
   # them all before assigning them to the appropriate entity reference field.
 
@@ -80,12 +80,12 @@ process:
       delimiter: ';'
     -
       plugin: entity_generate
-      value_key: title
-      bundle_key: type
+      value_key: name
+      bundle_key: vid
       bundle: person
-      entity_type: node
+      entity_type: taxonomy_term
       default_values:
-        type: person
+        vid: person
 
   temp_subjects: # Temporary field of subjects
     -
@@ -97,12 +97,12 @@ process:
       delimiter: ';'
     -
       plugin: entity_generate
-      value_key: title
+      value_key: name
       bundle: subject
-      bundle_key: type
-      entity_type: node
+      bundle_key: vid
+      entity_type: taxonomy_term
       default_values:
-        type: subject
+        vid: subject
 
   field_subjects: # Gather temp arrays into the destination field
     -

diff --git a/migrate_apollo.features.yml b/migrate_apollo.features.yml
@@ -0,0 +1,2 @@
+bundle: islandora
+required: true
diff --git a/migrate_apollo/migrate_apollo.info.yml → migrate_apollo.info.yml b/migrate_apollo/migrate_apollo.info.yml → migrate_apollo.info.yml
@@ -8,4 +8,4 @@ dependencies:
   - drupal:migrate
   - migrate_plus:migrate_plus
   - migrate_source_csv
-  - unlv_image
+  - islandora_defaults
diff --git a/migrate_apollo/drush.services.yml b/migrate_apollo/drush.services.yml