fix empty subcats and reharvest

.gitignore

@@ -23,3 +23,4 @@ Untitled*
 http_cache.sqlite
 data-rocrate
 subject-hierarchy.md
+ro-crate-metadata.json.v1.0.0

.zenodo.json

@@ -5,7 +5,7 @@
   "related_identifiers": [
     {
       "scheme": "url",
-      "identifier": "https://github.com/GLAM-Workbench/trove-web-archives/tree/v1.0.0",
+      "identifier": "https://github.com/GLAM-Workbench/trove-web-archives/tree/v1.0.1",
       "relation": "isDerivedFrom",
       "resource_type": "software"
     },
@@ -22,14 +22,14 @@
       "resource_type": "other"
     }
   ],
-  "version": "v1.0.0",
+  "version": "v1.0.1",
   "upload_type": "software",
   "keywords": [
     "digital humanities",
     "Jupyter",
     "GLAM Workbench"
   ],
-  "publication_date": "2024-05-06",
+  "publication_date": "2024-05-07",
   "creators": [
     {
       "name": "Sherratt, Tim",

create-datasets.ipynb

@@ -64,7 +64,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "9362f044-66c3-44b4-b4b4-8566603bb11d",
    "metadata": {
     "editable": true,
@@ -73,18 +73,7 @@
     },
     "tags": []
    },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 1,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "import os\n",
     "from datetime import datetime\n",
@@ -102,7 +91,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "1ab9af6c-01d7-459b-8595-31325aeb3558",
    "metadata": {
     "editable": true,
@@ -138,7 +127,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "cb8497ee-fae1-4252-bdb4-5fe1ab6fb12a",
    "metadata": {
     "editable": true,
@@ -212,7 +201,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "a9895f20-1891-4c5e-8bfe-bb3ef54f023f",
    "metadata": {
     "editable": true,
@@ -223,23 +212,10 @@
      "nbval-skip"
     ]
    },
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "Download dataset: <a href='datasets/pandora-subject-3-business-economy.zip', download>datasets/pandora-subject-3-business-economy.zip</a>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "create_subject_dataset(\n",
-    "    \"/subject/3\", include_subcategories=True, include_collections=True\n",
+    "    \"/subject/5\", include_subcategories=True, include_collections=True\n",
     ")"
    ]
   },
@@ -253,7 +229,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "a1f8b140-1453-4b1f-ba7f-6bdb3e77b11f",
    "metadata": {
     "editable": true,
@@ -297,7 +273,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "f9dff1c6-4305-4d87-976e-a11f72a914e2",
    "metadata": {
     "editable": true,
@@ -308,27 +284,14 @@
      "nbval-skip"
     ]
    },
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "Download dataset: <a href='datasets/pandora-col-21326-museums.zip', download>datasets/pandora-col-21326-museums.zip</a>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "create_collection_dataset(\"/col/21326\", include_subcollections=True)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "ea3b5a17-9b47-4c48-a471-7b42c5948371",
    "metadata": {
     "editable": true,
@@ -340,32 +303,7 @@
     },
     "tags": []
    },
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "Download dataset: <a href='datasets/pandora-subject-3-business-economy.zip', download>datasets/pandora-subject-3-business-economy.zip</a>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       "Download dataset: <a href='datasets/pandora-col-21326-museums.zip', download>datasets/pandora-col-21326-museums.zip</a>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "# IGNORE CELL -- TESTING ONLY\n",
     "\n",

harvest-pandora-subject-collections.ipynb

@@ -94,7 +94,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": null,
    "id": "1d6f6094-ce03-48d7-9ef2-7cdac14dba11",
    "metadata": {
     "editable": true,
@@ -177,6 +177,7 @@
     "                            \"id\": subject_id,\n",
     "                            \"collections\": collection_ids,\n",
     "                            \"titles\": title_ids,\n",
+    "                            \"subcategories\": []\n",
     "                        }\n",
     "                    )\n",
     "                    + \"\\n\"\n",

ro-crate-metadata.json

@@ -9,7 +9,7 @@
                     "@id": "https://orcid.org/0000-0001-7956-4498"
                 }
             ],
-            "datePublished": "2024-05-06",
+            "datePublished": "2024-05-07",
             "description": "This repository includes information on finding, understanding, and using Pandora's collections of archived web pages.\n\n[Pandora](http://pandora.nla.gov.au/) has been selecting web sites and online resources for preservation since 1996. It has assembled a collection of more than 80,000 titles, organised into subjects and collections. The archived websites are now part of the Australian Web Archive (AWA), which combines the selected titles with broader domain harvests, and is searchable through Trove. However, Pandora's curated collections offer a useful entry point for researchers trying to find web sites relating to particular topics or events.\n\nThe [Web Archives](https://glam-workbench.net/web-archives/) section of the GLAM Workbench provides documentation, tools, and examples to help you work with data from a range of web archives, including the Australian Web Archive. The title urls obtained through Pandora can be used to obtain additional data from the AWA for analysis.",
             "hasPart": [
                 {
@@ -39,7 +39,7 @@
             },
             "name": "trove-web-archives",
             "url": "https://github.com/GLAM-Workbench/trove-web-archives/",
-            "version": "v1.0.0"
+            "version": "v1.0.1"
         },
         {
             "@id": "ro-crate-metadata.json",
@@ -89,8 +89,8 @@
                 "File",
                 "Dataset"
             ],
-            "contentSize": 13200346,
-            "dateModified": "2024-05-03",
+            "contentSize": 13201943,
+            "dateModified": "2024-05-07",
             "encodingFormat": "text/csv",
             "isPartOf": {
                 "@id": "https://github.com/GLAM-Workbench/trove-web-archives-titles"
@@ -99,8 +99,8 @@
                 "@id": "https://creativecommons.org/publicdomain/zero/1.0/"
             },
             "name": "pandora-titles.csv",
-            "sdDatePublished": "2024-05-06",
-            "size": 87741,
+            "sdDatePublished": "2024-05-07",
+            "size": 87757,
             "url": "https://github.com/GLAM-Workbench/trove-web-archives-titles/raw/main/pandora-titles.csv"
         },
         {
@@ -109,16 +109,16 @@
                 "File",
                 "Dataset"
             ],
-            "contentSize": 1901413,
-            "dateModified": "2024-05-02",
+            "contentSize": 1904644,
+            "dateModified": "2024-05-07",
             "isPartOf": {
                 "@id": "https://github.com/GLAM-Workbench/trove-web-archives-collections"
             },
             "license": {
                 "@id": "https://creativecommons.org/publicdomain/zero/1.0/"
             },
             "name": "pandora-subjects.ndjson",
-            "sdDatePublished": "2024-05-06",
+            "sdDatePublished": "2024-05-07",
             "size": 149,
             "url": "https://github.com/GLAM-Workbench/trove-web-archives-collections/raw/main/pandora-subjects.ndjson"
         },
@@ -128,16 +128,16 @@
                 "File",
                 "Dataset"
             ],
-            "contentSize": 844738,
-            "dateModified": "2024-05-02",
+            "contentSize": 844866,
+            "dateModified": "2024-05-07",
             "isPartOf": {
                 "@id": "https://github.com/GLAM-Workbench/trove-web-archives-collections"
             },
             "license": {
                 "@id": "https://creativecommons.org/publicdomain/zero/1.0/"
             },
             "name": "pandora-collections.ndjson",
-            "sdDatePublished": "2024-05-06",
+            "sdDatePublished": "2024-05-07",
             "size": 1920,
             "url": "https://github.com/GLAM-Workbench/trove-web-archives-collections/raw/main/pandora-collections.ndjson"
         },
@@ -307,7 +307,7 @@
             "actionStatus": {
                 "@id": "http://schema.org/CompletedActionStatus"
             },
-            "endDate": "2024-05-02",
+            "endDate": "2024-05-07",
             "instrument": {
                 "@id": "harvest-pandora-subject-collections.ipynb"
             },
@@ -356,7 +356,7 @@
             "actionStatus": {
                 "@id": "http://schema.org/CompletedActionStatus"
             },
-            "endDate": "2024-05-03",
+            "endDate": "2024-05-07",
             "instrument": {
                 "@id": "harvest-pandora-titles.ipynb"
             },
@@ -375,6 +375,15 @@
             },
             "endDate": "2024-05-06",
             "name": "Create version v1.0.0"
+        },
+        {
+            "@id": "#create_version_v1_0_1",
+            "@type": "UpdateAction",
+            "actionStatus": {
+                "@id": "http://schema.org/CompletedActionStatus"
+            },
+            "endDate": "2024-05-07",
+            "name": "Create version v1.0.1"
         }
     ]
 }