Skip to content

Commit

Permalink
Merge pull request #1811 from mbana641/mbana641/kg_notebook_updates
Browse files Browse the repository at this point in the history 
Update create_and_load_knowledge_graph_backups.ipynb
  • Loading branch information
@jyaistMap
jyaistMap authored Jul 4, 2024
2 parents 7c98d00 + ca2d936 commit 8904538
Showing 1 changed file with 168 additions and 69 deletions.
237 changes: 168 additions & 69 deletions samples/05_content_publishers/create_and_load_knowledge_graph_backups.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
"Create a backup of an ArcGIS Knowledge graph you have created to store your data model and data. The backup will be in the format of json files that can read to recreate your graph or be shared for others to be able to create the same graph on a different server.\n",
"\n",
"_**Load a new graph easily from your backup files!**_\n",
"\n",
"Use the backup files with the load backup files portion of this notebook to create a graph with the same data model and data."
]
},
Expand Down Expand Up @@ -82,7 +83,7 @@
"outputs": [],
"source": [
"# imports\n",
"import os, json\n",
"import os, json, requests\n",
"from datetime import datetime\n",
"from uuid import UUID\n",
"\n",
Expand All @@ -107,14 +108,16 @@
"outputs": [],
"source": [
"# output folder name\n",
"output_folder = \"C:\\backups\\myknowledgegraph_backup\"\n",
"output_folder = r\"C:\\backups\\myknowledgegraph_backup\"\n",
"\n",
"# output backup json file names\n",
"dm_ent = \"datamodel_entities.json\"\n",
"dm_rel = \"datamodel_relationships.json\"\n",
"dm_prov = \"datamodel_provenance.json\"\n",
"all_ent = \"all_entities.json\"\n",
"all_rel = \"all_relationships.json\"\n",
"prov_file = \"provenance_entities.json\" # this will only be used if you want to backup provenance records"
"prov_file = \"provenance_entities.json\" # this will only be used if you want to backup provenance records\n",
"servicedef = \"service_definition.json\""
]
},
{
Expand Down Expand Up @@ -153,6 +156,39 @@
" raise Exception(\"Knowledge graph to backup does not exist\")"
]
},
{
"cell_type": "markdown",
"id": "17968c25",
"metadata": {},
"source": [
"Get the service definition for the knowledge graph service and save it to the service definition file"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8c5275cb",
"metadata": {},
"outputs": [],
"source": [
"# create a token for request\n",
"token_url = f\"https://myportal.com/portal/sharing/rest/generateToken\"\n",
"creds = {\n",
" \"username\": \"myUsername\",\n",
" \"password\": \"myPassword\",\n",
" \"referer\": \"https://myportal.com/portal\",\n",
" \"f\": \"json\",\n",
"}\n",
"token_response = requests.post(token_url, data=creds, verify=False)\n",
"sd_data = requests.get(\n",
" url=\"https://myportal.com/server/rest/services/Hosted/myknowledgegraph/KnowledgeGraphServer\",\n",
" params={\"f\": \"json\", \"token\": token_response.json()[\"token\"]},\n",
" verify=False,\n",
")\n",
"with open(os.path.join(output_folder, servicedef), \"w\") as f:\n",
" json.dump(sd_data.text, f)"
]
},
{
"cell_type": "markdown",
"id": "ced3a37e",
Expand Down Expand Up @@ -250,9 +286,7 @@
"outputs": [],
"source": [
"# query for all entities in graph\n",
"original_entities = knowledgegraph_backup.query_streaming(\n",
" \"MATCH (n) RETURN distinct n\"\n",
")"
"original_entities = knowledgegraph_backup.query_streaming(\"MATCH (n) RETURN distinct n\")"
]
},
{
Expand All @@ -272,9 +306,7 @@
" if type(curr_entity[\"_properties\"][prop]) == UUID:\n",
" curr_entity[\"_properties\"][prop] = str(curr_entity[\"_properties\"][prop])\n",
" # delete objectid, the server will handle creating new ones when we load the backup\n",
" del curr_entity[\"_properties\"][\n",
" \"objectid\"\n",
" ]\n",
" del curr_entity[\"_properties\"][\"objectid\"]\n",
" all_entities_fromquery.append(curr_entity)"
]
},
Expand Down Expand Up @@ -335,9 +367,7 @@
" curr_relationship[\"_properties\"][prop]\n",
" )\n",
" # delete objectid, the server will handle creating new ones when we load the backup\n",
" del curr_relationship[\"_properties\"][\n",
" \"objectid\"\n",
" ]\n",
" del curr_relationship[\"_properties\"][\"objectid\"]\n",
" all_relationships_fromquery.append(curr_relationship)"
]
},
Expand All @@ -362,6 +392,19 @@
"If you have provenance records that you want to maintain in the backups, this will get all provenance records and save them to a json backup file"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2769e71",
"metadata": {},
"outputs": [],
"source": [
"# write provenance information to the provenance data model file\n",
"prov_structure = knowledgegraph_backup.datamodel[\"meta_entity_types\"][\"Provenance\"]\n",
"with open(os.path.join(output_folder, dm_prov), \"w\") as f:\n",
" json.dump(prov_structure, f)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -394,9 +437,7 @@
" curr_provenance[\"_properties\"][prop]\n",
" )\n",
" # delete objectid, the server will handle creating new ones when we load the backup\n",
" del curr_provenance[\"_properties\"][\n",
" \"objectid\"\n",
" ]\n",
" del curr_provenance[\"_properties\"][\"objectid\"]\n",
" all_provenance_fromquery.append(curr_provenance)"
]
},
Expand All @@ -420,61 +461,18 @@
"## Load backup files"
]
},
{
"cell_type": "markdown",
"id": "db4a59d8",
"metadata": {},
"source": [
"### Connect to portal and create knowledge graph\n",
"Connect to the portal and create a new knowledge graph service to load data model and data into"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dd042d14",
"metadata": {},
"outputs": [],
"source": [
"# connect to portal via GIS\n",
"gis_load = GIS(\n",
" \"https://myportal.com/portal\", \"username\", \"password\"\n",
")\n",
"# create a knowledge graph without provenance enabled\n",
"result = gis_load.content.create_service(\n",
" name=\"myknowledgegraph\",\n",
" capabilities=\"Query,Editing,Create,Update,Delete\",\n",
" service_type=\"KnowledgeGraph\",\n",
")\n",
"# OPTIONAL: switch line above with line below to create knowledge graph WITH provenance enabled\n",
"# result = gis_load.content.create_service(name=\"\", service_type=\"KnowledgeGraph\",create_params={\"name\": \"myknowledgegraph\", \"capabilities\": \"Query\", \"jsonProperties\": {\"supportsProvenance\": True}})\n",
"knowledgegraph_load = KnowledgeGraph(result.url, gis=gis_load)"
]
},
{
"cell_type": "markdown",
"id": "70b11b8a",
"metadata": {},
"source": [
"### Populate data model from saved json files\n",
"Populate entity and relationship types from saved json files. This is using the variables defined in the 'Setup' section above."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ad6f6d63",
"id": "a74ddfe3",
"metadata": {},
"outputs": [],
"source": [
"# load data model json files into graph data model\n",
"with open(os.path.join(output_folder, dm_ent), \"r\") as file:\n",
" dm_ents = json.load(file)\n",
"with open(os.path.join(output_folder, dm_rel), \"r\") as file:\n",
" dm_rels = json.load(file)\n",
"knowledgegraph_load.named_object_type_adds(\n",
" entity_types=dm_ents, relationship_types=dm_rels\n",
")"
" dm_rels = json.load(file)"
]
},
{
Expand Down Expand Up @@ -519,6 +517,77 @@
" doc_rel_type_name = relationship_type[\"name\"]"
]
},
{
"cell_type": "markdown",
"id": "db4a59d8",
"metadata": {},
"source": [
"### Connect to portal and create knowledge graph\n",
"Connect to the portal and create a new knowledge graph service to load data model and data into"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dd042d14",
"metadata": {},
"outputs": [],
"source": [
"# connect to portal via GIS\n",
"gis_load = GIS(\"home\")\n",
"\n",
"with open(os.path.join(output_folder, servicedef), \"r\") as file:\n",
" service_def = json.load(file)\n",
"service_def = json.loads(service_def)\n",
"updated_service_def = {\n",
" \"name\": \"myNewKnowledgeGraph\", # replace with the name for your new knowledge graph\n",
" \"capabilities\": service_def[\"capabilities\"],\n",
" \"jsonProperties\": {\n",
" \"allowGeometryUpdates\": service_def[\"allowGeometryUpdates\"],\n",
" \"searchMaxRecordCount\": service_def[\"searchMaxRecordCount\"],\n",
" \"spatialReference\": service_def[\"spatialReference\"],\n",
" \"maxRecordCount\": service_def[\"maxRecordCount\"],\n",
" \"description\": service_def[\"description\"],\n",
" \"copyrightText\": service_def[\"copyrightText\"],\n",
" \"documentEntityTypeInfo\": {\n",
" \"documentEntityTypeName\": doc_type_name,\n",
" \"hasDocumentsRelationshipTypeName\": doc_rel_type_name,\n",
" },\n",
" \"supportsDocuments\": service_def[\"supportsDocuments\"],\n",
" \"supportsSearch\": service_def[\"supportsSearch\"],\n",
" \"supportsProvenance\": service_def[\"supportsProvenance\"],\n",
" },\n",
"}\n",
"\n",
"result = gis_load.content.create_service(\n",
" name=\"\", service_type=\"KnowledgeGraph\", create_params=updated_service_def\n",
")\n",
"\n",
"knowledgegraph_load = KnowledgeGraph(result.url, gis=gis_load)"
]
},
{
"cell_type": "markdown",
"id": "70b11b8a",
"metadata": {},
"source": [
"### Populate data model from saved json files\n",
"Populate entity and relationship types from saved json files. This is using the variables defined in the 'Setup' section above."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ad6f6d63",
"metadata": {},
"outputs": [],
"source": [
"# populate entity and relationship types based on data model (errors about Document/HasDocument in the output are expected)\n",
"knowledgegraph_load.named_object_type_adds(\n",
" entity_types=dm_ents, relationship_types=dm_rels\n",
")"
]
},
{
"cell_type": "markdown",
"id": "fd1eecc7",
Expand All @@ -535,7 +604,7 @@
"metadata": {},
"outputs": [],
"source": [
"# load any additional document entity type properties\n",
"# load any additional document entity type properties (errors about properties that already exist are expected)\n",
"origin_document_properties = None\n",
"for entity_type in dm_ents:\n",
" if entity_type[\"name\"] == doc_type_name:\n",
Expand All @@ -544,7 +613,7 @@
"for prop in origin_document_properties:\n",
" prop_list.append(origin_document_properties[prop])\n",
"knowledgegraph_load.graph_property_adds(\n",
" type_name=\"Document\", graph_properties=prop_list\n",
" type_name=doc_type_name, graph_properties=prop_list\n",
")"
]
},
Expand All @@ -555,15 +624,15 @@
"metadata": {},
"outputs": [],
"source": [
"# load any additional document relationship type properties\n",
"# load any additional document relationship type properties (errors about properties that already exist are expected)\n",
"for relationship_type in dm_rels:\n",
" if relationship_type[\"name\"] == doc_rel_type_name:\n",
" origin_document_rel_properties = relationship_type[\"properties\"]\n",
"prop_list = []\n",
"for prop in origin_document_rel_properties:\n",
" prop_list.append(origin_document_rel_properties[prop])\n",
"knowledgegraph_load.graph_property_adds(\n",
" type_name=\"HasDocument\", graph_properties=prop_list\n",
" type_name=doc_rel_type_name, graph_properties=prop_list\n",
")"
]
},
Expand Down Expand Up @@ -766,16 +835,18 @@
"outputs": [],
"source": [
"# add search indexes for all relationship text properties\n",
"for entity_type in load_dm[\"relationship_types\"]:\n",
"for relationship_type in load_dm[\"relationship_types\"]:\n",
" prop_list = []\n",
" for prop in load_dm[\"relationship_types\"][entity_type][\"properties\"]:\n",
" for prop in load_dm[\"relationship_types\"][relationship_type][\"properties\"]:\n",
" if (\n",
" load_dm[\"relationship_types\"][entity_type][\"properties\"][prop][\"fieldType\"]\n",
" load_dm[\"relationship_types\"][relationship_type][\"properties\"][prop][\n",
" \"fieldType\"\n",
" ]\n",
" == \"esriFieldTypeString\"\n",
" ):\n",
" prop_list.append(prop)\n",
" knowledgegraph_load.update_search_index(\n",
" adds={entity_type: {\"property_names\": prop_list}}\n",
" adds={relationship_type: {\"property_names\": prop_list}}\n",
" )"
]
},
Expand All @@ -788,6 +859,34 @@
"This will only apply if you created a backup of provenance records and have enabled provenance on your knowledge graph service."
]
},
{
"cell_type": "markdown",
"id": "aa2099c5",
"metadata": {},
"source": [
"### Add additional provenance entity type properties\n",
"In the case additional properties exist in the original graph on the provenance type, we need to add those properties to the data model using graph_property_adds."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c2fddb86",
"metadata": {},
"outputs": [],
"source": [
"with open(os.path.join(output_folder, dm_prov), \"r\") as file:\n",
" prov_dm = json.load(file)\n",
"\n",
"prop_list = []\n",
"for prop in prov_dm[\"properties\"]:\n",
" prop_list.append(prov_dm[\"properties\"][prop])\n",
"knowledgegraph_load.graph_property_adds(\n",
" type_name=\"Provenance\", graph_properties=prop_list\n",
")\n",
"# errors about already existing properties are expected"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -838,7 +937,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.17"
"version": "3.9.16"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 8904538

Please sign in to comment.