notebooks/Ewings Sarcoma - FHIR and DRS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run WES workflow on Kids First DRS files searched on NCPI FHIR server\n",
    "\n",
    "This script demonstrates uses of the NCPI FHIR server to find patients and files from the Familial Leukemia study. The query returns DRS ids allowing the files accessed via the Kid's First Data Resource Center DRS server. The workflow is run on the Seven Bridges Cancer Genomics Cloud WES server. \n",
    "\n",
    "The FHIR server is accessed through the fhir-py Python client.\n",
    "\n",
    "### Set up the FHIR client\n",
    "Access is via a cookie obtained as described [here](https://github.com/NIH-NCPI/ncpi-api-fhir-service)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import json\n",
    "import requests\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "import pprint\n",
    "\n",
    "pprint = pprint.PrettyPrinter(indent=1).pprint\n",
    "# pprint() is now available to pretty-print any JSON\n",
    "\n",
    "FHIR_SERVER = 'https://kf-api-fhir-service.kidsfirstdrc.org'\n",
    "\n",
    "# Optional: Turn off SSL verification. Useful when dealing with a corporate proxy with self-signed certificates.\n",
    "# This should be set to True unless you actually see certificate errors.\n",
    "VERIFY_SSL = False\n",
    "\n",
    "if not VERIFY_SSL:\n",
    "    requests.packages.urllib3.disable_warnings()\n",
    "\n",
    "\n",
    "\n",
    "# Kids First uses cookie-based authentication, save the cookie locally here.\n",
    "full_cookie_path = os.path.expanduser('~/.keys/ncpi_prod_fhir_cookie.json')\n",
    "\n",
    "with open(full_cookie_path) as f:\n",
    "    cookies = json.load(f)\n",
    "kf_cookie = cookies['Cookie']\n",
    "            \n",
    "# We make a requests.Session to ensure consistent headers/cookie across all the requests we make\n",
    "s = requests.Session()\n",
    "s.headers.update({'Accept': 'application/fhir+json'})\n",
    "s.verify = VERIFY_SSL\n",
    "s.cookies['AWSELBAuthSessionCookie-0'] = kf_cookie\n",
    "\n",
    "\n",
    "# Test out the cookie by querying the server metadata\n",
    "r = s.get(f\"{FHIR_SERVER}/metadata\")\n",
    "\n",
    "if \"<!DOCTYPE html>\" in r.text:\n",
    "    sys.stderr.write('ERROR: Could not authenticate with Kids First. The cookie may need to be updated')\n",
    "    \n",
    "\n",
    "# This helper method allows us to easily switch between printing an entire Bundle, or just the first 20 lines.\n",
    "# Set truncate_for_github = False for actual use,\n",
    "# or just replace the function with a `return bundle`\n",
    "\n",
    "def print_bundle(bundle, truncate_for_github = False):\n",
    "    if not truncate_for_github:\n",
    "        return bundle\n",
    "    \n",
    "    lines = json.dumps(bundle, indent=2).split('\\n')\n",
    "    if len(lines) <= 20:\n",
    "        return bundle\n",
    "    else:   \n",
    "        print('\\n'.join(lines[:20]))\n",
    "        print('...\\nBundle truncated. Change the \"print_bundle\" function above to print the full content.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Resolves all pages for the bundle. Returns an array with all Bundles, including the original Bundle.\n",
    "def resolve_pages(bundle):\n",
    "    next_page_link = next(filter(lambda link: link['relation'] == 'next', bundle['link']), None)\n",
    "    if next_page_link:\n",
    "        next_page = s.get(next_page_link['url']).json()\n",
    "        return [bundle] + resolve_pages(next_page)\n",
    "    else:\n",
    "        return [bundle]\n",
    "\n",
    "# NOTE: No cell output.\n",
    "\n",
    "# Function to run a query for any kind of resources.\n",
    "def runQuery(query):\n",
    "    r = s.get(f\"{FHIR_SERVER}/{query}\")\n",
    "    first_bundle = r.json()\n",
    "    if 'entry' not in first_bundle:\n",
    "        resources = []\n",
    "    else:    \n",
    "        all_bundles = resolve_pages(first_bundle)\n",
    "        resources = [entry['resource'] for bundle in all_bundles for entry in bundle['entry']]\n",
    "        \n",
    "    print(f\"Total  Resources: {len(resources)}\")\n",
    "    return resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total  Resources: 29\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'resourceType': 'ResearchStudy',\n",
       "  'id': '687202',\n",
       "  'meta': {'versionId': '1',\n",
       "   'lastUpdated': '2022-06-20T14:25:32.953+00:00',\n",
       "   'source': '#RrhpFw6OupcziLQu',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'use': 'official',\n",
       "    'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_HGHFVPFD'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_HGHFVPFD'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs000424.v8.p2'}],\n",
       "  'title': 'Common Fund (CF) Genotype-Tissue Expression Project (GTEx)',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-GTEX'}]}]},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '276195',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:38:53.070+00:00',\n",
       "   'source': '#Vi7u1eZZ5de8QLJp',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_9PYZAHHE'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_9PYZAHHE'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001168.v2.p2'}],\n",
       "  'title': 'Genomic Studies of Orofacial Cleft Birth Defects',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-OCEA'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/117866'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '272532',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:35:53.533+00:00',\n",
       "   'source': '#iyM8irsuxNH9R023',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_7NQ9151J'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_7NQ9151J'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001987.v1.p1'}],\n",
       "  'title': 'Genome-wide Sequencing to Identify the Genes Responsible for Enchondromatoses and Related Malignant Tumors',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-ED'}]}]},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '269246',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:35:07.120+00:00',\n",
       "   'source': '#lP4G6TrlFDc3QLh0',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_B8X3C1MX'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_B8X3C1MX'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs002130.v1.p1'}],\n",
       "  'title': 'Kids First: Craniofacial Microsomia: Genetic Causes and Pathway Discovery',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-CM'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/268642'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '267437',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:33:08.229+00:00',\n",
       "   'source': '#DyQ38hhNWt3svQnd',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_DK0KRWK8'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_DK0KRWK8'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001997.v1.p1'}],\n",
       "  'title': 'Whole Genome Sequencing of African and Asian Orofacial Clefts Case-Parent Triads',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-OFCAA'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/265786'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '260974',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:30:12.571+00:00',\n",
       "   'source': '#ACpsEgNBIzUZFvRZ',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_NMVV8A1Y'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_NMVV8A1Y'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs002162.v1.p1'}],\n",
       "  'title': 'Kids First: Genetics of Structural Defects of the Kidney and Urinary Tract',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-KUT'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/258082'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '125635',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:28:20.867+00:00',\n",
       "   'source': '#lxGVxWt5Pnrz3V6B',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_YNSSAPHE'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_YNSSAPHE'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs000467.v17.p7'}],\n",
       "  'title': 'TARGET: Neuroblastoma (NBL)',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'TARGET'}]},\n",
       "   {'coding': [{'code': 'TARGET-NB'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/96500'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '123556',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:26:25.167+00:00',\n",
       "   'source': '#cdPsDsTirYYBOYpQ',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_W0V965XZ'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_W0V965XZ'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001738.v1.p1'}],\n",
       "  'title': 'Genomic Analysis of Familial Leukemia',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-FALL'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/122286'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '119523',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:24:24.020+00:00',\n",
       "   'source': '#Yq9Eck7A5dgx8r6P',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_R0EPRSGS'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_R0EPRSGS'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001420.v1.p1'}],\n",
       "  'title': 'Kids First: Genomics of Orofacial Cleft Birth Defects in Latin American Families',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-OFCLA'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/117866'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '116586',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:19:40.980+00:00',\n",
       "   'source': '#AiOJh4qrdW96baX4',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_RM8AFW0R'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_RM8AFW0R'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001410.v1.p1'}],\n",
       "  'title': 'Kids First: Genomics of Orthopaedic Disease Program',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-AIS'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/115209'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '114612',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:18:11.425+00:00',\n",
       "   'source': '#QuVPEvGz25b5x9Mw',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_DZ4GPQX6'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_DZ4GPQX6'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs002172.v1.p1'}],\n",
       "  'title': 'Kids First: The Genetics of Microtia in Hispanic Populations',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-GMHP'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/113832'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '112832',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:15:59.150+00:00',\n",
       "   'source': '#3KpEIj9PhLALo6KY',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_ZFGDG5YS'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_ZFGDG5YS'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001683.v2.p1'}],\n",
       "  'title': 'Gabriella Miller Kids First Pediatric Research Program in Novel Cancer Susceptibility in Families (from BASIC3)',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-NCSF'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/111363'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '104182',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T01:14:47.204+00:00',\n",
       "   'source': '#mzLhrVzqSrERkC7B',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_0TYVY1TW'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_0TYVY1TW'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs002161.v1.p1'}],\n",
       "  'title': 'Genomic Analysis of Esophageal Atresia and Tracheoesophageal Fistulas and Associated Congenital Anomalies',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-EATF'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/22423'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '100031',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T00:56:41.161+00:00',\n",
       "   'source': '#N1PVao6WQD1cD5rc',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_DYPMEHHF'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_DYPMEHHF'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001436.v1.p1'}],\n",
       "  'title': 'Discovering the Genetic Basis of Human Neuroblastoma: A Gabriella Miller Kids First Pediatric Research Program (Kids First) Project',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-NBL'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/96500'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '95270',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-19T00:52:11.476+00:00',\n",
       "   'source': '#eeVpydPqbiebrtFz',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_VTTSHWV4'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_VTTSHWV4'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001785.v1.p1'}],\n",
       "  'title': 'Kids First: Genomic Analysis of a Cohort with Infantile Hemangiomas Associated with Multi-organ Structural Birth Defects',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-IHMOBD'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/94669'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '55858',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T23:05:53.571+00:00',\n",
       "   'source': '#5HNtCYQthZNpzgXb',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_DZTB5HRR'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_DZTB5HRR'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001247.v1.p1'}],\n",
       "  'title': 'Kids First: BCH Structural Birth Defects Collaboration: Syndromic cranial dysinnervation disorders',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-SCD'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/53916'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '52326',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T23:02:59.628+00:00',\n",
       "   'source': '#jJYgC0B3KRJY7yvf',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_YGVA0E1C'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_YGVA0E1C'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001228.v1.p1'}],\n",
       "  'title': 'Kids First: Expanded Ewing sarcoma cohort for tumor genomics and association with DNA repair deficiencies, clinical presentation, and outcome',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-ESGR'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/49919'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '49513',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T23:01:23.369+00:00',\n",
       "   'source': '#rYzF3dJlSTbMTjgC',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_6FPYJQBR'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_6FPYJQBR'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001178.v1.p1'}],\n",
       "  'title': 'Kids First: Genetic Basis of Disorders/Differences of Sex Development (DSD)',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-DSD'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/48897'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '48656',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T22:57:56.745+00:00',\n",
       "   'source': '#83Q7YI9F69RaW6Gn',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_8Y99QZJJ'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_8Y99QZJJ'}],\n",
       "  'title': 'Pediatric Brain Tumor Atlas: PNOC',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Pediatric Brain Tumor Atlas'}]},\n",
       "   {'coding': [{'code': 'PBTA-PNOC'}]}]},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '47035',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T22:47:40.542+00:00',\n",
       "   'source': '#vIKGnEdkonqdfT7D',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_P445ACHV'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_P445ACHV'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001806.v1.p1'}],\n",
       "  'title': 'Kids First: Whole genome sequencing of nonsyndromic craniosynostosis',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-NC'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/44846'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '42776',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T22:41:20.078+00:00',\n",
       "   'source': '#EpHvojQubVGr08D2',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_Z6MWD3H0'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_Z6MWD3H0'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs002330.v1.p1'}],\n",
       "  'title': 'Kids First: Genomic Analysis of Congenital Heart Defects and Acute Lymphoblastic Leukemia in Children with Down Syndrome',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'},\n",
       "     {'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCERANDBIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-CHDALL'}]}]},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '38019',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T22:39:32.861+00:00',\n",
       "   'source': '#ddzEGC1MjOTupWoI',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_46RR9ZR6'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_46RR9ZR6'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs000465.v18.p7'}],\n",
       "  'title': 'TARGET: Acute Myeloid Leukemia (AML)',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'TARGET'}]},\n",
       "   {'coding': [{'code': 'TARGET-AML'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/37336'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '35326',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T22:27:44.921+00:00',\n",
       "   'source': '#sFc745Ch5AES9OlV',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_PET7Q6F2'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_PET7Q6F2'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs002187.v1.p1'}],\n",
       "  'title': 'Kids First: Germline and Somatic Variants in Myeloid Malignancies in Children',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-MMC'}]}]},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '27033',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T22:20:56.107+00:00',\n",
       "   'source': '#aXN1I8EeUBCnqZwe',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_46SK55A3'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_46SK55A3'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001110.v3.p1'}],\n",
       "  'title': 'Kids First: Genomic Analysis of Congenital Diaphragmatic Hernia',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '276720006',\n",
       "      'display': 'Dysmorphism (disorder)'}],\n",
       "    'text': 'BIRTHDEFECT'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-CDH'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/22423'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '22088',\n",
       "  'meta': {'versionId': '2',\n",
       "   'lastUpdated': '2022-01-18T22:19:51.364+00:00',\n",
       "   'source': '#OOmTR2zrBaleX830',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_ZXJFFMEF'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_ZXJFFMEF'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001714.v1.p1'}],\n",
       "  'title': 'Kids First: An Integrated Clinical and Genomic Analysis of Treatment Failure in Pediatric Osteosarcoma',\n",
       "  'status': 'completed',\n",
       "  'category': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'CANCER'}],\n",
       "  'keyword': [{'coding': [{'code': 'Kids First'}]},\n",
       "   {'coding': [{'code': 'KF-OS'}]}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/21951'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '281300',\n",
       "  'meta': {'versionId': '1',\n",
       "   'lastUpdated': '2021-10-15T11:19:45.641+00:00',\n",
       "   'source': '#YWSMBscCf7B7QqSD',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_1P41Z782'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_1P41Z782'}],\n",
       "  'title': 'OpenDIPG: ICR London',\n",
       "  'status': 'completed',\n",
       "  'condition': [{'coding': [{'system': 'http://snomed.info/sct',\n",
       "      'code': '86049000',\n",
       "      'display': 'Malignant neoplasm, primary (morphologic abnormality)'}],\n",
       "    'text': 'Cancer'}],\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/281102'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '144813',\n",
       "  'meta': {'versionId': '1',\n",
       "   'lastUpdated': '2021-10-15T01:03:03.974+00:00',\n",
       "   'source': '#iEnyrOHjD2XNV1zY',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_PREASA7S'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_PREASA7S'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001138.v3.p2'}],\n",
       "  'title': 'National Heart, Lung, and Blood Institute (NHLBI) Bench to Bassinet Program: The Gabriella Miller Kids First Pediatric Research Program of the Pediatric Cardiac Genetics Consortium (PCGC)',\n",
       "  'status': 'completed',\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/138779'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '130099',\n",
       "  'meta': {'versionId': '1',\n",
       "   'lastUpdated': '2021-10-15T00:01:50.378+00:00',\n",
       "   'source': '#MzLN0bNN80sAsx1H',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_JWS3V24D'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_JWS3V24D'},\n",
       "   {'system': 'https://www.ncbi.nlm.nih.gov/projects/gap/cgi-bin/study.cgi?study_id=',\n",
       "    'value': 'phs001846.v1.p1'}],\n",
       "  'title': 'Kids First: Genetics at the Intersection of Childhood Cancer and Birth Defects',\n",
       "  'status': 'completed',\n",
       "  'principalInvestigator': {'reference': 'PractitionerRole/126192'}},\n",
       " {'resourceType': 'ResearchStudy',\n",
       "  'id': '76758',\n",
       "  'meta': {'versionId': '1',\n",
       "   'lastUpdated': '2021-10-14T20:38:18.232+00:00',\n",
       "   'source': '#1JZeWjbY1aYTteVz',\n",
       "   'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchStudy']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/studies/',\n",
       "    'value': 'SD_BHJXBDQK'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'ResearchStudy-SD_BHJXBDQK'}],\n",
       "  'title': \"Pediatric Brain Tumor Atlas - Children's Brain Tumor Tissue Consortium\",\n",
       "  'status': 'completed'}]"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "studies = runQuery('ResearchStudy')\n",
    "studies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Ewings Sarcoma study is id 52326"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total  Resources: 1203\n",
      "{'id': '53466',\n",
      " 'identifier': [{'value': 'Schiffman-232'},\n",
      "                {'system': 'https://kf-api-dataservice.kidsfirstdrc.org/participants/',\n",
      "                 'value': 'PT_SKYS7BYE'},\n",
      "                {'system': 'urn:kids-first:unique-string',\n",
      "                 'value': 'ResearchSubject-SD_YGVA0E1C-PT_SKYS7BYE'}],\n",
      " 'individual': {'reference': 'Patient/51063'},\n",
      " 'meta': {'lastUpdated': '2021-11-16T16:51:37.022+00:00',\n",
      "          'profile': ['http://hl7.org/fhir/StructureDefinition/ResearchSubject'],\n",
      "          'source': '#6cXjyvyFlK310TS1',\n",
      "          'tag': [{'code': 'SD_YGVA0E1C'}],\n",
      "          'versionId': '3'},\n",
      " 'resourceType': 'ResearchSubject',\n",
      " 'status': 'off-study',\n",
      " 'study': {'reference': 'ResearchStudy/52326'}}\n"
     ]
    }
   ],
   "source": [
    "study_id = \"52326\"\n",
    "subjects = runQuery(f'ResearchSubject?study={study_id}')\n",
    "pprint(subjects[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total  Resources: 1203\n",
      "{'extension': [{'extension': [{'url': 'text', 'valueString': 'Not Reported'}],\n",
      "                'url': 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-race'},\n",
      "               {'extension': [{'url': 'text', 'valueString': 'Not Reported'}],\n",
      "                'url': 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity'}],\n",
      " 'gender': 'female',\n",
      " 'id': '51063',\n",
      " 'identifier': [{'value': 'Schiffman-232'},\n",
      "                {'system': 'https://kf-api-dataservice.kidsfirstdrc.org/participants/',\n",
      "                 'value': 'PT_SKYS7BYE'},\n",
      "                {'system': 'urn:kids-first:unique-string',\n",
      "                 'value': 'Patient-SD_YGVA0E1C-PT_SKYS7BYE'}],\n",
      " 'meta': {'lastUpdated': '2021-11-16T08:31:55.743+00:00',\n",
      "          'profile': ['http://hl7.org/fhir/StructureDefinition/Patient'],\n",
      "          'source': '#ftuteWwUU3WhzzKW',\n",
      "          'tag': [{'code': 'SD_YGVA0E1C'}],\n",
      "          'versionId': '2'},\n",
      " 'resourceType': 'Patient'}\n"
     ]
    }
   ],
   "source": [
    "patients = runQuery(f'Patient?_has:ResearchSubject:individual:study={study_id}')\n",
    "pprint(patients[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve phenotypic data for each subject from FHIR\n",
    "Save the results to a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Patient/51063\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51063 :5\n",
      "Unauthorized for that DRS id\n",
      "Patient/51052\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51052 :5\n",
      "Patient/51059\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51059 :5\n",
      "Patient/51064\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51064 :0\n",
      "Patient/51056\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51056 :5\n",
      "Patient/51051\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51051 :5\n",
      "Patient/51040\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51040 :0\n",
      "Patient/51058\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51058 :7\n",
      "Patient/51060\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51060 :7\n",
      "Patient/51049\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51049 :5\n",
      "Patient/51046\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51046 :7\n",
      "Patient/51047\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51047 :5\n",
      "Patient/51053\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51053 :0\n",
      "Patient/51048\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51048 :0\n",
      "Patient/51045\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51045 :5\n",
      "Patient/51043\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51043 :5\n",
      "Patient/51055\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51055 :0\n",
      "Patient/51039\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51039 :5\n",
      "Patient/51044\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51044 :5\n",
      "Patient/51041\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51041 :5\n",
      "Patient/51031\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51031 :5\n",
      "Patient/51042\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51042 :0\n",
      "Patient/51038\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51038 :5\n",
      "Patient/51032\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51032 :5\n",
      "Patient/51035\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51035 :0\n",
      "Patient/51036\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51036 :5\n",
      "Patient/51037\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51037 :0\n",
      "Patient/51033\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51033 :7\n",
      "Patient/51029\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51029 :5\n",
      "Patient/51030\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51030 :5\n",
      "Patient/51028\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51028 :5\n",
      "Patient/51025\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51025 :5\n",
      "Patient/51034\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51034 :5\n",
      "Patient/51021\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51021 :5\n",
      "Patient/51020\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51020 :7\n",
      "Patient/51026\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51026 :5\n",
      "Patient/51023\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51023 :5\n",
      "Patient/51017\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51017 :5\n",
      "Patient/51027\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51027 :7\n",
      "Patient/51011\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51011 :5\n",
      "Patient/51024\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51024 :5\n",
      "Patient/51022\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51022 :5\n",
      "Patient/51014\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51014 :0\n",
      "Patient/51013\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51013 :5\n",
      "Patient/51012\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51012 :7\n",
      "Patient/51019\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51019 :5\n",
      "Patient/51009\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51009 :5\n",
      "Patient/51018\n",
      "Total  Resources: 7\n",
      "# of documents for subject 51018 :7\n",
      "Patient/51008\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51008 :0\n",
      "Patient/51015\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51015 :5\n",
      "Patient/51004\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51004 :5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>drs_uri</th>\n",
       "      <th>patient</th>\n",
       "      <th>self_uri</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/520bd083-82c1-4f00...</td>\n",
       "      <td>51063</td>\n",
       "      <td>drs://dg.F82A1A:520bd083-82c1-4f00-abf5-ca1783...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/805860ce-d0e3-406c...</td>\n",
       "      <td>51052</td>\n",
       "      <td>drs://dg.F82A1A:805860ce-d0e3-406c-b652-c60f72...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/d41fce40-b916-4d28...</td>\n",
       "      <td>51059</td>\n",
       "      <td>drs://dg.F82A1A:d41fce40-b916-4d28-a294-5364ad...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/938490ec-11b6-4a16...</td>\n",
       "      <td>51056</td>\n",
       "      <td>drs://dg.F82A1A:938490ec-11b6-4a16-ac42-e81df1...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/ea26fe12-a138-4325...</td>\n",
       "      <td>51051</td>\n",
       "      <td>drs://dg.F82A1A:ea26fe12-a138-4325-a783-d881ed...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/c9b83883-fc86-4ac5...</td>\n",
       "      <td>51058</td>\n",
       "      <td>drs://dg.F82A1A:c9b83883-fc86-4ac5-95b2-cb1aab...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/98eeb3eb-7652-4d74...</td>\n",
       "      <td>51060</td>\n",
       "      <td>drs://dg.F82A1A:98eeb3eb-7652-4d74-b565-c00aa5...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/a052d1a9-e71c-4ccc...</td>\n",
       "      <td>51049</td>\n",
       "      <td>drs://dg.F82A1A:a052d1a9-e71c-4ccc-a4ab-e6cdf0...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/31ef38aa-98d9-4b98...</td>\n",
       "      <td>51046</td>\n",
       "      <td>drs://dg.F82A1A:31ef38aa-98d9-4b98-ac56-dd7a3a...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/05493152-872b-41a4...</td>\n",
       "      <td>51047</td>\n",
       "      <td>drs://dg.F82A1A:05493152-872b-41a4-b2aa-d7a23b...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/8ffe7df7-a325-4697...</td>\n",
       "      <td>51045</td>\n",
       "      <td>drs://dg.F82A1A:8ffe7df7-a325-4697-9690-a8703c...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/57b26440-fe2d-4235...</td>\n",
       "      <td>51043</td>\n",
       "      <td>drs://dg.F82A1A:57b26440-fe2d-4235-ae9b-016340...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/ee506958-6695-47f1...</td>\n",
       "      <td>51039</td>\n",
       "      <td>drs://dg.F82A1A:ee506958-6695-47f1-a759-5c2f28...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/39d079b9-ea06-480e...</td>\n",
       "      <td>51044</td>\n",
       "      <td>drs://dg.F82A1A:39d079b9-ea06-480e-8eca-0fd04d...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/7a219c23-fe75-4cc6...</td>\n",
       "      <td>51041</td>\n",
       "      <td>drs://dg.F82A1A:7a219c23-fe75-4cc6-b714-88bcc9...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/414c56ef-4184-4330...</td>\n",
       "      <td>51031</td>\n",
       "      <td>drs://dg.F82A1A:414c56ef-4184-4330-9775-bb0c6e...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/85ec9830-08e4-49e5...</td>\n",
       "      <td>51038</td>\n",
       "      <td>drs://dg.F82A1A:85ec9830-08e4-49e5-9750-0f48bb...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/ee35442a-2dfb-4da9...</td>\n",
       "      <td>51032</td>\n",
       "      <td>drs://dg.F82A1A:ee35442a-2dfb-4da9-8542-010040...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/13e249f5-823c-4374...</td>\n",
       "      <td>51036</td>\n",
       "      <td>drs://dg.F82A1A:13e249f5-823c-4374-863f-078759...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/92e99c2c-8cee-465f...</td>\n",
       "      <td>51033</td>\n",
       "      <td>drs://dg.F82A1A:92e99c2c-8cee-465f-a1b0-cf7194...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/8d4c4dff-af6c-453d...</td>\n",
       "      <td>51029</td>\n",
       "      <td>drs://dg.F82A1A:8d4c4dff-af6c-453d-ba61-ec844d...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/6d69a86b-c7c5-422d...</td>\n",
       "      <td>51030</td>\n",
       "      <td>drs://dg.F82A1A:6d69a86b-c7c5-422d-a7f1-883b31...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/aeebab10-3512-4fda...</td>\n",
       "      <td>51028</td>\n",
       "      <td>drs://dg.F82A1A:aeebab10-3512-4fda-90b2-470d52...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/1298e48e-688c-4cea...</td>\n",
       "      <td>51025</td>\n",
       "      <td>drs://dg.F82A1A:1298e48e-688c-4cea-9cbc-edd226...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/dcdcd297-e464-44c3...</td>\n",
       "      <td>51034</td>\n",
       "      <td>drs://dg.F82A1A:dcdcd297-e464-44c3-a10b-80dc44...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/b4620d99-b35a-4ecd...</td>\n",
       "      <td>51021</td>\n",
       "      <td>drs://dg.F82A1A:b4620d99-b35a-4ecd-8aef-ceecc2...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/e0bd6fcc-7895-4392...</td>\n",
       "      <td>51020</td>\n",
       "      <td>drs://dg.F82A1A:e0bd6fcc-7895-4392-9487-c5559e...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/5f949e7b-7113-4e1f...</td>\n",
       "      <td>51026</td>\n",
       "      <td>drs://dg.F82A1A:5f949e7b-7113-4e1f-9107-87aa42...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/7dbc04e6-ac67-4ff5...</td>\n",
       "      <td>51023</td>\n",
       "      <td>drs://dg.F82A1A:7dbc04e6-ac67-4ff5-a49a-eefa54...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/1364dca4-1083-40bb...</td>\n",
       "      <td>51017</td>\n",
       "      <td>drs://dg.F82A1A:1364dca4-1083-40bb-8f76-228fce...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/116209d6-8850-489e...</td>\n",
       "      <td>51027</td>\n",
       "      <td>drs://dg.F82A1A:116209d6-8850-489e-8289-4cf2bd...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/68b76631-b528-4a65...</td>\n",
       "      <td>51011</td>\n",
       "      <td>drs://dg.F82A1A:68b76631-b528-4a65-af94-d12145...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/e34ec09f-0957-498f...</td>\n",
       "      <td>51024</td>\n",
       "      <td>drs://dg.F82A1A:e34ec09f-0957-498f-b136-834090...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/78dff19c-5bd7-450c...</td>\n",
       "      <td>51022</td>\n",
       "      <td>drs://dg.F82A1A:78dff19c-5bd7-450c-85fe-8baa5a...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/68549483-5975-49c7...</td>\n",
       "      <td>51013</td>\n",
       "      <td>drs://dg.F82A1A:68549483-5975-49c7-bd3d-091c61...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/87c3470c-d977-412b...</td>\n",
       "      <td>51012</td>\n",
       "      <td>drs://dg.F82A1A:87c3470c-d977-412b-a36f-8b4e3c...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/35d3bbba-f430-4ff8...</td>\n",
       "      <td>51019</td>\n",
       "      <td>drs://dg.F82A1A:35d3bbba-f430-4ff8-8bc1-810ea7...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/e2624dc8-a5ef-41c1...</td>\n",
       "      <td>51009</td>\n",
       "      <td>drs://dg.F82A1A:e2624dc8-a5ef-41c1-b01b-e6028d...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/4e5af417-bf31-479d...</td>\n",
       "      <td>51018</td>\n",
       "      <td>drs://dg.F82A1A:4e5af417-bf31-479d-b209-de00bf...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/dfa893c2-8f37-4f4e...</td>\n",
       "      <td>51015</td>\n",
       "      <td>drs://dg.F82A1A:dfa893c2-8f37-4f4e-ae93-b489c9...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>drs://data.kidsfirstdrc.org/005d8132-d976-40b9...</td>\n",
       "      <td>51004</td>\n",
       "      <td>drs://dg.F82A1A:005d8132-d976-40b9-a747-c80349...</td>\n",
       "      <td>https://kf-seq-data-washu.s3.amazonaws.com/Ewi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              drs_uri patient  \\\n",
       "0   drs://data.kidsfirstdrc.org/520bd083-82c1-4f00...   51063   \n",
       "1   drs://data.kidsfirstdrc.org/805860ce-d0e3-406c...   51052   \n",
       "2   drs://data.kidsfirstdrc.org/d41fce40-b916-4d28...   51059   \n",
       "3   drs://data.kidsfirstdrc.org/938490ec-11b6-4a16...   51056   \n",
       "4   drs://data.kidsfirstdrc.org/ea26fe12-a138-4325...   51051   \n",
       "5   drs://data.kidsfirstdrc.org/c9b83883-fc86-4ac5...   51058   \n",
       "6   drs://data.kidsfirstdrc.org/98eeb3eb-7652-4d74...   51060   \n",
       "7   drs://data.kidsfirstdrc.org/a052d1a9-e71c-4ccc...   51049   \n",
       "8   drs://data.kidsfirstdrc.org/31ef38aa-98d9-4b98...   51046   \n",
       "9   drs://data.kidsfirstdrc.org/05493152-872b-41a4...   51047   \n",
       "10  drs://data.kidsfirstdrc.org/8ffe7df7-a325-4697...   51045   \n",
       "11  drs://data.kidsfirstdrc.org/57b26440-fe2d-4235...   51043   \n",
       "12  drs://data.kidsfirstdrc.org/ee506958-6695-47f1...   51039   \n",
       "13  drs://data.kidsfirstdrc.org/39d079b9-ea06-480e...   51044   \n",
       "14  drs://data.kidsfirstdrc.org/7a219c23-fe75-4cc6...   51041   \n",
       "15  drs://data.kidsfirstdrc.org/414c56ef-4184-4330...   51031   \n",
       "16  drs://data.kidsfirstdrc.org/85ec9830-08e4-49e5...   51038   \n",
       "17  drs://data.kidsfirstdrc.org/ee35442a-2dfb-4da9...   51032   \n",
       "18  drs://data.kidsfirstdrc.org/13e249f5-823c-4374...   51036   \n",
       "19  drs://data.kidsfirstdrc.org/92e99c2c-8cee-465f...   51033   \n",
       "20  drs://data.kidsfirstdrc.org/8d4c4dff-af6c-453d...   51029   \n",
       "21  drs://data.kidsfirstdrc.org/6d69a86b-c7c5-422d...   51030   \n",
       "22  drs://data.kidsfirstdrc.org/aeebab10-3512-4fda...   51028   \n",
       "23  drs://data.kidsfirstdrc.org/1298e48e-688c-4cea...   51025   \n",
       "24  drs://data.kidsfirstdrc.org/dcdcd297-e464-44c3...   51034   \n",
       "25  drs://data.kidsfirstdrc.org/b4620d99-b35a-4ecd...   51021   \n",
       "26  drs://data.kidsfirstdrc.org/e0bd6fcc-7895-4392...   51020   \n",
       "27  drs://data.kidsfirstdrc.org/5f949e7b-7113-4e1f...   51026   \n",
       "28  drs://data.kidsfirstdrc.org/7dbc04e6-ac67-4ff5...   51023   \n",
       "29  drs://data.kidsfirstdrc.org/1364dca4-1083-40bb...   51017   \n",
       "30  drs://data.kidsfirstdrc.org/116209d6-8850-489e...   51027   \n",
       "31  drs://data.kidsfirstdrc.org/68b76631-b528-4a65...   51011   \n",
       "32  drs://data.kidsfirstdrc.org/e34ec09f-0957-498f...   51024   \n",
       "33  drs://data.kidsfirstdrc.org/78dff19c-5bd7-450c...   51022   \n",
       "34  drs://data.kidsfirstdrc.org/68549483-5975-49c7...   51013   \n",
       "35  drs://data.kidsfirstdrc.org/87c3470c-d977-412b...   51012   \n",
       "36  drs://data.kidsfirstdrc.org/35d3bbba-f430-4ff8...   51019   \n",
       "37  drs://data.kidsfirstdrc.org/e2624dc8-a5ef-41c1...   51009   \n",
       "38  drs://data.kidsfirstdrc.org/4e5af417-bf31-479d...   51018   \n",
       "39  drs://data.kidsfirstdrc.org/dfa893c2-8f37-4f4e...   51015   \n",
       "40  drs://data.kidsfirstdrc.org/005d8132-d976-40b9...   51004   \n",
       "\n",
       "                                             self_uri  \\\n",
       "0   drs://dg.F82A1A:520bd083-82c1-4f00-abf5-ca1783...   \n",
       "1   drs://dg.F82A1A:805860ce-d0e3-406c-b652-c60f72...   \n",
       "2   drs://dg.F82A1A:d41fce40-b916-4d28-a294-5364ad...   \n",
       "3   drs://dg.F82A1A:938490ec-11b6-4a16-ac42-e81df1...   \n",
       "4   drs://dg.F82A1A:ea26fe12-a138-4325-a783-d881ed...   \n",
       "5   drs://dg.F82A1A:c9b83883-fc86-4ac5-95b2-cb1aab...   \n",
       "6   drs://dg.F82A1A:98eeb3eb-7652-4d74-b565-c00aa5...   \n",
       "7   drs://dg.F82A1A:a052d1a9-e71c-4ccc-a4ab-e6cdf0...   \n",
       "8   drs://dg.F82A1A:31ef38aa-98d9-4b98-ac56-dd7a3a...   \n",
       "9   drs://dg.F82A1A:05493152-872b-41a4-b2aa-d7a23b...   \n",
       "10  drs://dg.F82A1A:8ffe7df7-a325-4697-9690-a8703c...   \n",
       "11  drs://dg.F82A1A:57b26440-fe2d-4235-ae9b-016340...   \n",
       "12  drs://dg.F82A1A:ee506958-6695-47f1-a759-5c2f28...   \n",
       "13  drs://dg.F82A1A:39d079b9-ea06-480e-8eca-0fd04d...   \n",
       "14  drs://dg.F82A1A:7a219c23-fe75-4cc6-b714-88bcc9...   \n",
       "15  drs://dg.F82A1A:414c56ef-4184-4330-9775-bb0c6e...   \n",
       "16  drs://dg.F82A1A:85ec9830-08e4-49e5-9750-0f48bb...   \n",
       "17  drs://dg.F82A1A:ee35442a-2dfb-4da9-8542-010040...   \n",
       "18  drs://dg.F82A1A:13e249f5-823c-4374-863f-078759...   \n",
       "19  drs://dg.F82A1A:92e99c2c-8cee-465f-a1b0-cf7194...   \n",
       "20  drs://dg.F82A1A:8d4c4dff-af6c-453d-ba61-ec844d...   \n",
       "21  drs://dg.F82A1A:6d69a86b-c7c5-422d-a7f1-883b31...   \n",
       "22  drs://dg.F82A1A:aeebab10-3512-4fda-90b2-470d52...   \n",
       "23  drs://dg.F82A1A:1298e48e-688c-4cea-9cbc-edd226...   \n",
       "24  drs://dg.F82A1A:dcdcd297-e464-44c3-a10b-80dc44...   \n",
       "25  drs://dg.F82A1A:b4620d99-b35a-4ecd-8aef-ceecc2...   \n",
       "26  drs://dg.F82A1A:e0bd6fcc-7895-4392-9487-c5559e...   \n",
       "27  drs://dg.F82A1A:5f949e7b-7113-4e1f-9107-87aa42...   \n",
       "28  drs://dg.F82A1A:7dbc04e6-ac67-4ff5-a49a-eefa54...   \n",
       "29  drs://dg.F82A1A:1364dca4-1083-40bb-8f76-228fce...   \n",
       "30  drs://dg.F82A1A:116209d6-8850-489e-8289-4cf2bd...   \n",
       "31  drs://dg.F82A1A:68b76631-b528-4a65-af94-d12145...   \n",
       "32  drs://dg.F82A1A:e34ec09f-0957-498f-b136-834090...   \n",
       "33  drs://dg.F82A1A:78dff19c-5bd7-450c-85fe-8baa5a...   \n",
       "34  drs://dg.F82A1A:68549483-5975-49c7-bd3d-091c61...   \n",
       "35  drs://dg.F82A1A:87c3470c-d977-412b-a36f-8b4e3c...   \n",
       "36  drs://dg.F82A1A:35d3bbba-f430-4ff8-8bc1-810ea7...   \n",
       "37  drs://dg.F82A1A:e2624dc8-a5ef-41c1-b01b-e6028d...   \n",
       "38  drs://dg.F82A1A:4e5af417-bf31-479d-b209-de00bf...   \n",
       "39  drs://dg.F82A1A:dfa893c2-8f37-4f4e-ae93-b489c9...   \n",
       "40  drs://dg.F82A1A:005d8132-d976-40b9-a747-c80349...   \n",
       "\n",
       "                                                  url  \n",
       "0                                                None  \n",
       "1   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "2   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "3   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "4   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "5   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "6   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "7   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "8   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "9   https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "10  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "11  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "12  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "13  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "14  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "15  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "16  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "17  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "18  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "19  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "20  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "21  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "22  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "23  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "24  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "25  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "26  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "27  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "28  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "29  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "30  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "31  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "32  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "33  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "34  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "35  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "36  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "37  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "38  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "39  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  \n",
       "40  https://kf-seq-data-washu.s3.amazonaws.com/Ewi...  "
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "drs_data = []\n",
    "ii=0\n",
    "for pjson in patients:\n",
    "    print (f\"Patient/{pjson['id']}\")\n",
    "    drs_uri = getFiles(pjson['id'], 'bam')\n",
    "    #print(drs_uri)\n",
    "    if drs_uri:\n",
    "        drs_id = drs_uri.split('/')[-1]\n",
    "        drs_details = drsClient.getObject(drs_id)\n",
    "        self_uri = drs_details['self_uri']\n",
    "        url = drsClient.getAccessURL(drs_id, 's3')\n",
    "        drs_data.append({'drs_uri':drs_uri, 'patient':pjson['id'], 'self_uri':self_uri, 'url':url})\n",
    "    \n",
    "    ii +=1\n",
    "    if ii > 50: break\n",
    "df = pd.DataFrame.from_dict(drs_data)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DocumentReference?subject=Patient/51020\n",
      "Total  Resources: 7\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "patient_id = '51063'\n",
    "q = f'DocumentReference?subject=Patient/51020'\n",
    "\n",
    "print(q)\n",
    "docs = runQuery(q)\n",
    "print(len(docs))\n",
    "\n",
    "#f\"{FHIR_SERVER}/DocumentReference?subject=Patient/{subject_id}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observation?subject=Patient/51063\n",
      "Total  Resources: 1\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "q = f'Observation?subject=Patient/{patient_id}'\n",
    "print(q)\n",
    "observations = runQuery(q)\n",
    "print(len(observations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'resourceType': 'Observation',\n",
       "  'id': '51331',\n",
       "  'meta': {'versionId': '1',\n",
       "   'lastUpdated': '2021-10-14T19:15:40.213+00:00',\n",
       "   'source': '#fYB2lw2s2oPs1vCF',\n",
       "   'profile': ['https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/family-relationship']},\n",
       "  'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/family-relationships/',\n",
       "    'value': 'FR_XP292D7F'},\n",
       "   {'system': 'urn:kids-first:unique-string',\n",
       "    'value': 'Observation-PT_28H2AWQ8-Father-PT_MSB54TSM'}],\n",
       "  'status': 'final',\n",
       "  'code': {'coding': [{'system': 'http://terminology.hl7.org/CodeSystem/v3-RoleCode',\n",
       "     'code': 'FAMMEMB',\n",
       "     'display': 'family member'}],\n",
       "   'text': 'Family relationship'},\n",
       "  'subject': {'reference': 'Patient/51020'},\n",
       "  'focus': [{'reference': 'Patient/51021'}],\n",
       "  'valueCodeableConcept': {'coding': [{'system': 'http://terminology.hl7.org/CodeSystem/v3-RoleCode',\n",
       "     'code': 'FTH',\n",
       "     'display': 'father'}],\n",
       "   'text': 'Father'}}]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'link'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/wz/jjbxsnr13v7dkw2jkbpmqd_dly65gq/T/ipykernel_73464/3022108309.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mpatientID\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"Patient/{pjson['id']}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;31m#resources = client.resources('')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mobservations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrunQuery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Observation?patientID={patientID}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m     \u001b[0;31m#observations = resources.search(subject=patientID).fetch_all()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/var/folders/wz/jjbxsnr13v7dkw2jkbpmqd_dly65gq/T/ipykernel_73464/955695700.py\u001b[0m in \u001b[0;36mrunQuery\u001b[0;34m(query)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{FHIR_SERVER}/{query}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mfirst_bundle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mall_bundles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresolve_pages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst_bundle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mresources\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mentry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'resource'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mbundle\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_bundles\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mentry\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbundle\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'entry'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/var/folders/wz/jjbxsnr13v7dkw2jkbpmqd_dly65gq/T/ipykernel_73464/3621109856.py\u001b[0m in \u001b[0;36mresolve_pages\u001b[0;34m(bundle)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Resolves all pages for the bundle. Returns an array with all Bundles, including the original Bundle.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mresolve_pages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbundle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mnext_page_link\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mlink\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlink\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'relation'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'next'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbundle\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'link'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnext_page_link\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mnext_page\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_page_link\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'url'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'link'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "mySubjects = []\n",
    "for pjson in patients:\n",
    "    #sjson = s.serialize()\n",
    "    #patientID = sjson['individual']['reference']\n",
    "    #thisSubject = {\"patientID\":patientID}\n",
    "    #patients = client.resources('Patient')\n",
    "    #p = patients.search(_id=patientID).get()\n",
    "    #pjson=p.serialize()\n",
    "    #print(json.dumps(pjson, indent=3))\n",
    "    \n",
    "\n",
    "    #print(\"Observations\",'.'*20)\n",
    "    patientID = f\"Patient/{pjson['id']}\"\n",
    "    #resources = client.resources('')\n",
    "    observations = runQuery(f'Observation?subject=Patient/{patientID}')\n",
    "    #observations = resources.search(subject=patientID).fetch_all()\n",
    "    on = 0\n",
    "    csn = 0\n",
    "    fmn = 0\n",
    "    for ojson in observations:\n",
    "        on = on+1\n",
    "        #ojson=o.serialize()\n",
    "        #print(json.dumps(ojson, indent=3))\n",
    "        if ojson['code']['text'] == 'Clinical status':\n",
    "            csn +=1\n",
    "            thisSubject['ClinStatus{}'.format(csn)] = ojson['valueCodeableConcept']['text']\n",
    "            #thisSubject['CS{}_code'.format(csn)] = ojson['code']['coding'][0]['code']\n",
    "        '''\n",
    "        elif ojson['code']['coding'][0]['code'] == 'FAMMEMB':\n",
    "            fmn +=1\n",
    "            thisSubject['FamMember{}'.format(fmn)] = ojson['valueCodeableConcept']['text']\n",
    "            thisSubject['FamMember{}_focus'.format(fmn)] = ojson['focus'][0]['reference']\n",
    "            #thisSubject['O{}_code'.format(on)] = ojson['code']['coding'][0]['code']\n",
    "\n",
    "        else:\n",
    "            thisSubject['O{}_valuetext'.format(on)] = ojson['valueCodeableConcept']['text']\n",
    "            thisSubject['O{}_code'.format(on)] = ojson['code']['coding'][0]['code']\n",
    "\n",
    "        print(\".\"*50)\n",
    "        '''\n",
    "\n",
    "    #print(\"Conditions\",'+'*20)\n",
    "    #resources = client.resources('Condition')\n",
    "    #conditions = resources.search(subject=patientID).fetch_all()\n",
    "    conditions = runQuery(f'Condition?patientID={patientID}')\n",
    "    cn = 0\n",
    "    for cjson in conditions:\n",
    "        cn += 1\n",
    "        #cjson=c.serialize()\n",
    "        #print(json.dumps(cjson, indent=3))\n",
    "        #print(\"+\"*50)\n",
    "        thisSubject['Cndtn{}_codetext'.format(cn)] = c['code']['text']\n",
    "        cdn = 0\n",
    "        if 'coding' in c['code']:\n",
    "            for cd in c['code']['coding']:\n",
    "                cdn +=1\n",
    "                thisSubject['Cndtn{}_code{}'.format(cn,cdn)] = cd['code']\n",
    "\n",
    "    mySubjects.append(thisSubject)\n",
    "    #print('_'*50)\n",
    "    \n",
    "df = pd.DataFrame(mySubjects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>ClinStatus1</th>\n",
       "      <th>ClinStatus2</th>\n",
       "      <th>Cndtn1_codetext</th>\n",
       "      <th>Cndtn1_code1</th>\n",
       "      <th>Cndtn1_code2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Patient/576466</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Patient/576465</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>ALL</td>\n",
       "      <td>MONDO:0004967</td>\n",
       "      <td>NCIT:C3167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Patient/576467</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Patient/576473</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NHL</td>\n",
       "      <td>MONDO:0018908</td>\n",
       "      <td>NCIT:C3211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Patient/576468</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Patient/576474</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkins Lymphoma</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Patient/576470</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkin's Lymphoma</td>\n",
       "      <td>MONDO:0004620</td>\n",
       "      <td>NCIT:C24182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Patient/576469</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Patient/576472</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Patient/576450</td>\n",
       "      <td>Reported Unknown</td>\n",
       "      <td>Reported Unknown</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patientID       ClinStatus1       ClinStatus2     Cndtn1_codetext  \\\n",
       "0  Patient/576466             Alive             Alive                 NaN   \n",
       "1  Patient/576465             Alive             Alive                 ALL   \n",
       "2  Patient/576467             Alive             Alive                 NaN   \n",
       "3  Patient/576473             Alive             Alive                 NHL   \n",
       "4  Patient/576468             Alive             Alive                  HL   \n",
       "5  Patient/576474             Alive             Alive   Hodgkins Lymphoma   \n",
       "6  Patient/576470             Alive             Alive  Hodgkin's Lymphoma   \n",
       "7  Patient/576469             Alive             Alive                 NaN   \n",
       "8  Patient/576472             Alive             Alive                 NaN   \n",
       "9  Patient/576450  Reported Unknown  Reported Unknown                  HL   \n",
       "\n",
       "    Cndtn1_code1 Cndtn1_code2  \n",
       "0            NaN          NaN  \n",
       "1  MONDO:0004967   NCIT:C3167  \n",
       "2            NaN          NaN  \n",
       "3  MONDO:0018908   NCIT:C3211  \n",
       "4  MONDO:0004952   NCIT:C9357  \n",
       "5  MONDO:0004952   NCIT:C9357  \n",
       "6  MONDO:0004620  NCIT:C24182  \n",
       "7            NaN          NaN  \n",
       "8            NaN          NaN  \n",
       "9  MONDO:0004952   NCIT:C9357  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use DRSMetaresolver to direct requests for DRS URIs to the right DRS server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Searching the GA4GH registry for org.ga4gh:drs services\n",
      "GA4GH registry unavailable, cannot get registered DRS services.\n",
      "Continuing with locally known DRS services.\n"
     ]
    }
   ],
   "source": [
    "from fasp.loc import DRSMetaResolver\n",
    "drsClient = DRSMetaResolver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set up a function to retrieve a DRS id from the FHIR server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'resourceType': 'Bundle',\n",
       " 'id': 'ea70c81d-a496-4425-8cba-fa5a7e06b50a',\n",
       " 'meta': {'lastUpdated': '2022-06-24T22:38:49.282+00:00'},\n",
       " 'type': 'searchset',\n",
       " 'total': 5,\n",
       " 'link': [{'relation': 'self',\n",
       "   'url': 'https://kf-api-fhir-service.kidsfirstdrc.org/DocumentReference?subject=Patient%2F51063'}],\n",
       " 'entry': [{'fullUrl': 'https://kf-api-fhir-service.kidsfirstdrc.org/DocumentReference/394851',\n",
       "   'resource': {'resourceType': 'DocumentReference',\n",
       "    'id': '394851',\n",
       "    'meta': {'versionId': '3',\n",
       "     'lastUpdated': '2022-01-18T23:05:04.173+00:00',\n",
       "     'source': '#pegoPWj2pe9YdkYf',\n",
       "     'profile': ['https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/drs-document-reference'],\n",
       "     'tag': [{'code': 'SD_YGVA0E1C'}]},\n",
       "    'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/genomic-files/',\n",
       "      'value': 'GF_61JFHQKF'},\n",
       "     {'system': 'urn:kids-first:unique-string',\n",
       "      'value': 'DocumentReference-SD_YGVA0E1C-GF_61JFHQKF'}],\n",
       "    'status': 'current',\n",
       "    'docStatus': 'final',\n",
       "    'type': {'text': 'gVCF'},\n",
       "    'subject': {'reference': 'Patient/51063'},\n",
       "    'securityLabel': [{'coding': [{'system': 'http://terminology.hl7.org/CodeSystem/v3-Confidentiality',\n",
       "        'code': 'R',\n",
       "        'display': 'restricted'}],\n",
       "      'text': 'True'},\n",
       "     {'text': 'SD_YGVA0E1C'},\n",
       "     {'coding': [{'code': 'c999'}], 'text': 'phs001228.c999'}],\n",
       "    'content': [{'attachment': {'url': 'drs://data.kidsfirstdrc.org/99ef25b4-c277-4c5a-b1cd-4490be373fdb'}},\n",
       "     {'attachment': {'extension': [{'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/file-size',\n",
       "         'valueDecimal': 11356427306},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'md5'}],\n",
       "          'text': '962fbfde22ed2c8f389230a7d986ecfe'}},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'sha256'}],\n",
       "          'text': '6e4abb6c1bb89aefb5e8e502dfa16a75e9fafb37aac3e828c5ee2a888ea165d3'}}],\n",
       "       'url': 's3://kf-study-us-east-1-prd-sd-ygva0e1c/harmonized/gvcf/e3cf2dae-d144-422e-a979-ffb935b38001.g.vcf.gz',\n",
       "       'title': 'e3cf2dae-d144-422e-a979-ffb935b38001.g.vcf.gz'},\n",
       "      'format': {'display': 'gVCF'}}]},\n",
       "   'search': {'mode': 'match'}},\n",
       "  {'fullUrl': 'https://kf-api-fhir-service.kidsfirstdrc.org/DocumentReference/394848',\n",
       "   'resource': {'resourceType': 'DocumentReference',\n",
       "    'id': '394848',\n",
       "    'meta': {'versionId': '3',\n",
       "     'lastUpdated': '2022-01-18T23:05:04.265+00:00',\n",
       "     'source': '#iVKLr5kiC9K32hHo',\n",
       "     'profile': ['https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/drs-document-reference'],\n",
       "     'tag': [{'code': 'SD_YGVA0E1C'}]},\n",
       "    'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/genomic-files/',\n",
       "      'value': 'GF_GKXYRKB4'},\n",
       "     {'system': 'urn:kids-first:unique-string',\n",
       "      'value': 'DocumentReference-SD_YGVA0E1C-GF_GKXYRKB4'}],\n",
       "    'status': 'current',\n",
       "    'docStatus': 'final',\n",
       "    'type': {'text': 'gVCF Index'},\n",
       "    'subject': {'reference': 'Patient/51063'},\n",
       "    'securityLabel': [{'coding': [{'system': 'http://terminology.hl7.org/CodeSystem/v3-Confidentiality',\n",
       "        'code': 'R',\n",
       "        'display': 'restricted'}],\n",
       "      'text': 'True'},\n",
       "     {'text': 'SD_YGVA0E1C'},\n",
       "     {'coding': [{'code': 'c999'}], 'text': 'phs001228.c999'}],\n",
       "    'content': [{'attachment': {'url': 'drs://data.kidsfirstdrc.org/5d998db4-24a2-41cb-911e-2db41439e33a'}},\n",
       "     {'attachment': {'extension': [{'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/file-size',\n",
       "         'valueDecimal': 4287606},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'md5'}],\n",
       "          'text': '783ea5848609db283c65bb53a78b0577'}},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'sha256'}],\n",
       "          'text': 'bc6b358a29fd96fde429f87018322058d0b03a202441c4378134ae4acef210ea'}}],\n",
       "       'url': 's3://kf-study-us-east-1-prd-sd-ygva0e1c/harmonized/gvcf/e3cf2dae-d144-422e-a979-ffb935b38001.g.vcf.gz.tbi',\n",
       "       'title': 'e3cf2dae-d144-422e-a979-ffb935b38001.g.vcf.gz.tbi'},\n",
       "      'format': {'display': 'tbi'}}]},\n",
       "   'search': {'mode': 'match'}},\n",
       "  {'fullUrl': 'https://kf-api-fhir-service.kidsfirstdrc.org/DocumentReference/394846',\n",
       "   'resource': {'resourceType': 'DocumentReference',\n",
       "    'id': '394846',\n",
       "    'meta': {'versionId': '3',\n",
       "     'lastUpdated': '2022-01-18T23:05:04.209+00:00',\n",
       "     'source': '#iOj7WTL5iCl8q8Sk',\n",
       "     'profile': ['https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/drs-document-reference'],\n",
       "     'tag': [{'code': 'SD_YGVA0E1C'}]},\n",
       "    'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/genomic-files/',\n",
       "      'value': 'GF_6D4KY8PE'},\n",
       "     {'system': 'urn:kids-first:unique-string',\n",
       "      'value': 'DocumentReference-SD_YGVA0E1C-GF_6D4KY8PE'}],\n",
       "    'status': 'current',\n",
       "    'docStatus': 'final',\n",
       "    'type': {'text': 'Aligned Reads'},\n",
       "    'subject': {'reference': 'Patient/51063'},\n",
       "    'securityLabel': [{'coding': [{'system': 'http://terminology.hl7.org/CodeSystem/v3-Confidentiality',\n",
       "        'code': 'R',\n",
       "        'display': 'restricted'}],\n",
       "      'text': 'True'},\n",
       "     {'text': 'SD_YGVA0E1C'},\n",
       "     {'coding': [{'code': 'c999'}], 'text': 'phs001228.c999'}],\n",
       "    'content': [{'attachment': {'url': 'drs://data.kidsfirstdrc.org/520bd083-82c1-4f00-abf5-ca1783d751aa'}},\n",
       "     {'attachment': {'extension': [{'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/file-size',\n",
       "         'valueDecimal': 59215639645},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'md5'}],\n",
       "          'text': '7b595bb3dc46b5df4107c395d12ccb47'}},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'sha256'}],\n",
       "          'text': 'ad41e608ed9499b4f410ffdca88fea9e4598d5f0a1f73aebc8d1bc628f7123b8'}}],\n",
       "       'url': 's3://kf-seq-data-washu/EwingSarcoma//db3cb5310167464996be1ac9a3cd1468.bam',\n",
       "       'title': 'db3cb5310167464996be1ac9a3cd1468.bam'},\n",
       "      'format': {'display': 'bam'}}]},\n",
       "   'search': {'mode': 'match'}},\n",
       "  {'fullUrl': 'https://kf-api-fhir-service.kidsfirstdrc.org/DocumentReference/394845',\n",
       "   'resource': {'resourceType': 'DocumentReference',\n",
       "    'id': '394845',\n",
       "    'meta': {'versionId': '3',\n",
       "     'lastUpdated': '2022-01-18T23:05:04.270+00:00',\n",
       "     'source': '#bmnyzlzZSaBI3uHk',\n",
       "     'profile': ['https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/drs-document-reference'],\n",
       "     'tag': [{'code': 'SD_YGVA0E1C'}]},\n",
       "    'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/genomic-files/',\n",
       "      'value': 'GF_H4ZB2F6F'},\n",
       "     {'system': 'urn:kids-first:unique-string',\n",
       "      'value': 'DocumentReference-SD_YGVA0E1C-GF_H4ZB2F6F'}],\n",
       "    'status': 'current',\n",
       "    'docStatus': 'final',\n",
       "    'type': {'text': 'Aligned Reads'},\n",
       "    'subject': {'reference': 'Patient/51063'},\n",
       "    'securityLabel': [{'coding': [{'system': 'http://terminology.hl7.org/CodeSystem/v3-Confidentiality',\n",
       "        'code': 'R',\n",
       "        'display': 'restricted'}],\n",
       "      'text': 'True'},\n",
       "     {'text': 'SD_YGVA0E1C'},\n",
       "     {'coding': [{'code': 'c999'}], 'text': 'phs001228.c999'}],\n",
       "    'content': [{'attachment': {'url': 'drs://data.kidsfirstdrc.org/5558e693-4dd6-4cd8-a0f9-88417acdbabb'}},\n",
       "     {'attachment': {'extension': [{'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/file-size',\n",
       "         'valueDecimal': 18142513567},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'md5'}],\n",
       "          'text': '99142a3673a964c919522d10c28f224e'}},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'sha256'}],\n",
       "          'text': '06c49545bfdf9d3f3971ccaf0d6e997a2eaaef8b8f4fadd0e1d7dc33ad182008'}}],\n",
       "       'url': 's3://kf-study-us-east-1-prd-sd-ygva0e1c/harmonized/cram/e3cf2dae-d144-422e-a979-ffb935b38001.cram',\n",
       "       'title': 'e3cf2dae-d144-422e-a979-ffb935b38001.cram'},\n",
       "      'format': {'display': 'cram'}}]},\n",
       "   'search': {'mode': 'match'}},\n",
       "  {'fullUrl': 'https://kf-api-fhir-service.kidsfirstdrc.org/DocumentReference/394844',\n",
       "   'resource': {'resourceType': 'DocumentReference',\n",
       "    'id': '394844',\n",
       "    'meta': {'versionId': '3',\n",
       "     'lastUpdated': '2022-01-18T23:05:04.230+00:00',\n",
       "     'source': '#WcReMSjDdWlmRM7Z',\n",
       "     'profile': ['https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/drs-document-reference'],\n",
       "     'tag': [{'code': 'SD_YGVA0E1C'}]},\n",
       "    'identifier': [{'system': 'https://kf-api-dataservice.kidsfirstdrc.org/genomic-files/',\n",
       "      'value': 'GF_78D87PFZ'},\n",
       "     {'system': 'urn:kids-first:unique-string',\n",
       "      'value': 'DocumentReference-SD_YGVA0E1C-GF_78D87PFZ'}],\n",
       "    'status': 'current',\n",
       "    'docStatus': 'final',\n",
       "    'type': {'text': 'Aligned Reads Index'},\n",
       "    'subject': {'reference': 'Patient/51063'},\n",
       "    'securityLabel': [{'coding': [{'system': 'http://terminology.hl7.org/CodeSystem/v3-Confidentiality',\n",
       "        'code': 'R',\n",
       "        'display': 'restricted'}],\n",
       "      'text': 'True'},\n",
       "     {'text': 'SD_YGVA0E1C'},\n",
       "     {'coding': [{'code': 'c999'}], 'text': 'phs001228.c999'}],\n",
       "    'content': [{'attachment': {'url': 'drs://data.kidsfirstdrc.org/7e282897-bbb3-49a2-b940-0e20c67f75f9'}},\n",
       "     {'attachment': {'extension': [{'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/file-size',\n",
       "         'valueDecimal': 1062966},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'md5'}],\n",
       "          'text': 'fb94ae1f2a24a0b96746815759034fc5'}},\n",
       "        {'url': 'https://nih-ncpi.github.io/ncpi-fhir-ig/StructureDefinition/hashes',\n",
       "         'valueCodeableConcept': {'coding': [{'display': 'sha256'}],\n",
       "          'text': '1336c3d724bcca2c9b15ea44cd5d4cbf8e95d9a767a9a152ac5555d8a2b3daa2'}}],\n",
       "       'url': 's3://kf-study-us-east-1-prd-sd-ygva0e1c/harmonized/cram/e3cf2dae-d144-422e-a979-ffb935b38001.cram.crai',\n",
       "       'title': 'e3cf2dae-d144-422e-a979-ffb935b38001.cram.crai'},\n",
       "      'format': {'display': 'crai'}}]},\n",
       "   'search': {'mode': 'match'}}]}"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = s.get(f\"{FHIR_SERVER}/DocumentReference?subject=Patient/51063\")\n",
    "bundle = r.json()\n",
    "print_bundle(bundle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getFiles(subject_id, docFormat):\n",
    "    #r = s.get(f\"{FHIR_SERVER}/DocumentReference?subject=Patient/{subject_id}\")\n",
    "    #bundle = r.json()\n",
    "    #resources = resources.search(subject=subject_id).limit(1000)\n",
    "    # The following works for the data for certain studies/patients, but not others\n",
    "    # resources.search(subject=subject_id, format__text=docFormat)\n",
    "    # So we do a query just based on subject, and then filter the results ourselves\n",
    "    # The following filters for the required format\n",
    "    myDocs = []\n",
    "    retval = None\n",
    "    documents =runQuery(f\"DocumentReference?subject=Patient/{subject_id}\")\n",
    "    #documents = list(map(lambda e: e['resource'], bundle['entry']))\n",
    "    #with open(\"docs.json\", \"w\") as f:\n",
    "        #print(json.dump(documents, f, indent=3))\n",
    "        #print(json.dumps(documents[0],indent=3))\n",
    "\n",
    "    print(\"# of documents for subject {} :{}\".format(subject_id, len(documents)))\n",
    "\n",
    "    for d in documents:\n",
    "        #djson = d.json()\n",
    "        #print(json.dumps(d, indent=3))\n",
    "\n",
    "        #print(d['content'][0])\n",
    "        #print(d['content'][1]['format']['display'])\n",
    "\n",
    "\n",
    "        if d['content'][1]['format']['display'] == docFormat:\n",
    "           retval = d['content'][0]['attachment']['url']\n",
    "    return retval\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total  Resources: 7\n",
      "# of documents for subject 51020 :7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'drs://data.kidsfirstdrc.org/e0bd6fcc-7895-4392-9487-c5559ec5d9a1'"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drsuri = getFiles('51020', 'bam')\n",
    "drsuri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total  Resources: 5\n",
      "# of documents for subject 49936 :5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'c2bcf2f7-e262-4e37-9262-217a647e9186'"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drsuri = getFiles('49936', 'bam')\n",
    "drsuri\n",
    "drs_id = drsuri.split('/')[-1]\n",
    "drs_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Iterate over each subject to submit the correspinding bam file to the Seven Bridges WES server. The worflow will execute SAMTools Stats on each file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'individual': {'reference': '51020'}}]\n"
     ]
    }
   ],
   "source": [
    "subj_id = {'individual':{'reference':'51020'}}\n",
    "subjects =[subj_id]\n",
    "print(subjects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total  Resources: 7\n",
      "# of documents for subject 51020 :7\n",
      "drs://data.kidsfirstdrc.org/e0bd6fcc-7895-4392-9487-c5559ec5d9a1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'e0bd6fcc-7895-4392-9487-c5559ec5d9a1'"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drsuri = getFiles(subjects[0]['individual']['reference'], 'bam')\n",
    "print(drsuri)\n",
    "drs_id = drsuri.split('/')[-1]\n",
    "drs_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fasp.loc import kfDRSClient\n",
    "drsClient = kfDRSClient(\"~/.keys/kf_credentials.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'access_methods': [{'access_id': 's3',\n",
       "   'access_url': {'url': 's3://kf-seq-data-washu/EwingSarcoma//db3cb5310167464996be1ac9a3cd1468.bam'},\n",
       "   'region': '',\n",
       "   'type': 's3'}],\n",
       " 'aliases': [],\n",
       " 'checksums': [{'checksum': 'ad41e608ed9499b4f410ffdca88fea9e4598d5f0a1f73aebc8d1bc628f7123b8',\n",
       "   'type': 'sha256'},\n",
       "  {'checksum': '7b595bb3dc46b5df4107c395d12ccb47', 'type': 'md5'}],\n",
       " 'created_time': '2021-07-13T17:17:51.162204',\n",
       " 'description': None,\n",
       " 'form': 'object',\n",
       " 'id': '520bd083-82c1-4f00-abf5-ca1783d751aa',\n",
       " 'mime_type': 'application/json',\n",
       " 'name': 'db3cb5310167464996be1ac9a3cd1468.bam',\n",
       " 'self_uri': 'drs://dg.F82A1A:520bd083-82c1-4f00-abf5-ca1783d751aa',\n",
       " 'size': 59215639645,\n",
       " 'updated_time': '2021-09-14T14:18:21.113043',\n",
       " 'version': 'e01b4f4f'}"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drsClient.getObject(drs_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unauthorized for that DRS id\n"
     ]
    }
   ],
   "source": [
    "drsClient.getAccessURL(drs_id, 's3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running /var/folders/wz/jjbxsnr13v7dkw2jkbpmqd_dly65gq/T/ipykernel_73464/261786707.py\n",
      "Patient/51063\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51063 :5\n",
      "subject=51063, DRS URI=drs://data.kidsfirstdrc.org/520bd083-82c1-4f00-abf5-ca1783d751aa\n",
      "{'access_methods': [{'access_id': 's3', 'access_url': {'url': 's3://kf-seq-data-washu/EwingSarcoma//db3cb5310167464996be1ac9a3cd1468.bam'}, 'region': '', 'type': 's3'}], 'aliases': [], 'checksums': [{'checksum': 'ad41e608ed9499b4f410ffdca88fea9e4598d5f0a1f73aebc8d1bc628f7123b8', 'type': 'sha256'}, {'checksum': '7b595bb3dc46b5df4107c395d12ccb47', 'type': 'md5'}], 'created_time': '2021-07-13T17:17:51.162204', 'description': None, 'form': 'object', 'id': '520bd083-82c1-4f00-abf5-ca1783d751aa', 'mime_type': 'application/json', 'name': 'db3cb5310167464996be1ac9a3cd1468.bam', 'self_uri': 'drs://dg.F82A1A:520bd083-82c1-4f00-abf5-ca1783d751aa', 'size': 59215639645, 'updated_time': '2021-09-14T14:18:21.113043', 'version': 'e01b4f4f'}\n",
      "Unauthorized for that DRS id\n",
      "could not get URL for subject=51063\n",
      "Patient/51052\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51052 :5\n",
      "subject=51052, DRS URI=drs://data.kidsfirstdrc.org/805860ce-d0e3-406c-b652-c60f72305eb9\n",
      "{'access_methods': [{'access_id': 's3', 'access_url': {'url': 's3://kf-seq-data-washu/EwingSarcoma//9ac3002273444eb7897911da173092bf.bam'}, 'region': '', 'type': 's3'}], 'aliases': [], 'checksums': [{'checksum': '019701794cba931ae44a5fef6ff7bb01d614d941de5c70754c18e7c12e1c6f5d', 'type': 'sha256'}, {'checksum': '8444b75d5c4e3e659a9f1a056f83f463', 'type': 'md5'}], 'created_time': '2021-07-13T18:36:40.262787', 'description': None, 'form': 'object', 'id': '805860ce-d0e3-406c-b652-c60f72305eb9', 'mime_type': 'application/json', 'name': '9ac3002273444eb7897911da173092bf.bam', 'self_uri': 'drs://dg.F82A1A:805860ce-d0e3-406c-b652-c60f72305eb9', 'size': 63251841828, 'updated_time': '2022-05-10T21:25:07.655571', 'version': 'c633c683'}\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:8f3b22d7-a02e-461b-ae7a-dc2a11e5022a\n",
      "Patient/51059\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51059 :5\n",
      "subject=51059, DRS URI=drs://data.kidsfirstdrc.org/d41fce40-b916-4d28-a294-5364adfe08df\n",
      "{'access_methods': [{'access_id': 's3', 'access_url': {'url': 's3://kf-seq-data-washu/EwingSarcoma//d443b834e4e6410a97e0d17cd90d539c.bam'}, 'region': '', 'type': 's3'}], 'aliases': [], 'checksums': [{'checksum': 'a2a36e052b5ce7c4f00346f27826c4b257c40d3032709911163dff29e1d6cad2', 'type': 'sha256'}, {'checksum': '3dc21d32e8621627f54678be06792cdc', 'type': 'md5'}], 'created_time': '2021-07-13T17:22:26.613222', 'description': None, 'form': 'object', 'id': 'd41fce40-b916-4d28-a294-5364adfe08df', 'mime_type': 'application/json', 'name': 'd443b834e4e6410a97e0d17cd90d539c.bam', 'self_uri': 'drs://dg.F82A1A:d41fce40-b916-4d28-a294-5364adfe08df', 'size': 52618180450, 'updated_time': '2022-05-10T15:41:21.439920', 'version': '0d2080aa'}\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:5aedde42-7a27-4991-844a-6fbfc17f932e\n",
      "Patient/51064\n",
      "Total  Resources: 0\n",
      "# of documents for subject 51064 :0\n",
      "No bam for subject=51064\n",
      "Patient/51056\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51056 :5\n",
      "subject=51056, DRS URI=drs://data.kidsfirstdrc.org/938490ec-11b6-4a16-ac42-e81df126b828\n",
      "{'access_methods': [{'access_id': 's3', 'access_url': {'url': 's3://kf-seq-data-washu/EwingSarcoma//0c3fca77b8254344902a7db192621977.bam'}, 'region': '', 'type': 's3'}], 'aliases': [], 'checksums': [{'checksum': '70066acc15e890e8e6cfb17f262e196e3a73714260137bf5ae4bd16adbbb068f', 'type': 'sha256'}, {'checksum': 'a36c01e45f15b72007643eb8c153b1e8', 'type': 'md5'}], 'created_time': '2021-07-13T17:43:47.512555', 'description': None, 'form': 'object', 'id': '938490ec-11b6-4a16-ac42-e81df126b828', 'mime_type': 'application/json', 'name': '0c3fca77b8254344902a7db192621977.bam', 'self_uri': 'drs://dg.F82A1A:938490ec-11b6-4a16-ac42-e81df126b828', 'size': 52753914562, 'updated_time': '2022-05-10T16:56:30.248544', 'version': 'd74bbf9c'}\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:46978154-0027-45d0-9818-39fe8fbf702f\n",
      "Patient/51051\n",
      "Total  Resources: 5\n",
      "# of documents for subject 51051 :5\n",
      "subject=51051, DRS URI=drs://data.kidsfirstdrc.org/ea26fe12-a138-4325-a783-d881edd7a302\n",
      "{'access_methods': [{'access_id': 's3', 'access_url': {'url': 's3://kf-seq-data-washu/EwingSarcoma//27cebfe412624cc39f5e8ea84652ce3b.bam'}, 'region': '', 'type': 's3'}], 'aliases': [], 'checksums': [{'checksum': 'fdc487981c61661ffd97f5f7ff23a4dd4d4d47673eaeb1dd645575df75588654', 'type': 'sha256'}, {'checksum': '4bffc0ab3ec0c6461654530d10a1468d', 'type': 'md5'}], 'created_time': '2021-07-13T17:31:40.159121', 'description': None, 'form': 'object', 'id': 'ea26fe12-a138-4325-a783-d881edd7a302', 'mime_type': 'application/json', 'name': '27cebfe412624cc39f5e8ea84652ce3b.bam', 'self_uri': 'drs://dg.F82A1A:ea26fe12-a138-4325-a783-d881edd7a302', 'size': 49540489536, 'updated_time': '2022-05-10T20:18:11.269841', 'version': '3c4be406'}\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:5ca73a9a-93b3-4950-93cd-7bfcea344065\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>run_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51063</td>\n",
       "      <td>350180ed-0558-427c-bf71-25d89d666a71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51052</td>\n",
       "      <td>8f3b22d7-a02e-461b-ae7a-dc2a11e5022a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51059</td>\n",
       "      <td>5aedde42-7a27-4991-844a-6fbfc17f932e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51064</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51056</td>\n",
       "      <td>46978154-0027-45d0-9818-39fe8fbf702f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51051</td>\n",
       "      <td>5ca73a9a-93b3-4950-93cd-7bfcea344065</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  patientID                                run_id\n",
       "0     51063  350180ed-0558-427c-bf71-25d89d666a71\n",
       "1     51052  8f3b22d7-a02e-461b-ae7a-dc2a11e5022a\n",
       "2     51059  5aedde42-7a27-4991-844a-6fbfc17f932e\n",
       "3     51064                                  None\n",
       "4     51056  46978154-0027-45d0-9818-39fe8fbf702f\n",
       "5     51051  5ca73a9a-93b3-4950-93cd-7bfcea344065"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from fasp.workflow import sbcgcWESClient\n",
    "from fasp.runner import FASPRunner\n",
    "import datetime\n",
    "\n",
    "faspRunner = FASPRunner()\n",
    "runNote = 'KF Ewings file retrieved from NCPI FHIR Server and computed on SB'\n",
    "\n",
    "# Set up WES Client for WES server of our choice\n",
    "settings = faspRunner.settings\n",
    "wesClient = sbcgcWESClient(settings['SevenBridgesProject'])\n",
    "\n",
    "drsClient = kfDRSClient(\"~/.keys/kf_credentials.json\")\n",
    "\n",
    "run_ids = []\n",
    "\n",
    "# repeat for each row of the query\n",
    "limit = 5\n",
    "nn= 0\n",
    "for pjson in patients:\n",
    "    patientID = pjson['id']\n",
    "    print (f\"Patient/{patientID}\")\n",
    "    drsuri = getFiles(patientID, 'bam')\n",
    "    \n",
    "    #DRS ids stored in the NCPI FHIR server include the host prefix. Just get the id\n",
    "    #subjectID = sjson['id']\n",
    "    \n",
    "    if drsuri:\n",
    "        print(\"subject={}, DRS URI={}\".format(patientID, drsuri))\n",
    "        #drsID = drsuri.split('/')[-1]\n",
    "        #Call the DRS server to get file details\n",
    "        # We're going to keep the file size\n",
    "        drs_id = drsuri.split('/')[-1]\n",
    "        objInfo = drsClient.getObject(drs_id)\n",
    "        print(objInfo)\n",
    "        fileSize = objInfo['size']\n",
    "        # Use DRS to get the URL\n",
    "        url = drsClient.getAccessURL(drs_id,'s3')\n",
    "        # Step 3 - Run a pipeline on the file at the drs url\n",
    "        if url:\n",
    "            outfile = \"{}.txt\".format(patientID)\n",
    "            pipeline_id = wesClient.runWorkflow(url, outfile)\n",
    "            print('submitted:{}'.format(pipeline_id))\n",
    "        else:\n",
    "            print(\"could not get URL for subject={}\".format(patientID))\n",
    "    else:\n",
    "        print(\"No bam for subject={}\".format(patientID))\n",
    "        pipeline_id = None\n",
    "        fileSize = None\n",
    "        \n",
    "\n",
    "    via = 'WES'\n",
    "    note = 'WES samtools on KF test'\n",
    "    time = datetime.datetime.now().strftime(\"%m/%d/%Y, %H:%M:%S\")\n",
    "    faspRunner.logRun(time, via, note,  pipeline_id, outfile, str(fileSize),\n",
    "        client, drsClient, wesClient)\n",
    "    run_ids.append({\"patientID\":patientID, \"run_id\":pipeline_id})\n",
    "    \n",
    "    nn += 1\n",
    "    if nn > limit:\n",
    "        break\n",
    "\n",
    "runs_df = pd.DataFrame(run_ids)\n",
    "runs_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check status of each run via the WES server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no run for 51064\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>run_id</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51063</td>\n",
       "      <td>350180ed-0558-427c-bf71-25d89d666a71</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51052</td>\n",
       "      <td>8f3b22d7-a02e-461b-ae7a-dc2a11e5022a</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51059</td>\n",
       "      <td>5aedde42-7a27-4991-844a-6fbfc17f932e</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51064</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51056</td>\n",
       "      <td>46978154-0027-45d0-9818-39fe8fbf702f</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51051</td>\n",
       "      <td>5ca73a9a-93b3-4950-93cd-7bfcea344065</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  patientID                                run_id    status\n",
       "0     51063  350180ed-0558-427c-bf71-25d89d666a71  COMPLETE\n",
       "1     51052  8f3b22d7-a02e-461b-ae7a-dc2a11e5022a  COMPLETE\n",
       "2     51059  5aedde42-7a27-4991-844a-6fbfc17f932e  COMPLETE\n",
       "3     51064                                  None       NaN\n",
       "4     51056  46978154-0027-45d0-9818-39fe8fbf702f  COMPLETE\n",
       "5     51051  5ca73a9a-93b3-4950-93cd-7bfcea344065  COMPLETE"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for index, row in runs_df.iterrows():\n",
    "    if row['run_id'] != None:\n",
    "        status = wesClient.getTaskStatus(row['run_id'])\n",
    "        runs_df.at[index, \"status\"] = status\n",
    "    else:\n",
    "        print ('no run for {}'.format(row['patientID']))\n",
    "runs_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define function to download a file and to extract the result from the output file from SAMTools Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import os\n",
    "def download(url, file_path):\n",
    "    '''Download a file from a URL to a local file path'''\n",
    "    with open(os.path.expanduser(file_path), \"wb\") as file:\n",
    "        response = requests.get(url)\n",
    "        file.write(response.content)\n",
    "        \n",
    "def getStats(filePath, statsList):\n",
    "    ''' Extract values from a SAMTools Stats results file'''\n",
    "    ### \n",
    "    \n",
    "    retDict = {}\n",
    "    f = open(filePath, \"r\")\n",
    "    for x in f:\n",
    "        if x.startswith('SN'):   \n",
    "            parts = x.split('\\t')\n",
    "            statName = parts[1].split(':')[0]\n",
    "            if statName in statsList:\n",
    "                retDict[statName] = parts[2].rstrip()\n",
    "    return retDict\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the completed workflows retrieve the results file and extract the required result values. Add the result values to a DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DRS URI for results for subject 51063: drs://cgc-ga4gh-api.sbgenomics.com/62b64d6a4e3edb6b1c397b59\n",
      "DRS URI for results for subject 51052: drs://cgc-ga4gh-api.sbgenomics.com/62b67ee14e3edb6b1c3988c3\n",
      "DRS URI for results for subject 51059: drs://cgc-ga4gh-api.sbgenomics.com/62b67cc24e3edb6b1c398859\n",
      "DRS URI for results for subject 51056: drs://cgc-ga4gh-api.sbgenomics.com/62b67cdaf08fea477057664d\n",
      "DRS URI for results for subject 51051: drs://cgc-ga4gh-api.sbgenomics.com/62b67ca44e3edb6b1c39883d\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>insert size average</th>\n",
       "      <th>insert size standard deviation</th>\n",
       "      <th>run_id</th>\n",
       "      <th>patientID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>364.3</td>\n",
       "      <td>83.7</td>\n",
       "      <td>350180ed-0558-427c-bf71-25d89d666a71</td>\n",
       "      <td>51063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>365.8</td>\n",
       "      <td>78.9</td>\n",
       "      <td>8f3b22d7-a02e-461b-ae7a-dc2a11e5022a</td>\n",
       "      <td>51052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>337.0</td>\n",
       "      <td>73.6</td>\n",
       "      <td>5aedde42-7a27-4991-844a-6fbfc17f932e</td>\n",
       "      <td>51059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>405.7</td>\n",
       "      <td>92.5</td>\n",
       "      <td>46978154-0027-45d0-9818-39fe8fbf702f</td>\n",
       "      <td>51056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>357.8</td>\n",
       "      <td>82.0</td>\n",
       "      <td>5ca73a9a-93b3-4950-93cd-7bfcea344065</td>\n",
       "      <td>51051</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  insert size average insert size standard deviation  \\\n",
       "0               364.3                           83.7   \n",
       "1               365.8                           78.9   \n",
       "2               337.0                           73.6   \n",
       "3               405.7                           92.5   \n",
       "4               357.8                           82.0   \n",
       "\n",
       "                                 run_id patientID  \n",
       "0  350180ed-0558-427c-bf71-25d89d666a71     51063  \n",
       "1  8f3b22d7-a02e-461b-ae7a-dc2a11e5022a     51052  \n",
       "2  5aedde42-7a27-4991-844a-6fbfc17f932e     51059  \n",
       "3  46978154-0027-45d0-9818-39fe8fbf702f     51056  \n",
       "4  5ca73a9a-93b3-4950-93cd-7bfcea344065     51051  "
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from fasp.loc import sbcgcDRSClient\n",
    "results_drs= sbcgcDRSClient(\"/Users/forei/.keys/sevenbridges_keys.json\", \"s3\")\n",
    "statsList = []\n",
    "for index, row in runs_df.iterrows():\n",
    "    if row['run_id'] != None and row['status'] == 'COMPLETE':\n",
    "        log = wesClient.getRunLog(row['run_id'])\n",
    "        results_drs_uri = log['outputs']['statistics']['path']\n",
    "        print(\"DRS URI for results for subject {}: {}\".format(row['patientID'] ,results_drs_uri))\n",
    "        resultsDRSID = results_drs_uri.split('/')[-1]\n",
    "        url = results_drs.getAccessURL(resultsDRSID,'s3')\n",
    "        fileName = log['outputs']['statistics']['name']\n",
    "        download(url, fileName)\n",
    "        statsRequired = ['insert size average','insert size standard deviation']\n",
    "        stats = getStats(fileName, statsRequired)\n",
    "        #print(stats)\n",
    "        stats['run_id'] = row['run_id']\n",
    "        stats['patientID'] = row['patientID']\n",
    "        statsList.append(stats)\n",
    "        os.remove(fileName)\n",
    "stats_df =  pd.DataFrame(statsList)\n",
    "stats_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>ClinStatus1</th>\n",
       "      <th>Cndtn1_codetext</th>\n",
       "      <th>Cndtn1_code1</th>\n",
       "      <th>Cndtn1_code2</th>\n",
       "      <th>insert size average</th>\n",
       "      <th>insert size standard deviation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Patient/576466</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>360.3</td>\n",
       "      <td>112.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Patient/576465</td>\n",
       "      <td>Alive</td>\n",
       "      <td>ALL</td>\n",
       "      <td>MONDO:0004967</td>\n",
       "      <td>NCIT:C3167</td>\n",
       "      <td>378.4</td>\n",
       "      <td>116.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Patient/576467</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>365.1</td>\n",
       "      <td>131.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Patient/576473</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NHL</td>\n",
       "      <td>MONDO:0018908</td>\n",
       "      <td>NCIT:C3211</td>\n",
       "      <td>346.4</td>\n",
       "      <td>107.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Patient/576468</td>\n",
       "      <td>Alive</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "      <td>350.7</td>\n",
       "      <td>113.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Patient/576474</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkins Lymphoma</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "      <td>371.8</td>\n",
       "      <td>98.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Patient/576470</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkin's Lymphoma</td>\n",
       "      <td>MONDO:0004620</td>\n",
       "      <td>NCIT:C24182</td>\n",
       "      <td>357.5</td>\n",
       "      <td>100.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Patient/576469</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>338.3</td>\n",
       "      <td>102.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Patient/576472</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Patient/576450</td>\n",
       "      <td>Reported Unknown</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "      <td>291.8</td>\n",
       "      <td>130.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patientID       ClinStatus1     Cndtn1_codetext   Cndtn1_code1  \\\n",
       "0  Patient/576466             Alive                 NaN            NaN   \n",
       "1  Patient/576465             Alive                 ALL  MONDO:0004967   \n",
       "2  Patient/576467             Alive                 NaN            NaN   \n",
       "3  Patient/576473             Alive                 NHL  MONDO:0018908   \n",
       "4  Patient/576468             Alive                  HL  MONDO:0004952   \n",
       "5  Patient/576474             Alive   Hodgkins Lymphoma  MONDO:0004952   \n",
       "6  Patient/576470             Alive  Hodgkin's Lymphoma  MONDO:0004620   \n",
       "7  Patient/576469             Alive                 NaN            NaN   \n",
       "8  Patient/576472             Alive                 NaN            NaN   \n",
       "9  Patient/576450  Reported Unknown                  HL  MONDO:0004952   \n",
       "\n",
       "  Cndtn1_code2 insert size average insert size standard deviation  \n",
       "0          NaN               360.3                          112.1  \n",
       "1   NCIT:C3167               378.4                          116.0  \n",
       "2          NaN               365.1                          131.9  \n",
       "3   NCIT:C3211               346.4                          107.9  \n",
       "4   NCIT:C9357               350.7                          113.6  \n",
       "5   NCIT:C9357               371.8                           98.6  \n",
       "6  NCIT:C24182               357.5                          100.7  \n",
       "7          NaN               338.3                          102.0  \n",
       "8          NaN                 NaN                            NaN  \n",
       "9   NCIT:C9357               291.8                          130.0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = pd.merge(\n",
    "    df,\n",
    "    stats_df,\n",
    "    how=\"outer\",\n",
    "    left_on='patientID',\n",
    "    right_on='patientID'\n",
    ")\n",
    "final_df.drop('run_id', axis=1, inplace=True)\n",
    "final_df.drop('ClinStatus2', axis=1, inplace=True)\n",
    "\n",
    "final_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}