notebooks/Kids First Familial Leukemia - FHIR and DRS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run WES workflow on Kid First DRS files searched on NCPI FHIR server\n",
    "\n",
    "This script demonstrates uses of the NCPI FHIR server to find patients and files from the Familial Leukemia study. The query returns DRS ids allowing the files accessed via the Kid's First Data Resource Center DRS server. The workflow is run on the Seven Bridges Cancer Genomics Cloud WES server. \n",
    "\n",
    "The FHIR server is accessed through the fhir-py Python client.\n",
    "\n",
    "### Set up the FHIR client\n",
    "Access is via a cookie obtained as described [here](https://github.com/NIH-NCPI/ncpi-api-fhir-service)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fhirpy import SyncFHIRClient\n",
    "import os\n",
    "import json\n",
    "\n",
    "\n",
    "endpoint = 'https://kf-api-fhir-service.kidsfirstdrc.org'\n",
    "full_cookie_path = os.path.expanduser('~/.keys/ncpi_prod_fhir_cookie.json')\n",
    "with open(full_cookie_path) as f:\n",
    "        cookies = json.load(f)\n",
    "\n",
    "client = SyncFHIRClient(endpoint, extra_headers=cookies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting value: line 1 column 1 (char 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/wz/jjbxsnr13v7dkw2jkbpmqd_dly65gq/T/ipykernel_71789/1239882772.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mstudyid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m577137\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mresources\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresources\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstudyid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0msubjects\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresources\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"# of subjects found {}:{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudyid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubjects\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m#for s in subjects:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/imfenvs/fasp/lib/python3.9/site-packages/fhirpy/base/lib.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    139\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mSyncSearchSet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAbstractSearchSet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m         bundle_data = self.client._fetch_resource(\n\u001b[0m\u001b[1;32m    142\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresource_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m         )\n",
      "\u001b[0;32m~/imfenvs/fasp/lib/python3.9/site-packages/fhirpy/base/lib.py\u001b[0m in \u001b[0;36m_fetch_resource\u001b[0;34m(self, path, params)\u001b[0m\n\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    135\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_fetch_resource\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 136\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/imfenvs/fasp/lib/python3.9/site-packages/fhirpy/base/lib.py\u001b[0m in \u001b[0;36m_do_request\u001b[0;34m(self, method, path, data, params)\u001b[0m\n\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;36m200\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 126\u001b[0;31m             return json.loads(\n\u001b[0m\u001b[1;32m    127\u001b[0m                 \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobject_hook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mAttrDict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m             ) if r.content else None\n",
      "\u001b[0;32m~/.pyenv/versions/3.9.7/lib/python3.9/json/__init__.py\u001b[0m in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    357\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mparse_constant\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m         \u001b[0mkw\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'parse_constant'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparse_constant\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 359\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.pyenv/versions/3.9.7/lib/python3.9/json/decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    335\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    336\u001b[0m         \"\"\"\n\u001b[0;32m--> 337\u001b[0;31m         \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    338\u001b[0m         \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    339\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.9.7/lib/python3.9/json/decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    353\u001b[0m             \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 355\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Expecting value\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    356\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
     ]
    }
   ],
   "source": [
    "resources = client.resources('ResearchSubject')\n",
    "studyid = 577137\n",
    "resources = resources.search(study=studyid).limit(10)\n",
    "subjects = resources.fetch()\n",
    "print(\"# of subjects found {}:{}\".format(studyid, len(subjects)))\n",
    "#for s in subjects:\n",
    "#    print(json.dumps(s.serialize(), indent=3))\n",
    "#    print('_'*50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve phenotypic data for each subject from FHIR\n",
    "Save the results to a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/forei/imfenvs/fasp/lib/python3.9/site-packages/pandas/compat/__init__.py:124: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "mySubjects = []\n",
    "for s in subjects:\n",
    "    sjson = s.serialize()\n",
    "    patientID = sjson['individual']['reference']\n",
    "    thisSubject = {\"patientID\":patientID}\n",
    "    patients = client.resources('Patient')\n",
    "    p = patients.search(_id=patientID).get()\n",
    "    pjson=p.serialize()\n",
    "    #print(json.dumps(pjson, indent=3))\n",
    "    \n",
    "\n",
    "    #print(\"Observations\",'.'*20)\n",
    "    resources = client.resources('Observation')\n",
    "    observations = resources.search(subject=patientID).fetch_all()\n",
    "    on = 0\n",
    "    csn = 0\n",
    "    fmn = 0\n",
    "    for o in observations:\n",
    "        on = on+1\n",
    "        ojson=o.serialize()\n",
    "        #print(json.dumps(ojson, indent=3))\n",
    "        if ojson['code']['text'] == 'Clinical status':\n",
    "            csn +=1\n",
    "            thisSubject['ClinStatus{}'.format(csn)] = ojson['valueCodeableConcept']['text']\n",
    "            #thisSubject['CS{}_code'.format(csn)] = ojson['code']['coding'][0]['code']\n",
    "        '''\n",
    "        elif ojson['code']['coding'][0]['code'] == 'FAMMEMB':\n",
    "            fmn +=1\n",
    "            thisSubject['FamMember{}'.format(fmn)] = ojson['valueCodeableConcept']['text']\n",
    "            thisSubject['FamMember{}_focus'.format(fmn)] = ojson['focus'][0]['reference']\n",
    "            #thisSubject['O{}_code'.format(on)] = ojson['code']['coding'][0]['code']\n",
    "\n",
    "        else:\n",
    "            thisSubject['O{}_valuetext'.format(on)] = ojson['valueCodeableConcept']['text']\n",
    "            thisSubject['O{}_code'.format(on)] = ojson['code']['coding'][0]['code']\n",
    "\n",
    "        print(\".\"*50)\n",
    "        '''\n",
    "\n",
    "    #print(\"Conditions\",'+'*20)\n",
    "    resources = client.resources('Condition')\n",
    "    conditions = resources.search(subject=patientID).fetch_all()\n",
    "    cn = 0\n",
    "    for c in conditions:\n",
    "        cn += 1\n",
    "        cjson=c.serialize()\n",
    "        #print(json.dumps(cjson, indent=3))\n",
    "        #print(\"+\"*50)\n",
    "        thisSubject['Cndtn{}_codetext'.format(cn)] = c['code']['text']\n",
    "        cdn = 0\n",
    "        if 'coding' in c['code']:\n",
    "            for cd in c['code']['coding']:\n",
    "                cdn +=1\n",
    "                thisSubject['Cndtn{}_code{}'.format(cn,cdn)] = cd['code']\n",
    "\n",
    "    mySubjects.append(thisSubject)\n",
    "    #print('_'*50)\n",
    "    \n",
    "df = pd.DataFrame(mySubjects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>ClinStatus1</th>\n",
       "      <th>ClinStatus2</th>\n",
       "      <th>Cndtn1_codetext</th>\n",
       "      <th>Cndtn1_code1</th>\n",
       "      <th>Cndtn1_code2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Patient/576466</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Patient/576465</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>ALL</td>\n",
       "      <td>MONDO:0004967</td>\n",
       "      <td>NCIT:C3167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Patient/576467</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Patient/576473</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NHL</td>\n",
       "      <td>MONDO:0018908</td>\n",
       "      <td>NCIT:C3211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Patient/576468</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Patient/576474</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkins Lymphoma</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Patient/576470</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkin's Lymphoma</td>\n",
       "      <td>MONDO:0004620</td>\n",
       "      <td>NCIT:C24182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Patient/576469</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Patient/576472</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Patient/576450</td>\n",
       "      <td>Reported Unknown</td>\n",
       "      <td>Reported Unknown</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patientID       ClinStatus1       ClinStatus2     Cndtn1_codetext  \\\n",
       "0  Patient/576466             Alive             Alive                 NaN   \n",
       "1  Patient/576465             Alive             Alive                 ALL   \n",
       "2  Patient/576467             Alive             Alive                 NaN   \n",
       "3  Patient/576473             Alive             Alive                 NHL   \n",
       "4  Patient/576468             Alive             Alive                  HL   \n",
       "5  Patient/576474             Alive             Alive   Hodgkins Lymphoma   \n",
       "6  Patient/576470             Alive             Alive  Hodgkin's Lymphoma   \n",
       "7  Patient/576469             Alive             Alive                 NaN   \n",
       "8  Patient/576472             Alive             Alive                 NaN   \n",
       "9  Patient/576450  Reported Unknown  Reported Unknown                  HL   \n",
       "\n",
       "    Cndtn1_code1 Cndtn1_code2  \n",
       "0            NaN          NaN  \n",
       "1  MONDO:0004967   NCIT:C3167  \n",
       "2            NaN          NaN  \n",
       "3  MONDO:0018908   NCIT:C3211  \n",
       "4  MONDO:0004952   NCIT:C9357  \n",
       "5  MONDO:0004952   NCIT:C9357  \n",
       "6  MONDO:0004620  NCIT:C24182  \n",
       "7            NaN          NaN  \n",
       "8            NaN          NaN  \n",
       "9  MONDO:0004952   NCIT:C9357  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use DRSMetaresolver to direct requests for DRS URIs to the right DRS server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Searching the GA4GH registry for org.ga4gh:drs services\n"
     ]
    }
   ],
   "source": [
    "from fasp.loc import DRSMetaResolver\n",
    "drsClient = DRSMetaResolver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set up a function to retrieve a DRS id from the FHIR server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getFiles(client, subject_id, docFormat):\n",
    "    resources = client.resources('DocumentReference')\n",
    "    resources = resources.search(subject=subject_id).limit(1000)\n",
    "    # The following works for the data for certain studies/patients, but not others\n",
    "    # resources.search(subject=subject_id, format__text=docFormat)\n",
    "    # So we do a query just based on subject, and then filter the results ourselves\n",
    "    documents = resources.fetch_all()\n",
    "    print(\"# of documents for subject {} :{}\".format(subject_id, len(documents)))\n",
    "    # The following filters for the required format\n",
    "    myDocs = []\n",
    "    retval = None\n",
    "    for d in documents:\n",
    "        djson = d.serialize()\n",
    "        if djson['content'][0]['format']['display'] == docFormat:\n",
    "            retval = djson['content'][0]['attachment']['url']\n",
    "    return retval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Iterate over each subject to submit the correspinding bam file to the Seven Bridges WES server. The worflow will execute SAMTools Stats on each file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running /var/folders/wz/jjbxsnr13v7dkw2jkbpmqd_dly65gq/T/ipykernel_39426/1934627716.py\n",
      "# of documents for subject Patient/576466 :8\n",
      "subject=Patient/576466, DRS URI=drs://data.kidsfirstdrc.org/e6eecb29-1ae4-4f65-ae83-9ecf1c496de1\n",
      "id:e6eecb29-1ae4-4f65-ae83-9ecf1c496de1\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:4f874352-27fa-4573-9b68-a5394bdf0e0c\n",
      "# of documents for subject Patient/576465 :8\n",
      "subject=Patient/576465, DRS URI=drs://data.kidsfirstdrc.org/a57f1be2-d756-4f47-810a-6455e7431aba\n",
      "id:a57f1be2-d756-4f47-810a-6455e7431aba\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:f9e75d07-9ce1-47bc-b5a7-34d4554e2903\n",
      "# of documents for subject Patient/576467 :8\n",
      "subject=Patient/576467, DRS URI=drs://data.kidsfirstdrc.org/f7448bea-ea5c-4e91-86e2-1aa41c3dc5c4\n",
      "id:f7448bea-ea5c-4e91-86e2-1aa41c3dc5c4\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:27b3d723-c811-4559-9fe2-f926b248e48b\n",
      "# of documents for subject Patient/576473 :8\n",
      "subject=Patient/576473, DRS URI=drs://data.kidsfirstdrc.org/5b445317-67fe-4e90-9d7d-faeda7cdbfb3\n",
      "id:5b445317-67fe-4e90-9d7d-faeda7cdbfb3\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:1c0cdacb-2b3f-4612-b9d8-c5dce9d60b5a\n",
      "# of documents for subject Patient/576468 :8\n",
      "subject=Patient/576468, DRS URI=drs://data.kidsfirstdrc.org/94f11311-2e6d-4c1c-90f4-6b4e774a0981\n",
      "id:94f11311-2e6d-4c1c-90f4-6b4e774a0981\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:e24917d7-035c-4459-b7da-58110f914e9d\n",
      "# of documents for subject Patient/576474 :8\n",
      "subject=Patient/576474, DRS URI=drs://data.kidsfirstdrc.org/d54b122a-c258-4f2f-98d8-91934aa83a53\n",
      "id:d54b122a-c258-4f2f-98d8-91934aa83a53\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:aca24b6e-5fa7-4ba2-b7f4-142db68a0991\n",
      "# of documents for subject Patient/576470 :8\n",
      "subject=Patient/576470, DRS URI=drs://data.kidsfirstdrc.org/0bbc58b9-0a5b-40be-83be-d6372c0a7e1f\n",
      "id:0bbc58b9-0a5b-40be-83be-d6372c0a7e1f\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:1f64713f-a679-4e29-a049-5400e7f89030\n",
      "# of documents for subject Patient/576469 :8\n",
      "subject=Patient/576469, DRS URI=drs://data.kidsfirstdrc.org/da1918f8-016a-47fb-92cd-470b0f4dc77f\n",
      "id:da1918f8-016a-47fb-92cd-470b0f4dc77f\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:87459b70-3e01-4bf2-85a3-e39df8941a53\n",
      "# of documents for subject Patient/576472 :6\n",
      "No bam for subject=Patient/576472\n",
      "# of documents for subject Patient/576450 :8\n",
      "subject=Patient/576450, DRS URI=drs://data.kidsfirstdrc.org/e6d4efe4-06e5-483d-aa15-bdbcdb9bda7a\n",
      "id:e6d4efe4-06e5-483d-aa15-bdbcdb9bda7a\n",
      "sending to: kfDRSClient\n",
      "https://cgc-ga4gh-api.sbgenomics.com/ga4gh/wes/v1\n",
      "submitted:60bf9c82-a7f8-4bb5-9482-33c157da3f70\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>run_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Patient/576466</td>\n",
       "      <td>4f874352-27fa-4573-9b68-a5394bdf0e0c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Patient/576465</td>\n",
       "      <td>f9e75d07-9ce1-47bc-b5a7-34d4554e2903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Patient/576467</td>\n",
       "      <td>27b3d723-c811-4559-9fe2-f926b248e48b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Patient/576473</td>\n",
       "      <td>1c0cdacb-2b3f-4612-b9d8-c5dce9d60b5a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Patient/576468</td>\n",
       "      <td>e24917d7-035c-4459-b7da-58110f914e9d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Patient/576474</td>\n",
       "      <td>aca24b6e-5fa7-4ba2-b7f4-142db68a0991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Patient/576470</td>\n",
       "      <td>1f64713f-a679-4e29-a049-5400e7f89030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Patient/576469</td>\n",
       "      <td>87459b70-3e01-4bf2-85a3-e39df8941a53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Patient/576472</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Patient/576450</td>\n",
       "      <td>60bf9c82-a7f8-4bb5-9482-33c157da3f70</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patientID                                run_id\n",
       "0  Patient/576466  4f874352-27fa-4573-9b68-a5394bdf0e0c\n",
       "1  Patient/576465  f9e75d07-9ce1-47bc-b5a7-34d4554e2903\n",
       "2  Patient/576467  27b3d723-c811-4559-9fe2-f926b248e48b\n",
       "3  Patient/576473  1c0cdacb-2b3f-4612-b9d8-c5dce9d60b5a\n",
       "4  Patient/576468  e24917d7-035c-4459-b7da-58110f914e9d\n",
       "5  Patient/576474  aca24b6e-5fa7-4ba2-b7f4-142db68a0991\n",
       "6  Patient/576470  1f64713f-a679-4e29-a049-5400e7f89030\n",
       "7  Patient/576469  87459b70-3e01-4bf2-85a3-e39df8941a53\n",
       "8  Patient/576472                                  None\n",
       "9  Patient/576450  60bf9c82-a7f8-4bb5-9482-33c157da3f70"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from fasp.workflow import sbcgcWESClient\n",
    "from fasp.runner import FASPRunner\n",
    "import datetime\n",
    "\n",
    "faspRunner = FASPRunner()\n",
    "runNote = 'KF Leukemia bam files retrieved from NCPI FHIR Server and computed on SB'\n",
    "\n",
    "# Set up WES Client for WES server of our choice\n",
    "settings = faspRunner.settings\n",
    "wesClient = sbcgcWESClient(settings['SevenBridgesProject'])\n",
    "\n",
    "run_ids = []\n",
    "\n",
    "# repeat for each row of the query\n",
    "for s in subjects:\n",
    "    sjson = s.serialize()\n",
    "    patientID = sjson['individual']['reference']\n",
    "    drsuri = getFiles(client, patientID, 'bam')\n",
    "    \n",
    "    #DRS ids stored in the NCPI FHIR server include the host prefix. Just get the id\n",
    "    #subjectID = sjson['id']\n",
    "    \n",
    "    if drsuri:\n",
    "        print(\"subject={}, DRS URI={}\".format(patientID, drsuri))\n",
    "        #drsID = drsuri.split('/')[-1]\n",
    "        #Call the DRS server to get file details\n",
    "        # We're going to keep the file size\n",
    "        objInfo = drsClient.getObject2(drsuri)\n",
    "        fileSize = objInfo['size']\n",
    "        # Use DRS to get the URL\n",
    "        url = drsClient.getAccessURL2(drsuri,'s3')\n",
    "        # Step 3 - Run a pipeline on the file at the drs url\n",
    "        outfile = \"{}.txt\".format(sjson['id'])\n",
    "        pipeline_id = wesClient.runWorkflow(url, outfile)\n",
    "        print('submitted:{}'.format(pipeline_id))\n",
    "    else:\n",
    "        print(\"No bam for subject={}\".format(patientID))\n",
    "        pipeline_id = None\n",
    "        fileSize = None\n",
    "        \n",
    "\n",
    "    via = 'WES'\n",
    "    note = 'WES samtools on KF Leukemia'\n",
    "    time = datetime.datetime.now().strftime(\"%m/%d/%Y, %H:%M:%S\")\n",
    "    faspRunner.logRun(time, via, note,  pipeline_id, outfile, str(fileSize),\n",
    "        client, drsClient, wesClient)\n",
    "    run_ids.append({\"patientID\":patientID, \"run_id\":pipeline_id})\n",
    "\n",
    "runs_df = pd.DataFrame(run_ids)\n",
    "runs_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check status of each run via the WES server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no run for Patient/576472\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>run_id</th>\n",
       "      <th>status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Patient/576466</td>\n",
       "      <td>4f874352-27fa-4573-9b68-a5394bdf0e0c</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Patient/576465</td>\n",
       "      <td>f9e75d07-9ce1-47bc-b5a7-34d4554e2903</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Patient/576467</td>\n",
       "      <td>27b3d723-c811-4559-9fe2-f926b248e48b</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Patient/576473</td>\n",
       "      <td>1c0cdacb-2b3f-4612-b9d8-c5dce9d60b5a</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Patient/576468</td>\n",
       "      <td>e24917d7-035c-4459-b7da-58110f914e9d</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Patient/576474</td>\n",
       "      <td>aca24b6e-5fa7-4ba2-b7f4-142db68a0991</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Patient/576470</td>\n",
       "      <td>1f64713f-a679-4e29-a049-5400e7f89030</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Patient/576469</td>\n",
       "      <td>87459b70-3e01-4bf2-85a3-e39df8941a53</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Patient/576472</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Patient/576450</td>\n",
       "      <td>60bf9c82-a7f8-4bb5-9482-33c157da3f70</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patientID                                run_id    status\n",
       "0  Patient/576466  4f874352-27fa-4573-9b68-a5394bdf0e0c  COMPLETE\n",
       "1  Patient/576465  f9e75d07-9ce1-47bc-b5a7-34d4554e2903  COMPLETE\n",
       "2  Patient/576467  27b3d723-c811-4559-9fe2-f926b248e48b  COMPLETE\n",
       "3  Patient/576473  1c0cdacb-2b3f-4612-b9d8-c5dce9d60b5a  COMPLETE\n",
       "4  Patient/576468  e24917d7-035c-4459-b7da-58110f914e9d  COMPLETE\n",
       "5  Patient/576474  aca24b6e-5fa7-4ba2-b7f4-142db68a0991  COMPLETE\n",
       "6  Patient/576470  1f64713f-a679-4e29-a049-5400e7f89030  COMPLETE\n",
       "7  Patient/576469  87459b70-3e01-4bf2-85a3-e39df8941a53  COMPLETE\n",
       "8  Patient/576472                                  None       NaN\n",
       "9  Patient/576450  60bf9c82-a7f8-4bb5-9482-33c157da3f70  COMPLETE"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for index, row in runs_df.iterrows():\n",
    "    if row['run_id'] != None:\n",
    "        status = wesClient.getTaskStatus(row['run_id'])\n",
    "        runs_df.at[index, \"status\"] = status\n",
    "    else:\n",
    "        print ('no run for {}'.format(row['patientID']))\n",
    "runs_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define function to download a file and to extract the result from the output file from SAMTools Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import os\n",
    "def download(url, file_path):\n",
    "    '''Download a file from a URL to a local file path'''\n",
    "    with open(os.path.expanduser(file_path), \"wb\") as file:\n",
    "        response = requests.get(url)\n",
    "        file.write(response.content)\n",
    "        \n",
    "def getStats(filePath, statsList):\n",
    "    ''' Extract values from a SAMTools Stats results file'''\n",
    "    ### \n",
    "    \n",
    "    retDict = {}\n",
    "    f = open(filePath, \"r\")\n",
    "    for x in f:\n",
    "        if x.startswith('SN'):   \n",
    "            parts = x.split('\\t')\n",
    "            statName = parts[1].split(':')[0]\n",
    "            if statName in statsList:\n",
    "                retDict[statName] = parts[2].rstrip()\n",
    "    return retDict\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the completed workflows retrieve the results file and extract the required result values. Add the result values to a DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>insert size average</th>\n",
       "      <th>insert size standard deviation</th>\n",
       "      <th>run_id</th>\n",
       "      <th>patientID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>360.3</td>\n",
       "      <td>112.1</td>\n",
       "      <td>4f874352-27fa-4573-9b68-a5394bdf0e0c</td>\n",
       "      <td>Patient/576466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>378.4</td>\n",
       "      <td>116.0</td>\n",
       "      <td>f9e75d07-9ce1-47bc-b5a7-34d4554e2903</td>\n",
       "      <td>Patient/576465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>365.1</td>\n",
       "      <td>131.9</td>\n",
       "      <td>27b3d723-c811-4559-9fe2-f926b248e48b</td>\n",
       "      <td>Patient/576467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>346.4</td>\n",
       "      <td>107.9</td>\n",
       "      <td>1c0cdacb-2b3f-4612-b9d8-c5dce9d60b5a</td>\n",
       "      <td>Patient/576473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>350.7</td>\n",
       "      <td>113.6</td>\n",
       "      <td>e24917d7-035c-4459-b7da-58110f914e9d</td>\n",
       "      <td>Patient/576468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>371.8</td>\n",
       "      <td>98.6</td>\n",
       "      <td>aca24b6e-5fa7-4ba2-b7f4-142db68a0991</td>\n",
       "      <td>Patient/576474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>357.5</td>\n",
       "      <td>100.7</td>\n",
       "      <td>1f64713f-a679-4e29-a049-5400e7f89030</td>\n",
       "      <td>Patient/576470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>338.3</td>\n",
       "      <td>102.0</td>\n",
       "      <td>87459b70-3e01-4bf2-85a3-e39df8941a53</td>\n",
       "      <td>Patient/576469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>291.8</td>\n",
       "      <td>130.0</td>\n",
       "      <td>60bf9c82-a7f8-4bb5-9482-33c157da3f70</td>\n",
       "      <td>Patient/576450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  insert size average insert size standard deviation  \\\n",
       "0               360.3                          112.1   \n",
       "1               378.4                          116.0   \n",
       "2               365.1                          131.9   \n",
       "3               346.4                          107.9   \n",
       "4               350.7                          113.6   \n",
       "5               371.8                           98.6   \n",
       "6               357.5                          100.7   \n",
       "7               338.3                          102.0   \n",
       "8               291.8                          130.0   \n",
       "\n",
       "                                 run_id       patientID  \n",
       "0  4f874352-27fa-4573-9b68-a5394bdf0e0c  Patient/576466  \n",
       "1  f9e75d07-9ce1-47bc-b5a7-34d4554e2903  Patient/576465  \n",
       "2  27b3d723-c811-4559-9fe2-f926b248e48b  Patient/576467  \n",
       "3  1c0cdacb-2b3f-4612-b9d8-c5dce9d60b5a  Patient/576473  \n",
       "4  e24917d7-035c-4459-b7da-58110f914e9d  Patient/576468  \n",
       "5  aca24b6e-5fa7-4ba2-b7f4-142db68a0991  Patient/576474  \n",
       "6  1f64713f-a679-4e29-a049-5400e7f89030  Patient/576470  \n",
       "7  87459b70-3e01-4bf2-85a3-e39df8941a53  Patient/576469  \n",
       "8  60bf9c82-a7f8-4bb5-9482-33c157da3f70  Patient/576450  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from fasp.loc import sbcgcDRSClient\n",
    "statsList = []\n",
    "for index, row in runs_df.iterrows():\n",
    "    if row['run_id'] != None and row['status'] == 'COMPLETE':\n",
    "        log = wesClient.GetRunLog(row['run_id'])\n",
    "        resultsDRSID = log['outputs']['statistics']['path']\n",
    "        #print(\"DRS URI for results for subject {}: {}\".format(row['patientID'] ,resultsDRSID))\n",
    "        url = drsClient.getAccessURL2(resultsDRSID,'s3')\n",
    "        fileName = log['outputs']['statistics']['name']\n",
    "        download(url, fileName)\n",
    "        statsRequired = ['insert size average','insert size standard deviation']\n",
    "        stats = getStats(fileName, statsRequired)\n",
    "        #print(stats)\n",
    "        stats['run_id'] = row['run_id']\n",
    "        stats['patientID'] = row['patientID']\n",
    "        statsList.append(stats)\n",
    "        os.remove(fileName)\n",
    "stats_df =  pd.DataFrame(statsList)\n",
    "stats_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientID</th>\n",
       "      <th>ClinStatus1</th>\n",
       "      <th>Cndtn1_codetext</th>\n",
       "      <th>Cndtn1_code1</th>\n",
       "      <th>Cndtn1_code2</th>\n",
       "      <th>insert size average</th>\n",
       "      <th>insert size standard deviation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Patient/576466</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>360.3</td>\n",
       "      <td>112.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Patient/576465</td>\n",
       "      <td>Alive</td>\n",
       "      <td>ALL</td>\n",
       "      <td>MONDO:0004967</td>\n",
       "      <td>NCIT:C3167</td>\n",
       "      <td>378.4</td>\n",
       "      <td>116.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Patient/576467</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>365.1</td>\n",
       "      <td>131.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Patient/576473</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NHL</td>\n",
       "      <td>MONDO:0018908</td>\n",
       "      <td>NCIT:C3211</td>\n",
       "      <td>346.4</td>\n",
       "      <td>107.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Patient/576468</td>\n",
       "      <td>Alive</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "      <td>350.7</td>\n",
       "      <td>113.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Patient/576474</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkins Lymphoma</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "      <td>371.8</td>\n",
       "      <td>98.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Patient/576470</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Hodgkin's Lymphoma</td>\n",
       "      <td>MONDO:0004620</td>\n",
       "      <td>NCIT:C24182</td>\n",
       "      <td>357.5</td>\n",
       "      <td>100.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Patient/576469</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>338.3</td>\n",
       "      <td>102.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Patient/576472</td>\n",
       "      <td>Alive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Patient/576450</td>\n",
       "      <td>Reported Unknown</td>\n",
       "      <td>HL</td>\n",
       "      <td>MONDO:0004952</td>\n",
       "      <td>NCIT:C9357</td>\n",
       "      <td>291.8</td>\n",
       "      <td>130.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patientID       ClinStatus1     Cndtn1_codetext   Cndtn1_code1  \\\n",
       "0  Patient/576466             Alive                 NaN            NaN   \n",
       "1  Patient/576465             Alive                 ALL  MONDO:0004967   \n",
       "2  Patient/576467             Alive                 NaN            NaN   \n",
       "3  Patient/576473             Alive                 NHL  MONDO:0018908   \n",
       "4  Patient/576468             Alive                  HL  MONDO:0004952   \n",
       "5  Patient/576474             Alive   Hodgkins Lymphoma  MONDO:0004952   \n",
       "6  Patient/576470             Alive  Hodgkin's Lymphoma  MONDO:0004620   \n",
       "7  Patient/576469             Alive                 NaN            NaN   \n",
       "8  Patient/576472             Alive                 NaN            NaN   \n",
       "9  Patient/576450  Reported Unknown                  HL  MONDO:0004952   \n",
       "\n",
       "  Cndtn1_code2 insert size average insert size standard deviation  \n",
       "0          NaN               360.3                          112.1  \n",
       "1   NCIT:C3167               378.4                          116.0  \n",
       "2          NaN               365.1                          131.9  \n",
       "3   NCIT:C3211               346.4                          107.9  \n",
       "4   NCIT:C9357               350.7                          113.6  \n",
       "5   NCIT:C9357               371.8                           98.6  \n",
       "6  NCIT:C24182               357.5                          100.7  \n",
       "7          NaN               338.3                          102.0  \n",
       "8          NaN                 NaN                            NaN  \n",
       "9   NCIT:C9357               291.8                          130.0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = pd.merge(\n",
    "    df,\n",
    "    stats_df,\n",
    "    how=\"outer\",\n",
    "    left_on='patientID',\n",
    "    right_on='patientID'\n",
    ")\n",
    "final_df.drop('run_id', axis=1, inplace=True)\n",
    "final_df.drop('ClinStatus2', axis=1, inplace=True)\n",
    "\n",
    "final_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}