(E) Process Replicate and Longitudinal Samples – Collect SNV Calls with Differing Alt AF between Sample Pairs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### This notebook tests the pipeline for calling differing Base Calls (Single Nucleotide Variant) between two samples from a pair of samples (*Longitudinal* or *Replicate* pair). This notebook also submits the jobs to run this variant extraction pipeline for each sample pair in our study."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "############################################################################################################################################################################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the script"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "############################################################################################################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import vcf\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from itertools import compress\n",
    "import ast\n",
    "import itertools\n",
    "import time\n",
    "import sys\n",
    "import pickle\n",
    "import shutil\n",
    "\n",
    "import Bio\n",
    "from Bio.Alphabet import IUPAC\n",
    "from Bio.Blast.Applications import NcbiblastnCommandline\n",
    "from Bio.Blast import NCBIXML\n",
    "from Bio.Seq import Seq\n",
    "from Bio.SeqRecord import SeqRecord\n",
    "from Bio.SeqFeature import SeqFeature, FeatureLocation\n",
    "from Bio import pairwise2\n",
    "from Bio import SeqIO\n",
    "from Bio.Graphics import GenomeDiagram\n",
    "from Bio.SeqUtils import GC\n",
    "from Bio.Align.Applications import MuscleCommandline\n",
    "from StringIO import StringIO\n",
    "from Bio import AlignIO\n",
    "from Bio.Align import AlignInfo\n",
    "from Bio.Seq import MutableSeq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### *Functions* to extract Base Calls (that have different alterante allele frequencies) from a pair of VCF files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Get_Base_Calls_from_VCF(parent_output_dir, tag):\n",
    "\n",
    "    '''\n",
    "    This function takes as input, the isolate tag, then extracts SINGLE base calls from\n",
    "    the VCF file generated by Pilon. The function returns a DataFrame for a single isolate\n",
    "    that contains [1] PASS calls that support the reference allele, [2] PASS calls that support\n",
    "    an alternate allele and [3] AMB calls that support both the reference and alternate alleles.\n",
    "    Note: Many base calls won't have an Alternate Allele.\n",
    "    '''\n",
    "\n",
    "    #directory where VCF file for sequenced isolated is stored & filename\n",
    "    VCF_file = parent_output_dir + tag + '/pilon/' + tag + '.vcf'\n",
    "\n",
    "    vcf_reader = vcf.Reader(open(VCF_file , 'r'))\n",
    "\n",
    "    #position-base dictionaries\n",
    "    base_order = {0:'A' , 1:'C' , 2:'G', 3:'T'} #order of output in Pilon\n",
    "    base_order_r = {'A':0 , 'C':1 , 'G':2 , 'T':3}\n",
    "\n",
    "    #create dictionaries to store information for each call\n",
    "    ref_bases = {}\n",
    "    qry_bases = {}\n",
    "    ref_positions = {}\n",
    "    quality_scores = {}\n",
    "    SNP_type = {}\n",
    "    PASS_filter = {}\n",
    "    INFO_for_call = {}\n",
    "    alt_allele_freq = {}\n",
    "    valid_depth = {}\n",
    "\n",
    "    #store the DEPTH of all base & variant calls (depth not stored for major structural variants)\n",
    "    coverage_for_all_base_and_small_indel_calls = []\n",
    "\n",
    "    #indexer for dataframe containing SNPs\n",
    "    index = 0\n",
    "\n",
    "    #iterate through each record from VCF file\n",
    "    for record in vcf_reader:\n",
    "\n",
    "        ######## store the DEPTH of all base & variant calls (depth not stored for major structural variants) ########\n",
    "        try:\n",
    "            coverage_for_all_base_and_small_indel_calls.append( record.INFO['DP'] )\n",
    "\n",
    "        except KeyError: #major structural variant with no depth value\n",
    "            pass \n",
    "\n",
    "        ######## Classify SNP-type according to Pilon Flags ########\n",
    "\n",
    "        ### Filtering Criteria\n",
    "        #---> Check that Filter is either PASS ([]) or AMBIGUOUS (['Amb']) , no 'DEL' or 'LOWCOV' flags from Pilon\n",
    "\n",
    "        ### Call supporting REFERENCE\n",
    "        #if there's no alternate allele and Filter is PASS; PASS call supporting Ref allele\n",
    "        if (len(record.ALT) == 1) and (record.ALT == [None]) and (record.FILTER == []):\n",
    "\n",
    "            alt_allele = 'Z' #no alternate allele supported by reads (might change through filtering process further down if small fraction of reads support an alternate)\n",
    "            ref_allele = record.REF #allele on H37Rv\n",
    "            SNP_type_for_Call = 'Ref_PASS'\n",
    "\n",
    "        ### Call supporting ALTERNATE \n",
    "        #if there's 1 alternate allele, and Filter is PASS; PASS call supporting Alt allele\n",
    "        elif (len(record.ALT) == 1) and (record.ALT != [None]) and (record.FILTER == []):\n",
    "\n",
    "            alt_allele = str( record.ALT[0] ) #alternate allele supported by reads\n",
    "            ref_allele = record.REF #allele on H37Rv\n",
    "            SNP_type_for_Call = 'Alt_PASS'\n",
    "\n",
    "        ### Call ambiguous between REFERNENCE and ALTERNATE\n",
    "        #if there's 1 alternate allele, and Filter is AMB; AMB call supporting supporting neither Alt nor Ref\n",
    "        elif (len(record.ALT) == 1) and (record.ALT != [None]) and (record.FILTER == ['Amb']):\n",
    "\n",
    "            alt_allele = str( record.ALT[0] ) #alternate allele supported by reads\n",
    "            ref_allele = record.REF #allele on H37Rv\n",
    "            SNP_type_for_Call = 'Amb'\n",
    "\n",
    "        ### Bad Quality Call, don't store information for this Base Call, move onto next\n",
    "        else:\n",
    "\n",
    "            continue\n",
    "\n",
    "        ### Filtering Criteria (for Reference Confirmation or SNP)\n",
    "        #---> Make sure that the length of all any alleles is 1 (No Variant or 1 Base) ; if variant, then variant is a SNP and not an INDEL\n",
    "        if (len(ref_allele) == 1) and (len(alt_allele) == 1):\n",
    "\n",
    "            ######## Retrieve Relevant information for filtering quality of Base Call ########\n",
    "\n",
    "            # Mean Base Quality @ locus\n",
    "            BQ = record.INFO['BQ']\n",
    "\n",
    "            # Mean Mapping Quality @ locus\n",
    "            MQ = record.INFO['MQ']\n",
    "\n",
    "            # Number of Reads w/ Deletion \n",
    "            DC = record.INFO['DC']\n",
    "\n",
    "            # Number of Reads w/ Insertion\n",
    "            IC = record.INFO['IC']  \n",
    "            \n",
    "            # Depth of Valid Reads in Pileup\n",
    "            VD = record.INFO['DP']\n",
    "\n",
    "            ### Filtering Criteria\n",
    "            #---> Mean Base Quality > 20\n",
    "            #---> Mean Mapping Quality > 30\n",
    "            #---> No Reads Supporting Insertions\n",
    "            #---> No Reads Supporting Deletions\n",
    "            #---> Number of High Quality Reads >= 25\n",
    "            if (BQ > 20) and (MQ > 30) and (DC == 0) and (IC == 0) and (VD >= 25):\n",
    "\n",
    "                # Quality-Percentage - Percentage of As, Cs, Gs, Ts weighted by Q & MQ at locus\n",
    "                QP = record.INFO['QP']\n",
    "\n",
    "                ### Filtering Criteria\n",
    "                #---> Make sure that Reads support at most 2 alleles (either confirm the Reference or provide evidence for an alternate allele)\n",
    "                #-----> If there are 2 alleles (make sure 1 of them is the Reference Allele)\n",
    "\n",
    "                if (sum(np.array(QP) == 0) == 3): #1 Allele supported by Reads (either Reference or an Alternate), store information for this Base Call (3 bases have NO reads supporting)\n",
    "\n",
    "                    #create a Boolean list for the Bases that have at least 1% of support from reads\n",
    "                    supported_bases_boolean = list( np.array( QP ) > 0 )\n",
    "\n",
    "                    #positions of Bases [A,C,G,T] with at least 1% of support from reads\n",
    "                    supported_bases_i = [i for i,x in enumerate(supported_bases_boolean) if x == True] #returns list of length \n",
    "                    supported_base = base_order[supported_bases_i[0]]\n",
    "\n",
    "                    #if the Reference Allele is supported\n",
    "                    if supported_base == ref_allele:\n",
    "                        alt_allele_frequency = 0.0 #no alternate allele\n",
    "\n",
    "                    #if an Alternate Allele is supported:\n",
    "                    if supported_base != ref_allele:\n",
    "                        alt_allele_frequency = 1.0 #alternate allele\n",
    "\n",
    "                elif (sum(np.array(QP) == 0) == 2): #2 Alleles supported, make sure 1 is the Reference & confirm the Alternate (2 bases have NO reads supporting)\n",
    "\n",
    "                    #position of the Reference Allele in [A,C,G,T]\n",
    "                    ref_allele_base_i = base_order_r[ref_allele] \n",
    "\n",
    "                    #create a Boolean list for the Bases that have at least 1% of support from reads\n",
    "                    supported_bases_boolean = list( np.array( QP ) > 0 )\n",
    "\n",
    "                    #positions of Bases [A,C,G,T] with at least 1% of support from reads\n",
    "                    supported_bases_i = [i for i,x in enumerate(supported_bases_boolean) if x == True] #returns list of length 2\n",
    "\n",
    "                    #check that Reference Allele is supported by at least 1% of the reads\n",
    "                    if ref_allele_base_i in supported_bases_i:\n",
    "\n",
    "                        #find the Alternate Allele that was supported by at least 1% of the reads\n",
    "                        for supported_base_i in supported_bases_i:\n",
    "\n",
    "                            if supported_base_i != ref_allele_base_i: #then must be alternate allele\n",
    "\n",
    "                                alt_allele = base_order[supported_base_i] #Alternate Allele\n",
    "                                alt_allele_frequency = float(QP[supported_base_i]) / 100.0 #weighted percentage of reads supporting Alternate Allele\n",
    "                                break\n",
    "                    \n",
    "                    #Reference Allele is NOT supported by at least 1% of the reads (2 alleles supported but neither is the reference), don't store information for this Base Call\n",
    "                    else:\n",
    "                        continue\n",
    "\n",
    "                #more than 2 alleles supported by reads, don't store information for this Base Call\n",
    "                else: \n",
    "                    continue \n",
    "\n",
    "                #After extensive filtering and categorization, store all of the pertinent information about the Base Call\n",
    "                ref_bases[index] = ref_allele\n",
    "                qry_bases[index] = alt_allele\n",
    "                ref_positions[index] = record.POS\n",
    "                PASS_filter[index] = record.FILTER\n",
    "                quality_scores[index] = record.QUAL\n",
    "                SNP_type[index] = SNP_type_for_Call\n",
    "                INFO_for_call[index] = record.INFO\n",
    "                alt_allele_freq[index] = alt_allele_frequency\n",
    "                valid_depth[index] = record.INFO['DP']\n",
    "\n",
    "                index += 1\n",
    "\n",
    "    #convert dictionaries to series\n",
    "    ref_bases = pd.Series(ref_bases)\n",
    "    qry_bases = pd.Series(qry_bases)\n",
    "    ref_positions = pd.Series(ref_positions)\n",
    "    quality_scores = pd.Series(quality_scores)\n",
    "    SNP_type = pd.Series(SNP_type)\n",
    "    PASS_filter = pd.Series(PASS_filter)\n",
    "    INFO_for_call = pd.Series(INFO_for_call)\n",
    "    alt_allele_freq = pd.Series(alt_allele_freq)\n",
    "    valid_depth = pd.Series(valid_depth)\n",
    "    tag_series = pd.Series([tag]*len(ref_bases.index) , index = ref_bases.index) #psuedo-series to store the tag id of the isolate\n",
    "\n",
    "    #create DataFrame to hold all base calls for a given sample\n",
    "    Variant_Call_DF = pd.DataFrame()\n",
    "    Variant_Call_DF['ref_base'] = ref_bases\n",
    "    Variant_Call_DF['alt_base'] = qry_bases\n",
    "    Variant_Call_DF['ref_position'] = ref_positions\n",
    "    Variant_Call_DF['quality'] = quality_scores\n",
    "    Variant_Call_DF['SNP_type'] = SNP_type\n",
    "    Variant_Call_DF['PASS_filter'] = PASS_filter\n",
    "    Variant_Call_DF['INFO'] = INFO_for_call\n",
    "    Variant_Call_DF['alt_AF'] = alt_allele_freq\n",
    "    Variant_Call_DF['depth'] = valid_depth\n",
    "    Variant_Call_DF['tag'] = tag_series\n",
    "\n",
    "    ### Filtering Criteria (high coverage)\n",
    "    #---> Filter out Base Calls with total # of reads aligning above the 99th percentile of covered positions in sample\n",
    "    ninety_ninth_percentile_coverage = np.percentile( np.array( coverage_for_all_base_and_small_indel_calls ) , 99 )\n",
    "    Variant_Call_DF = Variant_Call_DF[Variant_Call_DF.depth < ninety_ninth_percentile_coverage] #keep base calls that dont have unusually high coverage\n",
    "\n",
    "    #reset index after dropping base calls with unusually high coverage\n",
    "    Variant_Call_DF.reset_index(inplace = True , drop = True)\n",
    "\n",
    "    #save REFERENCE POSITIONS only for filtered (good) single base call variants as a pickle in same directory as VCF file\n",
    "    Variant_Call_DF.loc[: , 'ref_position'].to_pickle(parent_output_dir + tag + '/' + tag + '_ref_positions_for_filtered_base_calls.pkl')\n",
    "\n",
    "    ##SIDE NOTE: use \"pd.read_pickle(parent_output_dir + tag + '/' + tag + '_filtered_base_calls.pkl')\" to unpickle series object\n",
    "\n",
    "    return Variant_Call_DF #DataFrame for base calls for a single isolate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Get_different_Base_Calls_between_isolates(parent_output_dir, tag_list):\n",
    "    \n",
    "    '''\n",
    "    This function uses the previously defined function to filter the VCF files corresponding to multiple\n",
    "    isolates from the same isolate pair. The input is a list of tags (fastq ids) for isolates obtained for a given isolate pair\n",
    "    Once the VCF files have been seperately filtered for good calls (PASS supporting Reference Allele or PASS supporting \n",
    "    Alternate Allele or ALT supporting two Alleles; and no unusually high coverage) this function then compares the \n",
    "    single base calls between the multiple isolates. The output is a DataFrame that contains GOOD base calls\n",
    "    for both isolates but for which the Alternate Allee Frequencies DIFFERED.\n",
    "    '''\n",
    "\n",
    "    #store VCF output for both isolates, key: tag , value: DF for variant output\n",
    "    Variant_Call_DFs = {}\n",
    "\n",
    "    #iterate through both isolates that came from subject\n",
    "    for tag in tag_list:\n",
    "\n",
    "        #call function to get variants for isolate and store in dict\n",
    "        Variant_Call_DFs[tag] = Get_Base_Calls_from_VCF(parent_output_dir, tag)\n",
    "\n",
    "    #concatenate DataFrames for each isolate into 1 DataFrame\n",
    "    Variant_Call_Master_DF = pd.concat(Variant_Call_DFs.values())\n",
    "\n",
    "    #reset index of master dataframe\n",
    "    Variant_Call_Master_DF.reset_index(inplace = True, drop = True)\n",
    "\n",
    "    #sort values in order of reference position\n",
    "    Variant_Call_Master_DF.sort_values(by = 'ref_position' , ascending = True , inplace = True)\n",
    "\n",
    "    ############################################################################################\n",
    "\n",
    "    #create a list to store all of the indices that should be dropped\n",
    "    base_calls_to_drop = [] #indices in dataframe to drop later\n",
    "\n",
    "    #### FILTER methodology ####\n",
    "    ## if a call for a reference position is PASS-REFERENCE with alt AF of 0.0 for both isolates, then drop base call (for each isolate) (all reads from both sequenced isolates support the same allele)\n",
    "    PASS_REF_filter = [(Ref_PASS and zero_alt_AF) for Ref_PASS , zero_alt_AF in  zip( list( Variant_Call_Master_DF.SNP_type == 'Ref_PASS' ) , list( Variant_Call_Master_DF.alt_AF == 0.0 ) ) ]\n",
    "\n",
    "    ## if a call for a reference position is PASS-ALTERNATE with alt AF of 1.0 for both isolates, then drop base call (for each isolate) (all reads from both sequenced isolates support the same allele)\n",
    "    PASS_ALT_filter = [(Alt_PASS and one_alt_AF) for Alt_PASS , one_alt_AF in  zip( list( Variant_Call_Master_DF.SNP_type == 'Alt_PASS' ) , list( Variant_Call_Master_DF.alt_AF == 1.0 ) ) ]\n",
    "\n",
    "    #subset to base calls that support criteria for each pair matching criteria\n",
    "    PASS_REF_base_calls = Variant_Call_Master_DF[PASS_REF_filter]\n",
    "    PASS_ALT_base_calls = Variant_Call_Master_DF[PASS_ALT_filter]\n",
    "\n",
    "    #find matching base calls that support Reference allele entirely\n",
    "    matching_PASS_REF_base_calls_filter = PASS_REF_base_calls.duplicated(subset = ['ref_base' , 'alt_base' , 'ref_position' , 'SNP_type' , 'alt_AF'] , keep = False)\n",
    "    matching_base_call_indices_to_drop = list( PASS_REF_base_calls[matching_PASS_REF_base_calls_filter].index )\n",
    "    base_calls_to_drop = base_calls_to_drop + matching_base_call_indices_to_drop\n",
    "\n",
    "    #find matching base calls that support Alternate allele entirely\n",
    "    matching_PASS_ALT_base_calls_filter = PASS_ALT_base_calls.duplicated(subset = ['ref_base' , 'alt_base' , 'ref_position' , 'SNP_type' , 'alt_AF'] , keep = False)\n",
    "    matching_base_call_indices_to_drop = list( PASS_ALT_base_calls[matching_PASS_ALT_base_calls_filter].index )\n",
    "    base_calls_to_drop = base_calls_to_drop + matching_base_call_indices_to_drop\n",
    "\n",
    "    #drop base calls from Master variant DF that match between isolates AND support 1 allele completely\n",
    "    Variant_Call_Master_DF.drop(labels = base_calls_to_drop , axis = 0 , inplace = True)\n",
    "\n",
    "    ############################################################################################\n",
    "\n",
    "    #### FILTER methodology ####\n",
    "    ## from the remaining base calls, find base calls for which we have a reference position for BOTH isolates (calls in both samples were good quality)\n",
    "    ## that is, we will have a PASS call or AMB call (w/o low coverage or deletion flags) for a reference position for BOTH isolates\n",
    "    Variant_Call_Master_DF = Variant_Call_Master_DF[Variant_Call_Master_DF.duplicated(subset = 'ref_position' , keep = False)]\n",
    "\n",
    "    #reset index of filtered SNP-variant Master DataFrame\n",
    "    Variant_Call_Master_DF.reset_index(inplace = True, drop = True)\n",
    "    \n",
    "    ############################################################################################\n",
    "    \n",
    "    #### FILTER methodology ####\n",
    "    ## filter out paired Base Calls that have a difference in Alternate Allele Frequency of less than 5%\n",
    "    \n",
    "    alt_AF_isolate_A = Variant_Call_Master_DF.loc[range(0 , np.shape(Variant_Call_Master_DF)[0] , 2) , 'alt_AF']\n",
    "    alt_AF_isolate_B = Variant_Call_Master_DF.loc[range(1 , np.shape(Variant_Call_Master_DF)[0] , 2) , 'alt_AF']\n",
    "\n",
    "    alt_AF_diff_btwn_paired_isolates = abs(alt_AF_isolate_A.values - alt_AF_isolate_B.values)\n",
    "\n",
    "    isolate_A_Base_Call_indices_small_change_alt_AF = list(alt_AF_isolate_A[alt_AF_diff_btwn_paired_isolates < 0.05].index)\n",
    "    isolate_B_Base_Call_indices_small_change_alt_AF = list(alt_AF_isolate_B[alt_AF_diff_btwn_paired_isolates < 0.05].index)\n",
    "\n",
    "    Variant_Call_Indices_SMALL_Alt_AF_Diff = isolate_A_Base_Call_indices_small_change_alt_AF + isolate_B_Base_Call_indices_small_change_alt_AF\n",
    "\n",
    "    #drop paired Base Calls w/ corresponding change in Alterante Allele Frequency < 5%\n",
    "    Variant_Call_Master_DF.drop(Variant_Call_Indices_SMALL_Alt_AF_Diff , axis = 0 , inplace = True)\n",
    "    \n",
    "    #reset index of final filtered SNP-variant Master DataFrame\n",
    "    Variant_Call_Master_DF.reset_index(inplace = True, drop = True)\n",
    "    \n",
    "    return Variant_Call_Master_DF #DataFrame for good base calls that differed between isolates from same subject or replicate pair"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Call Functions to call SNPs between a (*longitudinal* or *replicate*) pair of isolates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### *For Testing* (longitudinal pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INPUT: each sub-directory has the pilon-outputed VCF file\n",
    "parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/JankyPipe/output/'\n",
    "\n",
    "#INPUT: Sample annotation file path for longitudinal or replicate samples (already filtered with Kraken and F2)\n",
    "sample_annotation_file_path = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/CSV_files/sample_annotation_files/Longitudinal_fastq_path_names_and_JankyPipe_tags_filtered_with_Kraken_and_F2.csv'\n",
    "\n",
    "#INPUT: either subject ID (longitudinal) or isolate pair ID (replicate)\n",
    "isolate_pair_ID = '1960'\n",
    "\n",
    "#INPUT: parent directory where all SNPs\n",
    "SNP_call_parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/pickled_files/variant_calling/longitudinal_SNPs/all_SNPs_between_longitudinal_pairs/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### *For Testing* (replicate pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INPUT: each sub-directory has the pilon-outputed VCF file\n",
    "parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/JankyPipe/output_REPLICATES/'\n",
    "\n",
    "#INPUT: Sample annotation file path for longitudinal or replicate samples (already filtered with Kraken and F2)\n",
    "sample_annotation_file_path = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/CSV_files/sample_annotation_files/Replicate_fastq_path_names_and_JankyPipe_tags_filtered_with_Kraken_and_F2.csv'\n",
    "\n",
    "#INPUT: either subject ID (longitudinal) or isolate pair ID (replicate)\n",
    "isolate_pair_ID = 'replicate_pair_67'\n",
    "\n",
    "#INPUT: parent directory where all SNPs\n",
    "SNP_call_parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/pickled_files/variant_calling/replicate_SNPs/all_SNPs_between_replicate_pairs/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### *For Script*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INPUT: each sub-directory has the pilon-outputed VCF file\n",
    "parent_output_dir = sys.argv[1]\n",
    "\n",
    "#INPUT: Sample annotation file path for longitudinal or replicate samples (already filtered with Kraken and F2)\n",
    "sample_annotation_file_path = sys.argv[2]\n",
    "\n",
    "#INPUT: either subject ID (longitudinal) or isolate pair ID (replicate)\n",
    "isolate_pair_ID = sys.argv[3]\n",
    "\n",
    "#INPUT: parent directory where all SNPs \n",
    "SNP_call_parent_output_dir = sys.argv[4] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sample Annotation & Tag Information for Isolate Pair\n",
    "##############################################################################################################################\n",
    "\n",
    "#import annotation file for all isolates and corresponding tags\n",
    "sample_annotation = pd.read_csv(sample_annotation_file_path , sep = ',').set_index('patient_id')\n",
    "\n",
    "#subset to both of the isolates taken from the isolate pair\n",
    "sequenced_isolates_annot_for_sample_pair = sample_annotation.loc[isolate_pair_ID , :] #annotation (only) for both isolates from the sample pair\n",
    "\n",
    "#get list of tags for both longitudinal isolates from isolate pair\n",
    "tags = []\n",
    "\n",
    "#iterate across both isolates\n",
    "for isolate_i in [0 , 1]:\n",
    "    \n",
    "    #subset to 1 of the isolates taken from the isolate pair\n",
    "    sequenced_isolate_annot = sequenced_isolates_annot_for_sample_pair.iloc[isolate_i , :] #annotation for 1 of the isolates from isolate pair\n",
    "    \n",
    "    #get relevant information for specific isolate\n",
    "    population = sequenced_isolate_annot.population\n",
    "    isolate_pair_ID = sequenced_isolate_annot.name\n",
    "    tag = sequenced_isolate_annot.tag\n",
    "    \n",
    "    #append tag to tags list to get Variants From and compare other other isolates in tag list\n",
    "    tags.append(tag)\n",
    "    \n",
    "#make necessary directory for isolate pair (to store variation between longitudinal isolates)\n",
    "SNP_call_output_dir = SNP_call_parent_output_dir + population + '_' + isolate_pair_ID\n",
    "    \n",
    "if os.path.exists(SNP_call_output_dir):\n",
    "    shutil.rmtree(SNP_call_output_dir)\n",
    "    os.makedirs(SNP_call_output_dir)\n",
    "elif not os.path.exists(SNP_call_output_dir):\n",
    "    os.makedirs(SNP_call_output_dir)\n",
    "    \n",
    "#BASE CALLS\n",
    "##############################################################################################################################\n",
    "    \n",
    "#get 'good' calls that differ between isolates for the same isolate pair (input is the tags corresponding to the isolates from isolate pair)\n",
    "Base_Call_Master_DF = Get_different_Base_Calls_between_isolates(parent_output_dir, tags)\n",
    "\n",
    "#append psuedo-columns to note population & isolate pair ID\n",
    "population_series = pd.Series( [population] * np.shape(Base_Call_Master_DF)[0] , index = Base_Call_Master_DF.index )\n",
    "isolate_pair_ID_series = pd.Series( [isolate_pair_ID] * np.shape(Base_Call_Master_DF)[0] , index = Base_Call_Master_DF.index )\n",
    "\n",
    "Base_Call_Master_DF['population'] = population_series\n",
    "Base_Call_Master_DF['patient_id'] = isolate_pair_ID_series\n",
    "\n",
    "##pickle CSV file containing base calls that differ between paired isolates\n",
    "Base_Call_Master_DF.to_pickle(SNP_call_output_dir + '/' + 'base_calls_different_between_isolates.pkl') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "############################################################################################################################################################################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Submit Jobs - Longitudinal Isolates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "############################################################################################################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from slurmpy import Slurm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import annotation file for all sample pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#annotation file for all isolates and corresponding tags\n",
    "sample_annotation = pd.read_csv('/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/CSV_files/sample_annotation_files/Longitudinal_fastq_path_names_and_JankyPipe_tags_filtered_with_Kraken_and_F2.csv' , sep = ',').set_index('patient_id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find which subjects have 2 valid VCF files for both longitudinal isolates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "successful_JankyPipe_run = []\n",
    "\n",
    "for isolate_i in range(0 , np.shape(sample_annotation)[0]):\n",
    "\n",
    "    #get the tag ID for the fastq files (same as ID for fastq files)\n",
    "    tag = sample_annotation.tag[isolate_i]\n",
    "\n",
    "    #where pilon VCF and lineage information will be stored [LAB FOLDER]\n",
    "    output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/JankyPipe/output/' + tag\n",
    "    \n",
    "    #check to see 'Lineage Call' folder exists in the output directory (last thing that is run in JankyPipe)\n",
    "    if os.path.exists(output_dir + '/fast-lineage-caller/'):\n",
    "        \n",
    "        successful_JankyPipe_run.append('yes')\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        successful_JankyPipe_run.append('no')\n",
    "        \n",
    "sample_annotation['successful_JankyPipe_run'] = successful_JankyPipe_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fastq_files</th>\n",
       "      <th>population</th>\n",
       "      <th>run_ID</th>\n",
       "      <th>sample_ID</th>\n",
       "      <th>sample_order</th>\n",
       "      <th>tag</th>\n",
       "      <th>isolate_type</th>\n",
       "      <th>successful_JankyPipe_run</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>patient_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [fastq_files, population, run_ID, sample_ID, sample_order, tag, isolate_type, successful_JankyPipe_run]\n",
       "Index: []"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_annotation[sample_annotation.successful_JankyPipe_run == 'no']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#subjects for which we do not have 2 good VCF files for\n",
    "bad_isolate_pair_IDs = set( list( sample_annotation[sample_annotation.successful_JankyPipe_run == 'no'].index ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "print list(bad_isolate_pair_IDs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get list of subjects (longitudinal isolate pairs) to get SNP variants from"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "isolate_pair_ID_list = list( set( list( sample_annotation.index ) ) - bad_isolate_pair_IDs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['P000183', '1960', '2718', '2491', '2968', '2020E', '5017Y', '4146V', '24', 'KPS_9', '2321', '27', '3157Q', 'P000089', '23', 'P233', 'KPS_1', 'KPS_2', 'KPS_3', 'KPS_4', 'P12', 'KPS_7', 'I0000024-5', '1678', 'KPS_39', 'KPS_34', 'KPS_35', 'KPS_36', 'KPS_37', 'KPS_30', 'KPS_31', '1672', '1673', '1700', 'P000027', 'KPS_58', 'KPS_59', 'KPS_56', 'KPS_57', 'KPS_52', 'KPS_53', 'KPS_50', '1620', '4137X', 'P000155', 'P000156', '2556', '4154E', '2319', 'P000094', 'P000095', '1827', '2492', 'P000226', 'P03', 'P02', 'KPS_29', 'KPS_28', 'P07', 'P06', 'P04', 'KPS_23', 'KPS_22', '3129J', 'P08', 'KPS_27', 'KPS_26', '3061R', '3644X', 'P043', 'KPS_49', 'KPS_48', 'I0003165-3', 'KPS_46', 'KPS_41', 'I0000180-5', '3149E', '3047X', '4085U', '3117W', '3673', 'KPS_66', '3100U', 'P052', '1719', 'P01', '2307', 'P10', 'P11', '3', '2043R', '7', 'P000026', '3018C', 'P000227', '1657', 'P000225', 'KPS_62', 'KPS_79', '4084F', 'KPS_21', '4161B', 'KPS_70', 'KPS_71', 'KPS_72', 'P000259', 'KPS_74', '1948', 'KPS_77', 'P062', '20', '2523', 'P09', 'I0003758-5', 'I0002615-8', '4078X', '1664', '2050N', '2028K', '32', '30', '1846', '34', '3096', '2027W', 'P000035', '4063W', 'P000030', 'KPS_13', '1941', 'I0000068-2', 'P377', 'P372', 'KPS_5', 'I0004240-3', 'P000320', '3142P', 'P355', '1682', 'P179', '3451', '4241M', 'I0004746-9', 'I0003398-0', 'I0002457-5', 'P000048', '2688', '6', 'KPS_38', 'P000128', '2041Q', '4023C', '4068M', 'B', 'P367', '3147D', '2741', '2776', '8', 'I0005235-2', '3725', 'I0004236-1', 'KPS_24', 'I0002960-8', 'P108', '11', '10', '13', '12', '15', '14', '16', 'I0004172-8', 'P160', 'I0005229-5', 'P164', 'KPS_67', 'P000059', '2026', 'KPS_64', 'I0005973-8', 'I0001112-7', 'KPS_60', 'I0004013-4', '2511', 'P000056', 'KPS_68', '2731', '2436', '3103J', 'P318', '4024R', '4144U', 'KPS_81', 'KPS_80', 'P316', '2049', 'I0004304-7', 'P000189', 'I0000221-7', 'KPS_75', 'KPS_82', '3058W', 'P288', '3430', 'KPS_16', 'P251', 'KPS_84', 'P000267', '2918', 'KPS_10', 'KPS_11', 'P000263', 'KPS_17', 'KPS_14', 'KPS_15', 'I0004003-5', 'KPS_18', '2047T', '28', 'D', '3143B', '1687', '3439', 'I0001031-9', 'I0001560-7', 'I0003710-6', 'P305', '4', '26', 'P000071', 'P000076', 'KPS_73', 'P000075', 'P221', 'I0001162-2', '1972', 'P227', 'P000276', '1', 'P000271', 'P000272', 'P000273', 'P000176']\n"
     ]
    }
   ],
   "source": [
    "print isolate_pair_ID_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "244"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(isolate_pair_ID_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Submit a job for each subject to get the SNPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663382\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000183 : 2663382\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663383\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1960 : 2663383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663384\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2718 : 2663384\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663385\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2491 : 2663385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663386\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2968 : 2663386\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663387\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020E : 2663387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663388\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5017Y : 2663388\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663389\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4146V : 2663389\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663390\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24 : 2663390\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663391\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_9 : 2663391\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663392\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2321 : 2663392\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663393\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27 : 2663393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663394\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3157Q : 2663394\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663395\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000089 : 2663395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663396\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23 : 2663396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663397\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P233 : 2663397\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663398\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_1 : 2663398\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663399\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_2 : 2663399\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_3 : 2663400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663401\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_4 : 2663401\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P12 : 2663402\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663403\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_7 : 2663403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663404\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0000024-5 : 2663404\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663405\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1678 : 2663405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663406\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_39 : 2663406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663407\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_34 : 2663407\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663408\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_35 : 2663408\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663409\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_36 : 2663409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663410\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_37 : 2663410\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663411\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_30 : 2663411\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663412\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_31 : 2663412\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663413\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1672 : 2663413\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663414\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1673 : 2663414\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663415\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1700 : 2663415\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663416\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000027 : 2663416\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663418\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_58 : 2663418\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663419\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_59 : 2663419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663420\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_56 : 2663420\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663421\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_57 : 2663421\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663422\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_52 : 2663422\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663423\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_53 : 2663423\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663424\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_50 : 2663424\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663425\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1620 : 2663425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663426\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4137X : 2663426\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663427\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000155 : 2663427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663428\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000156 : 2663428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663429\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2556 : 2663429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663430\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4154E : 2663430\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663431\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2319 : 2663431\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663432\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000094 : 2663432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663433\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000095 : 2663433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663434\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1827 : 2663434\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663435\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2492 : 2663435\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663436\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000226 : 2663436\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663437\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P03 : 2663437\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663438\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P02 : 2663438\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663439\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_29 : 2663439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663440\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_28 : 2663440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663441\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P07 : 2663441\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663442\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P06 : 2663442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663443\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P04 : 2663443\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663444\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_23 : 2663444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663445\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_22 : 2663445\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663446\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3129J : 2663446\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663447\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P08 : 2663447\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663448\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_27 : 2663448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663449\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_26 : 2663449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663450\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3061R : 2663450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663451\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3644X : 2663451\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663452\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P043 : 2663452\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663465\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_49 : 2663465\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663466\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_48 : 2663466\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663469\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0003165-3 : 2663469\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663470\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_46 : 2663470\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663471\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_41 : 2663471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663472\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0000180-5 : 2663472\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663473\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3149E : 2663473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663474\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3047X : 2663474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663475\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4085U : 2663475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663476\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3117W : 2663476\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663477\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3673 : 2663477\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663478\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_66 : 2663478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663481\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3100U : 2663481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663482\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P052 : 2663482\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663483\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1719 : 2663483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663484\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P01 : 2663484\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663485\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2307 : 2663485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663486\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P10 : 2663486\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663487\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P11 : 2663487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663488\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 : 2663488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663489\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2043R : 2663489\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663490\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 : 2663490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663491\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000026 : 2663491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663492\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3018C : 2663492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663493\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000227 : 2663493\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663494\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1657 : 2663494\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663495\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000225 : 2663495\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_62 : 2663496\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663497\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_79 : 2663497\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663498\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4084F : 2663498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663499\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_21 : 2663499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4161B : 2663500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663501\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_70 : 2663501\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663502\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_71 : 2663502\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663503\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_72 : 2663503\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663504\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000259 : 2663504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663505\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_74 : 2663505\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663506\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1948 : 2663506\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663507\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_77 : 2663507\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663508\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P062 : 2663508\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663509\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 : 2663509\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663510\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2523 : 2663510\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663511\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P09 : 2663511\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663512\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0003758-5 : 2663512\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663513\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0002615-8 : 2663513\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663514\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4078X : 2663514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1664 : 2663515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663516\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2050N : 2663516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663517\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2028K : 2663517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663518\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32 : 2663518\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663519\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30 : 2663519\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663520\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1846 : 2663520\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663521\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34 : 2663521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663522\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3096 : 2663522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663523\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2027W : 2663523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663524\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000035 : 2663524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663525\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4063W : 2663525\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663527\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000030 : 2663527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663528\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_13 : 2663528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663529\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1941 : 2663529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663530\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0000068-2 : 2663530\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663531\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P377 : 2663531\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663532\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P372 : 2663532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663533\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_5 : 2663533\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663534\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004240-3 : 2663534\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663535\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000320 : 2663535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663536\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3142P : 2663536\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663537\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P355 : 2663537\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663538\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1682 : 2663538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663539\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P179 : 2663539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663540\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3451 : 2663540\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663541\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4241M : 2663541\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663542\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004746-9 : 2663542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663543\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0003398-0 : 2663543\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663544\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0002457-5 : 2663544\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663545\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000048 : 2663545\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663546\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2688 : 2663546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663547\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 : 2663547\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663548\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_38 : 2663548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663549\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000128 : 2663549\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663562\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2041Q : 2663562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663563\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4023C : 2663563\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663564\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4068M : 2663564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663566\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "B : 2663566\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663567\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P367 : 2663567\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663568\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3147D : 2663568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663569\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2741 : 2663569\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663570\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2776 : 2663570\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663571\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 : 2663571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663572\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0005235-2 : 2663572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663573\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3725 : 2663573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663575\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004236-1 : 2663575\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663576\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_24 : 2663576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663577\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0002960-8 : 2663577\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663578\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P108 : 2663578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663579\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11 : 2663579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663580\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 : 2663580\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663581\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13 : 2663581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663583\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 : 2663583\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663584\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15 : 2663584\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663585\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14 : 2663585\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663586\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 : 2663586\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663587\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004172-8 : 2663587\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663588\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P160 : 2663588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663589\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0005229-5 : 2663589\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663590\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P164 : 2663590\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663591\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_67 : 2663591\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000059 : 2663592\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663593\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2026 : 2663593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663594\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_64 : 2663594\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663595\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0005973-8 : 2663595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663596\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0001112-7 : 2663596\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663597\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_60 : 2663597\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663598\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004013-4 : 2663598\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663599\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2511 : 2663599\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663600\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000056 : 2663600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663601\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_68 : 2663601\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663602\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2731 : 2663602\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663603\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2436 : 2663603\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663604\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3103J : 2663604\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663605\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P318 : 2663605\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663606\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4024R : 2663606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663607\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4144U : 2663607\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663608\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_81 : 2663608\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663609\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_80 : 2663609\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663610\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P316 : 2663610\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663611\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2049 : 2663611\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663612\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004304-7 : 2663612\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663613\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000189 : 2663613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663614\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0000221-7 : 2663614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663615\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_75 : 2663615\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663616\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_82 : 2663616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663617\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3058W : 2663617\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663618\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P288 : 2663618\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663619\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3430 : 2663619\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663620\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_16 : 2663620\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663621\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P251 : 2663621\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663622\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_84 : 2663622\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663623\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000267 : 2663623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663624\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2918 : 2663624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_10 : 2663625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663626\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_11 : 2663626\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663627\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000263 : 2663627\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663628\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_17 : 2663628\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663629\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_14 : 2663629\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663630\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_15 : 2663630\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663631\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004003-5 : 2663631\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663632\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_18 : 2663632\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663633\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2047T : 2663633\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663634\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28 : 2663634\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663635\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D : 2663635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663636\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3143B : 2663636\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663637\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1687 : 2663637\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663639\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3439 : 2663639\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663640\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0001031-9 : 2663640\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663641\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0001560-7 : 2663641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663642\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0003710-6 : 2663642\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663643\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P305 : 2663643\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663644\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 : 2663644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663645\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26 : 2663645\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663647\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000071 : 2663647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663654\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000076 : 2663654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663656\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KPS_73 : 2663656\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663658\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000075 : 2663658\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663659\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P221 : 2663659\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663660\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0001162-2 : 2663660\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663661\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1972 : 2663661\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663662\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P227 : 2663662\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663663\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000276 : 2663663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663665\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 : 2663665\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663666\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000271 : 2663666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663668\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000272 : 2663668\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663669\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P000273 : 2663669\n",
      "P000176 : 2663670\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663670\n"
     ]
    }
   ],
   "source": [
    "parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/JankyPipe/output/'\n",
    "sample_annotation_file_path = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/CSV_files/sample_annotation_files/Longitudinal_fastq_path_names_and_JankyPipe_tags_filtered_with_Kraken_and_F2.csv'\n",
    "SNP_call_parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/pickled_files/variant_calling/longitudinal_SNPs/all_SNPs_between_longitudinal_pairs/'\n",
    "\n",
    "for isolate_pair_ID in isolate_pair_ID_list:\n",
    "    \n",
    "    longitudinal_SNP_variants_job = 'python /n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/python_scripts/BaseCall_variant_between_isolate_pair_collection.py ' + parent_output_dir + ' ' + sample_annotation_file_path + ' ' + isolate_pair_ID + ' ' + SNP_call_parent_output_dir\n",
    "\n",
    "    #directory where you want output + error files\n",
    "    os.chdir('/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/SLURM_logs/BaseCalls_between_isolate_pairs_SLURM_logs/')\n",
    "\n",
    "    job_name = isolate_pair_ID\n",
    "\n",
    "    s = Slurm(job_name , {'partition':'short' , 'n':'1' , 't':'0-03:00:00' , 'mem-per-cpu':'24G' , 'mail-type':'FAIL' , 'mail-user':'roger_vargas@g.harvard.edu'})\n",
    "\n",
    "    #submits the job\n",
    "    job_id = s.run(longitudinal_SNP_variants_job)\n",
    "\n",
    "    print job_name  + ' : ' +  str(job_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "############################################################################################################################################################################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Submit Jobs - Replicate Isolates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "############################################################################################################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from slurmpy import Slurm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import annotation file for all sample pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#annotation file for all isolates and corresponding tags\n",
    "sample_annotation = pd.read_csv('/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/CSV_files/sample_annotation_files/Replicate_fastq_path_names_and_JankyPipe_tags_filtered_with_Kraken_and_F2.csv' , sep = ',').set_index('patient_id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find which replicate pairs have 2 valid VCF files for both replicate isolates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "successful_JankyPipe_run = []\n",
    "\n",
    "for isolate_i in range(0 , np.shape(sample_annotation)[0]):\n",
    "\n",
    "    #get the tag ID for the fastq files (same as ID for fastq files)\n",
    "    tag = sample_annotation.tag[isolate_i]\n",
    "\n",
    "    #where pilon VCF and lineage information will be stored [LAB FOLDER]\n",
    "    output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/JankyPipe/output_REPLICATES/' + tag\n",
    "    \n",
    "    #check to see 'Lineage Call' folder exists in the output directory (last thing that is run in JankyPipe)\n",
    "    if os.path.exists(output_dir + '/fast-lineage-caller/'):\n",
    "        \n",
    "        successful_JankyPipe_run.append('yes')\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        successful_JankyPipe_run.append('no')\n",
    "        \n",
    "sample_annotation['successful_JankyPipe_run'] = successful_JankyPipe_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fastq_files</th>\n",
       "      <th>population</th>\n",
       "      <th>run_ID</th>\n",
       "      <th>sample_ID</th>\n",
       "      <th>sample_order</th>\n",
       "      <th>tag</th>\n",
       "      <th>isolate_type</th>\n",
       "      <th>successful_JankyPipe_run</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>patient_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [fastq_files, population, run_ID, sample_ID, sample_order, tag, isolate_type, successful_JankyPipe_run]\n",
       "Index: []"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_annotation[sample_annotation.successful_JankyPipe_run == 'no']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#subjects for which we do not have 2 good VCF files for\n",
    "bad_isolate_pair_IDs = set( list( sample_annotation[sample_annotation.successful_JankyPipe_run == 'no'].index ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "print list(bad_isolate_pair_IDs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get list of subjects (longitudinal isolate pairs) to get SNP variants from"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "isolate_pair_ID_list = list( set( list( sample_annotation.index ) ) - bad_isolate_pair_IDs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['replicate_pair_19', 'replicate_pair_18', 'replicate_pair_27', 'replicate_pair_26', 'replicate_pair_11', 'replicate_pair_36', 'replicate_pair_13', 'replicate_pair_34', 'replicate_pair_15', 'replicate_pair_32', 'replicate_pair_57', 'replicate_pair_16', 'P12_1', 'replicate_pair_54', 'P12_3', 'P12_2', 'replicate_pair_51', 'replicate_pair_50', 'replicate_pair_73', 'replicate_pair_72', 'replicate_pair_71', 'replicate_pair_70', 'replicate_pair_59', 'replicate_pair_58', 'replicate_pair_74', 'replicate_pair_9', 'replicate_pair_8', 'replicate_pair_1', 'replicate_pair_0', 'replicate_pair_3', 'replicate_pair_2', 'replicate_pair_5', 'replicate_pair_4', 'replicate_pair_7', 'replicate_pair_6', 'I0002918-6', 'replicate_pair_37', 'replicate_pair_42', 'replicate_pair_48', 'replicate_pair_10', 'replicate_pair_45', 'replicate_pair_12', 'replicate_pair_30', 'replicate_pair_33', 'replicate_pair_14', 'replicate_pair_46', 'replicate_pair_47', 'replicate_pair_44', 'replicate_pair_23', 'replicate_pair_24', 'replicate_pair_69', 'replicate_pair_40', 'replicate_pair_41', 'replicate_pair_28', 'replicate_pair_29', 'replicate_pair_66', 'replicate_pair_67', 'replicate_pair_60', 'replicate_pair_61', 'replicate_pair_62', 'replicate_pair_49', 'I0003922-7', 'replicate_pair_21', 'replicate_pair_22', 'replicate_pair_68', 'replicate_pair_56', 'I0004220-5', 'I0003710-6', 'replicate_pair_43', 'replicate_pair_25']\n"
     ]
    }
   ],
   "source": [
    "print isolate_pair_ID_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(isolate_pair_ID_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Submit a job for each subject to get the SNPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663672\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_19 : 2663672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663673\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_18 : 2663673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663674\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_27 : 2663674\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663675\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_26 : 2663675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663676\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_11 : 2663676\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663677\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_36 : 2663677\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663678\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_13 : 2663678\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663679\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_34 : 2663679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663680\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_15 : 2663680\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663681\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_32 : 2663681\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663682\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_57 : 2663682\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663683\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_16 : 2663683\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663684\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P12_1 : 2663684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663685\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_54 : 2663685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663686\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P12_3 : 2663686\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663687\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P12_2 : 2663687\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_51 : 2663688\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663689\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_50 : 2663689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663690\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_73 : 2663690\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663691\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_72 : 2663691\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663692\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_71 : 2663692\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663693\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_70 : 2663693\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663694\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_59 : 2663694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663695\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_58 : 2663695\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663696\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_74 : 2663696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663697\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_9 : 2663697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663698\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_8 : 2663698\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663699\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_1 : 2663699\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663700\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_0 : 2663700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663701\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_3 : 2663701\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663702\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_2 : 2663702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663703\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_5 : 2663703\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663704\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_4 : 2663704\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663705\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_7 : 2663705\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663706\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_6 : 2663706\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663707\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0002918-6 : 2663707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663708\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_37 : 2663708\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663709\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_42 : 2663709\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663710\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_48 : 2663710\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663711\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_10 : 2663711\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663712\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_45 : 2663712\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663714\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_12 : 2663714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663715\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_30 : 2663715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663716\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_33 : 2663716\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663717\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_14 : 2663717\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663718\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_46 : 2663718\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663719\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_47 : 2663719\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663720\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_44 : 2663720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663721\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_23 : 2663721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663722\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_24 : 2663722\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663723\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_69 : 2663723\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663724\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_40 : 2663724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663725\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_41 : 2663725\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663726\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_28 : 2663726\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663727\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_29 : 2663727\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663728\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_66 : 2663728\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663748\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_67 : 2663748\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663750\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_60 : 2663750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663751\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_61 : 2663751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663752\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_62 : 2663752\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663753\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_49 : 2663753\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663754\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0003922-7 : 2663754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663755\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_21 : 2663755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663756\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_22 : 2663756\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663757\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_68 : 2663757\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663759\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_56 : 2663759\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663760\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0004220-5 : 2663760\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663761\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I0003710-6 : 2663761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663762\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replicate_pair_43 : 2663762\n",
      "replicate_pair_25 : 2663763\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "submitted: Submitted batch job 2663763\n"
     ]
    }
   ],
   "source": [
    "parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/JankyPipe/output_REPLICATES/'\n",
    "sample_annotation_file_path = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/CSV_files/sample_annotation_files/Replicate_fastq_path_names_and_JankyPipe_tags_filtered_with_Kraken_and_F2.csv'\n",
    "SNP_call_parent_output_dir = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/pickled_files/variant_calling/replicate_SNPs/all_SNPs_between_replicate_pairs/'\n",
    "\n",
    "for isolate_pair_ID in isolate_pair_ID_list:\n",
    "    \n",
    "    replicate_SNP_variants_job = 'python /n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/python_scripts/BaseCall_variant_between_isolate_pair_collection.py ' + parent_output_dir + ' ' + sample_annotation_file_path + ' ' + isolate_pair_ID + ' ' + SNP_call_parent_output_dir\n",
    "\n",
    "    #directory where you want output + error files\n",
    "    os.chdir('/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/SLURM_logs/BaseCalls_between_isolate_pairs_SLURM_logs/')\n",
    "\n",
    "    job_name = isolate_pair_ID\n",
    "\n",
    "    s = Slurm(job_name , {'partition':'short' , 'n':'1' , 't':'0-03:00:00' , 'mem-per-cpu':'24G' , 'mail-type':'FAIL' , 'mail-user':'roger_vargas@g.harvard.edu'})\n",
    "\n",
    "    #submits the job\n",
    "    job_id = s.run(replicate_SNP_variants_job)\n",
    "\n",
    "    print job_name  + ' : ' +  str(job_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}