5_run_ldsc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import glob\n",
    "import subprocess as sp\n",
    "import multiprocessing as mp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from basic_tools import *\n",
    "\n",
    "debug=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_ldsc(pheno_code,ld,output,mode='original',samp_prev=np.nan,pop_prev=np.nan):\n",
    "    if os.path.exists(ldsc_path.format(pheno_code)+'.log'):\n",
    "        print(\"Congratulations!. ldsc result of\",pheno_code,\"exists. passed.\")\n",
    "        return\n",
    "    if mode=='original':\n",
    "        script=['ldsc.py','--h2',sumstats_path.format(pheno_code)+'.sumstats.gz', \n",
    "                 '--ref-ld-chr',ld_path.format(ld,''),\n",
    "                 '--w-ld-chr',wld_path,\n",
    "                 '--out',ldsc_path.format(output)]\n",
    "    elif mode=='my':\n",
    "        script=['ldsc_my.py','--h2',sumstats_path.format(pheno_code)+'.sumstats.gz', \n",
    "         '--ref-ld-chr',ld_path.format(ld,''),\n",
    "         '--w-ld-chr',wld_path,\n",
    "         '--out',ldsc_path.format(output)] \n",
    "    else:\n",
    "        print(\"run_ldsc mode Error!!!!!!!\")\n",
    "    \n",
    "    if np.isnan(samp_prev)==False and np.isnan(pop_prev)==False:\n",
    "        script+=['--samp-prev',str(samp_prev),'--pop-prev',str(pop_prev)]\n",
    "    \n",
    "    print('Started:',' '.join(script))\n",
    "    sp.call(script) \n",
    "    print('Finished:',' '.join(script))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_ldsc_wrapper(prefix,scale,pheno_code,samp_prev=np.nan,pop_prev=np.nan):\n",
    "    run_ldsc(pheno_code,prefix,'{}.{}'.format(prefix,pheno_code),mode='original' if mode=='uni' else 'my',samp_prev=samp_prev,pop_prev=pop_prev)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/ch6845/tools/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py',\n",
       " '-f',\n",
       " '/run/user/1119/jupyter/kernel-5c24e7ec-669b-4b5c-9ad1-fb1c6743f311.json']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.argv#uni 0 20 x x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "invalid literal for int() with base 10: '/run/user/1119/jupyter/kernel-1b1e6c2c-6a82-43fe-98d2-b11eef56c902.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-0f0330acaa60>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mscale\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mcores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: invalid literal for int() with base 10: '/run/user/1119/jupyter/kernel-1b1e6c2c-6a82-43fe-98d2-b11eef56c902.json'"
     ]
    }
   ],
   "source": [
    "mode=sys.argv[1]\n",
    "scale=int(sys.argv[2])\n",
    "cores=int(sys.argv[3])\n",
    "start=int(sys.argv[4])\n",
    "end=int(sys.argv[5])\n",
    "\n",
    "if mode=='uni':\n",
    "    prefix=mode\n",
    "else:\n",
    "    prefix=mode+str(scale)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#start,end,prefix=0,1000,'bp300'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenotypes_uni_filtered['prevalence']=phenotypes_uni_filtered['n_cases']/phenotypes_uni_filtered['n_non_missing']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(561, 16)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phenotypes_uni_filtered.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'start' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-ec3fcc97a9bf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mpheno_code_list_todo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mphenotypes_uni_filtered\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mldsc_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{}.{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprefix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.log'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;31m#print(ldsc_path.format('{}.{}'.format(prefix,idx))+'.log','exists')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'start' is not defined"
     ]
    }
   ],
   "source": [
    "pheno_code_list_todo=[]\n",
    "\n",
    "for idx,row in phenotypes_uni_filtered.iloc[start:end].iterrows():\n",
    "    if os.path.exists(ldsc_path.format('{}.{}'.format(prefix,idx))+'.log'):\n",
    "        #print(ldsc_path.format('{}.{}'.format(prefix,idx))+'.log','exists')\n",
    "        continue\n",
    "    print(idx,end=' ')\n",
    "    pheno_code_list_todo.append((idx,row['prevalence']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "phenotypes_filtered['prevalence']=phenotypes_filtered['n_cases']/phenotypes_filtered['n_non_missing']\n",
    "\n",
    "phenotypes_filtered.shape\n",
    "\n",
    "pheno_code_list_todo=[]\n",
    "for idx,row in phenotypes_filtered.iloc[start:end].iterrows():\n",
    "    if os.path.exists(ldsc_path.format('{}.{}'.format(prefix,idx))+'.log'):\n",
    "        continue\n",
    "    print(idx,end=' ')\n",
    "    pheno_code_list_todo.append((idx,row['prevalence']))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "jupyter nbconvert 5_run_ldsc.ipynb --to script\n",
    "\n",
    "export SCREENDIR=$HOME/.screen\n",
    "\n",
    "start=0;end=600;mode=uni\n",
    "python 5_run_ldsc.py $mode 0 10 $start $end \n",
    "\n",
    "start=0;end=600;mode=bp\n",
    "python 5_run_ldsc.py $mode 300 10 $start $end && python 5_run_ldsc.py $mode 128 10 $start $end && python 5_run_ldsc.py $mode 64 5 $start $end && python 5_run_ldsc.py $mode 32 5 $start $end && python 5_run_ldsc.py $mode 16 5 $start $end && python 5_run_ldsc.py $mode 8 2 $start $end\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pool = mp.Pool(processes=15)\n",
    "#pool.starmap(run_ldsc_wrapper,[(mode,scale,pheno_code,prevelence,prevelence) for (pheno_code,prevelence) in pheno_code_list_todo])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pool = mp.Pool(processes=cores)\n",
    "#pool.starmap(run_ldsc_wrapper,[(mode,scale,pheno_code) for pheno_code in pheno_code_list_todo])\n",
    "pool.starmap(run_ldsc_wrapper,[(prefix,scale,pheno_code,prevelence,prevelence) for (pheno_code,prevelence) in pheno_code_list_todo])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}