1A.[GO set]Compare_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up model comparison\n",
    "\n",
    "**gpt model**\n",
    "\n",
    "gpt-4-1106-preview\n",
    "\n",
    "**local model on server**\n",
    "\n",
    "available models:\n",
    "\n",
    "| NAME           | ID           | SIZE   |\n",
    "|----------------|--------------|--------|\n",
    "| llama2:70b     | c3a7af098300 | 38 GB  |\n",
    "| llama2:7b      | fe938a131f40 | 3.8 GB |\n",
    "| llama2:latest  | fe938a131f40 | 3.8 GB |\n",
    "| mistral:7b     | 4d9f4b269c33 | 4.1 GB |\n",
    "| mixtral:latest | 99a9202f8a7a | 26 GB  |\n",
    "| mixtral:instruct| 7708c059a8bb | 26 GB  |\t\n",
    "\n",
    "\n",
    "\n",
    "**API for calling Google Gemini pro**\n",
    "\n",
    "GO TO: https://makersuite.google.com/app/apikey to get the apikey for gemini pro\n",
    "\n",
    "export GOOGLEAI_KEY = xxxx\n",
    "\n",
    "model = 'gemini-pro'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json \n",
    "from utils.openai_query import openai_chat\n",
    "from utils.prompt_factory import make_user_prompt_with_score\n",
    "from utils.server_model_query import server_model_chat\n",
    "from utils.llm_analysis_utils import process_analysis, save_progress\n",
    "from utils.genai_query import query_genai_model\n",
    "from tqdm import tqdm\n",
    "import constant\n",
    "import openai\n",
    "import os\n",
    "import logging\n",
    "import re\n",
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Example for running in the jupyter notebook**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load variables\n",
    "initialize = True # if True, then initialize the input table with llm names, analysis and score to None \n",
    "# Replace with your actual values\n",
    "config_file = './jsonFiles/toyexample_gpt35.json'  # replace with your actual config file \n",
    "input_file = './data/GO_term_analysis/100_selected_go_contaminated.csv' # replace with your actual input file\n",
    "input_sep = ','  # replace with the separator\n",
    "set_index = 'GO'  # replace with your column name that you want to set as index or None\n",
    "gene_column = 'Genes'  # replace with your actual column name for the gene list\n",
    "gene_sep = ' '  # replace with your actual separator\n",
    "gene_features = None  # replace with your path to the gene features or None if you don't want to include in the prompt\n",
    "direct = False # if True, then the prompt will be a direct sentence asking for a name and analysis from the gene set, otherwise default or customized prompt\n",
    "out_file = 'data/GO_term_analysis/model_compare/LLM_processed_model_compare_gpt_35'  # replace with your actual output file name\n",
    "\n",
    "customized_prompt = False # if True, then the prompt will be the custom prompt, if False, then the prompt will use default\n",
    "\n",
    "# load the config file\n",
    "with open(config_file) as json_file:\n",
    "    config = json.load(json_file)\n",
    "\n",
    "if customized_prompt:\n",
    "    # make sure the file exist \n",
    "    if os.path.isfile(config['CUSTOM_PROMPT_FILE']):\n",
    "        with open(config['CUSTOM_PROMPT_FILE'], 'r') as f: # replace with your actual customized prompt file\n",
    "            customized_prompt = f.read()\n",
    "            assert len(customized_prompt) > 1, \"Customized prompt is empty\"\n",
    "    else:\n",
    "        print(\"Customized prompt file does not exist\")\n",
    "        customized_prompt = None\n",
    "else:\n",
    "    customized_prompt = None\n",
    "\n",
    "# Load OpenAI key, context, and model used \n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "context = config['CONTEXT']\n",
    "model = config['MODEL']\n",
    "temperature = config['TEMP']\n",
    "max_tokens = config['MAX_TOKENS']\n",
    "if model.startswith('gpt'):\n",
    "    rate_per_token = config['RATE_PER_TOKEN']\n",
    "    DOLLAR_LIMIT = config['DOLLAR_LIMIT']\n",
    "LOG_FILE = config['LOG_NAME']+'_.log'\n",
    "\n",
    "SEED = constant.SEED\n",
    "column_prefix = model.split('-')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# handle the logger so it create a new one for each model run\n",
    "def get_logger(filename):\n",
    "    logger = logging.getLogger(filename)\n",
    "    logger.setLevel(logging.INFO)\n",
    "    if not logger.handlers:\n",
    "        file_handler = logging.FileHandler(filename)\n",
    "        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n",
    "        file_handler.setFormatter(formatter)\n",
    "        logger.addHandler(file_handler)\n",
    "    return logger\n",
    "\n",
    "\n",
    "def main(df):\n",
    "    analysis_dict  = {}\n",
    "\n",
    "    logger = get_logger(f'{out_file}.log')\n",
    "\n",
    "    i = 0 #used for track progress and saving the file\n",
    "    for idx, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "        #only process None rows \n",
    "        if pd.notna(row[f'{column_prefix} Analysis']):\n",
    "            continue\n",
    "        \n",
    "        gene_data = row[gene_column]\n",
    "        # if gene_data is not a string, then skip\n",
    "        if type(gene_data) != str:\n",
    "            \n",
    "            logger.warning(f'Gene set {idx} is not a string, skipping')\n",
    "            continue\n",
    "        genes = gene_data.split(gene_sep)\n",
    "        \n",
    "        if len(genes) >1000:\n",
    "            logger.warning(f'Gene set {idx} is too big, skipping')\n",
    "            continue\n",
    "\n",
    "        try:\n",
    "            prompt = make_user_prompt_with_score(genes)\n",
    "            # print(prompt)\n",
    "            finger_print = None\n",
    "            if model.startswith('gpt'):\n",
    "                print(\"Accessing OpenAI API\")\n",
    "                analysis, finger_print = openai_chat(context, prompt, model, temperature, max_tokens, rate_per_token, LOG_FILE, DOLLAR_LIMIT, SEED)\n",
    "            elif model.startswith('gemini'):\n",
    "                print(\"Using Google Gemini API\")\n",
    "                analysis, error_message = query_genai_model(f\"{context}\\n{prompt}\", model, temperature, max_tokens, LOG_FILE) \n",
    "            else:\n",
    "                print(\"Using server model\")\n",
    "                analysis, error_message= server_model_chat(context, prompt, model, temperature, max_tokens,LOG_FILE, SEED)\n",
    "\n",
    "            \n",
    "            if analysis:\n",
    "                # print(analysis)\n",
    "                llm_name, llm_score, llm_analysis = process_analysis(analysis)\n",
    "                # clean up the score and return float\n",
    "                try:\n",
    "                    llm_score_value =  float(re.sub(\"[^0-9.-]\", \"\", llm_score))\n",
    "                except ValueError:\n",
    "                    llm_score_value = llm_score\n",
    "            \n",
    "                \n",
    "                df.loc[idx, f'{column_prefix} Name'] = llm_name\n",
    "                df.loc[idx, f'{column_prefix} Analysis'] = llm_analysis\n",
    "                df.loc[idx, f'{column_prefix} Score'] = llm_score_value\n",
    "                \n",
    "                analysis_dict[f'{idx}_{column_prefix}'] = analysis\n",
    "                # Log success with fingerprint\n",
    "                logger.info(f'Success for {idx} {column_prefix}.')\n",
    "                if finger_print:\n",
    "                    logger.info(f'GPT_Fingerprint for {idx}: {finger_print}')\n",
    "                    \n",
    "            else:\n",
    "                logger.error(f'Error for query gene set {idx}: {error_message}')\n",
    "\n",
    "        except Exception as e:\n",
    "            logger.error(f'Error for {idx}: {e}')\n",
    "            continue\n",
    "        i += 1\n",
    "        if i % 10 == 0:\n",
    "            # bin scores into no score, low score, medium score, high score\n",
    "            bins = [-np.inf, 0, 0.79, 0.86, np.inf] # 0 is no score (name not assigned), between 0 to 0.79 is low score, between 0.8 to 0.86 is medium score, above 0.86 is high score\n",
    "            labels = ['Name not assigned', 'Low Score', 'Medium Score', 'High Score']  # Define the corresponding labels\n",
    "            \n",
    "            df[f'{column_prefix} Score bins'] = pd.cut(df[f'{column_prefix} Score'], bins=bins, labels=labels)\n",
    "                \n",
    "            save_progress(df, analysis_dict, out_file)\n",
    "            # df.to_csv(f'{out_file}.tsv', sep='\\t', index=True)\n",
    "            print(f\"Saved progress for {i} genesets\")\n",
    "    # save the final file\n",
    "    save_progress(df, analysis_dict, out_file)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "from glob import glob\n",
    "\n",
    "\n",
    "initialize = True \n",
    "input_file = 'data/GO_term_analysis/toy_example_w_contaminated.csv'\n",
    "input_sep = constant.GO_FILE_SEP\n",
    "set_index = constant.GO_INDEX_COL  \n",
    "gene_column = constant.GO_GENE_COL \n",
    "gene_sep = ' '\n",
    "\n",
    "## create a param file \n",
    "configs = glob('./jsonFiles/toyexample_*.json')\n",
    "params = []\n",
    "for conf_file in configs:\n",
    "    model_names = '_'.join(conf_file.split('/')[-1].split('.')[0].split('_')[1:])\n",
    "    # print(model_names)\n",
    "    out_file = f'data/GO_term_analysis/LLM_processed_toy_example_w_contamination_{model_names}'  \n",
    "    param = f\"--config {conf_file} \\\n",
    "        --initialize \\\n",
    "        --input {input_file} \\\n",
    "        --input_sep  '{input_sep}'\\\n",
    "        --set_index {set_index} \\\n",
    "        --gene_column {gene_column}\\\n",
    "        --gene_sep '{gene_sep}' \\\n",
    "        --start 0 \\\n",
    "        --end 10 \\\n",
    "        --output_file {out_file}\"\n",
    "    print(param)\n",
    "    params.append(param)\n",
    "\n",
    "with open('toy_example_params.txt', 'w') as f:\n",
    "    for p in params:\n",
    "        f.write(p+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define your own loop for running the pipeline\n",
    "## 12-18-2023: this loop is for run the default gene set and the contaminated gene sets \n",
    "## can modify this loop for different models or only run on default gene set\n",
    "\n",
    "##12-27-23: edited the prompt \n",
    "\n",
    "##01-26-2023: test with bin scores\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    df = pd.read_csv(input_file, sep=input_sep, index_col=set_index)\n",
    "    \n",
    "    if 'gpt' in model:\n",
    "        name_fix = '_'.join(model.split('-')[:2])\n",
    "    else:\n",
    "        name_fix = model.replace(':', '_')\n",
    "    column_prefix = name_fix + '_default'\n",
    "    print(column_prefix)\n",
    "    if initialize:\n",
    "        # initialize the input file with llm names, analysis and score to None\n",
    "        df[f'{column_prefix} Name'] = None\n",
    "        df[f'{column_prefix} Analysis'] = None\n",
    "        df[f'{column_prefix} Score'] = -np.inf\n",
    "    main(df)  ## run with the real set \n",
    "    \n",
    "    ## run the pipeline for contaiminated gene sets \n",
    "    contaminated_columns = [col for col in df.columns if col.endswith('contaminated_Genes')]\n",
    "    # print(contaminated_columns)\n",
    "    for col in contaminated_columns:\n",
    "        gene_column = col ## Note need to change the gene_column to the contaminated column\n",
    "        contam_prefix = '_'.join(col.split('_')[0:2])\n",
    "        \n",
    "        column_prefix = name_fix + '_' +contam_prefix\n",
    "        print(column_prefix)\n",
    "\n",
    "        if initialize:\n",
    "            # initialize the input file with llm names, analysis and score to None\n",
    "            df[f'{column_prefix} Name'] = None\n",
    "            df[f'{column_prefix} Analysis'] = None\n",
    "            df[f'{column_prefix} Score'] = -np.inf\n",
    "        main(df)\n",
    "    df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mixtral_latest_default\n",
      "Genes\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/11 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using server model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11/11 [01:34<00:00,  8.58s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50perc_contaminated_Genes\n",
      "mixtral_latest_50perc_contaminated\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11/11 [00:00<00:00, 15246.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100perc_contaminated_Genes\n",
      "mixtral_latest_100perc_contaminated\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11/11 [00:00<00:00, 16098.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# check if there is any None in the analysis column, then rerun the pipeline\n",
    "\n",
    "initialize = False \n",
    "\n",
    "SEED = 42\n",
    "model_options = ['gemini-pro','mistral:7b', 'mixtral:latest', 'llama2:7b', 'llama2:70b']\n",
    "# model_options = ['mixtral:latest']  # llama2 7b has formatting issue, ingore \n",
    "input_sep = '\\t'\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    for m in model_options:\n",
    "        input_file\n",
    "        model = m\n",
    "        \n",
    "        if '-' in model:\n",
    "            name_fix = '_'.join(model.split('-')[:2])\n",
    "        else:\n",
    "            name_fix = model.replace(':', '_')\n",
    "        input_file = f'data/GO_term_analysis/LLM_processed_toy_example_w_contamination_{name_fix}.tsv' # replace with your actual input file\n",
    "        out_file = f'data/GO_term_analysis/LLM_processed_toy_example_w_contamination_{name_fix}'  # save to the same file name as the input file\n",
    "        LOG_FILE = config['LOG_NAME']+f'_{name_fix}'+'.log'\n",
    "\n",
    "        df = pd.read_csv(input_file, sep=input_sep, index_col=set_index)\n",
    "        # print(df.head())\n",
    "        column_prefix = name_fix + '_default' #this is default\n",
    "        print(column_prefix)\n",
    "        \n",
    "        gene_column = constant.GO_GENE_COL\n",
    "        print(gene_column)\n",
    "        if initialize:\n",
    "            # initialize the input file with llm names, analysis and score to None\n",
    "            df[f'{column_prefix} Name'] = None\n",
    "            df[f'{column_prefix} Analysis'] = None\n",
    "            df[f'{column_prefix} Score'] = None\n",
    "        main(df)  ## run with the real set \n",
    "        \n",
    "        ## run the pipeline for contaiminated gene sets \n",
    "        contaminated_columns = [col for col in df.columns if col.endswith('contaminated_Genes')]\n",
    "        # print(contaminated_columns)\n",
    "        for col in contaminated_columns:\n",
    "            gene_column = col ## Note need to change the gene_column to the contaminated column\n",
    "            print(gene_column)\n",
    "            contam_prefix = '_'.join(col.split('_')[0:2])\n",
    "            column_prefix = name_fix + '_' +contam_prefix\n",
    "            print(column_prefix)\n",
    "\n",
    "            if initialize:\n",
    "                # initialize the input file with llm names, analysis and score to None\n",
    "                df[f'{column_prefix} Name'] = None\n",
    "                df[f'{column_prefix} Analysis'] = None\n",
    "                df[f'{column_prefix} Score'] = None\n",
    "            main(df)\n",
    "            \n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run for the 100 sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "comparison_gpt_4\n",
      "--config ./jsonFiles/model_comparison_gpt_4.json             --initialize  True            --input data/GO_term_analysis/model_comparison_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 0             --end 50             --output_file data/GO_term_analysis/model_compare/LLM_processed_model_compare_comparison_gpt_4_0_50\n",
      "comparison_mixtral_latest\n",
      "--config ./jsonFiles/model_comparison_mixtral_latest.json             --initialize  True            --input data/GO_term_analysis/model_comparison_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 0             --end 50             --output_file data/GO_term_analysis/model_compare/LLM_processed_model_compare_comparison_mixtral_latest_0_50\n",
      "comparison_gemini_pro\n",
      "--config ./jsonFiles/model_comparison_gemini_pro.json             --initialize  True            --input data/GO_term_analysis/model_comparison_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 0             --end 50             --output_file data/GO_term_analysis/model_compare/LLM_processed_model_compare_comparison_gemini_pro_0_50\n",
      "comparison_gpt_4\n",
      "--config ./jsonFiles/model_comparison_gpt_4.json             --initialize  True            --input data/GO_term_analysis/model_comparison_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 50             --end 100             --output_file data/GO_term_analysis/model_compare/LLM_processed_model_compare_comparison_gpt_4_50_100\n",
      "comparison_mixtral_latest\n",
      "--config ./jsonFiles/model_comparison_mixtral_latest.json             --initialize  True            --input data/GO_term_analysis/model_comparison_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 50             --end 100             --output_file data/GO_term_analysis/model_compare/LLM_processed_model_compare_comparison_mixtral_latest_50_100\n",
      "comparison_gemini_pro\n",
      "--config ./jsonFiles/model_comparison_gemini_pro.json             --initialize  True            --input data/GO_term_analysis/model_comparison_terms.csv             --input_sep  ','            --set_index GO             --gene_column Genes            --gene_sep ' '             --start 50             --end 100             --output_file data/GO_term_analysis/model_compare/LLM_processed_model_compare_comparison_gemini_pro_50_100\n",
      "number of params:  6\n"
     ]
    }
   ],
   "source": [
    "## set up parameters for running the pipeline for every 50 rows\n",
    "import os \n",
    "from glob import glob\n",
    "# Define start, step, and end values\n",
    "start = 0\n",
    "step = 50\n",
    "end = 100\n",
    "\n",
    "# Create a range list\n",
    "range_list = list(range(start, end + step, step))\n",
    "\n",
    "# Create tuples for each consecutive pair in the list\n",
    "tuple_list = [(range_list[i], range_list[i+1]) for i in range(len(range_list)-1)]\n",
    "\n",
    "\n",
    "initialize = True \n",
    "input_file = 'data/GO_term_analysis/model_comparison_terms.csv'\n",
    "input_sep = constant.GO_FILE_SEP\n",
    "set_index = constant.GO_INDEX_COL  \n",
    "gene_column = constant.GO_GENE_COL \n",
    "gene_sep = ' '\n",
    "\n",
    "## create a param file \n",
    "configs = glob('./jsonFiles/model_comparison_*.json')\n",
    "params = []\n",
    "for start, end in tuple_list:\n",
    "    for conf_file in configs:\n",
    "        model_names = '_'.join(conf_file.split('/')[-1].split('.')[0].split('_')[1:])\n",
    "        print(model_names)\n",
    "        \n",
    "        out_file = f'data/GO_term_analysis/model_compare/LLM_processed_model_compare_{model_names}_{start}_{end}'  \n",
    "        param = f\"--config {conf_file} \\\n",
    "            --initialize \\\n",
    "            --input {input_file} \\\n",
    "            --input_sep  '{input_sep}'\\\n",
    "            --set_index {set_index} \\\n",
    "            --gene_column {gene_column}\\\n",
    "            --gene_sep '{gene_sep}' \\\n",
    "            --run_contaminated \\\n",
    "            --start {start} \\\n",
    "            --end {end} \\\n",
    "            --output_file {out_file}\"\n",
    "        print(param)\n",
    "        params.append(param)\n",
    "print('number of params: ', len(params))\n",
    "    \n",
    "\n",
    "with open('model_compare_params.txt', 'w') as f:\n",
    "    for p in params:\n",
    "        f.write(p+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checkout and combine the output from the batch run "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compare_100set_mixtral_latest mixtral_latest_default Analysis pass\n",
      "-----------------------\n",
      "compare_100set_mixtral_latest mixtral_latest_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "compare_100set_mixtral_latest mixtral_latest_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "llama2_70b_50_100 llama2_70b_default Analysis pass\n",
      "-----------------------\n",
      "llama2_70b_50_100 llama2_70b_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "llama2_70b_50_100 llama2_70b_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gemini_pro_50_100 gemini_pro_default Analysis pass\n",
      "-----------------------\n",
      "gemini_pro_50_100 gemini_pro_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gemini_pro_50_100 gemini_pro_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "mixtral_latest_0_50 mixtral_latest_default Analysis pass\n",
      "-----------------------\n",
      "mixtral_latest_0_50 mixtral_latest_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "mixtral_latest_0_50 mixtral_latest_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "compare_100set_gpt_4 gpt_4_default Analysis pass\n",
      "-----------------------\n",
      "compare_100set_gpt_4 gpt_4_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "compare_100set_gpt_4 gpt_4_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "compare_100set_llama2_70b llama2_70b_default Analysis pass\n",
      "-----------------------\n",
      "compare_100set_llama2_70b llama2_70b_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "compare_100set_llama2_70b llama2_70b_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "compare_100set_gemini_pro gemini_pro_default Analysis pass\n",
      "-----------------------\n",
      "compare_100set_gemini_pro gemini_pro_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "compare_100set_gemini_pro gemini_pro_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "mixtral_instruct_0_50 mixtral_instruct_default Analysis pass\n",
      "-----------------------\n",
      "mixtral_instruct_0_50 mixtral_instruct_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "mixtral_instruct_0_50 mixtral_instruct_100perc_contaminated Analysis has 10 None in the analysis column\n",
      "-----------------------\n",
      "mixtral_latest_50_100 mixtral_latest_default Analysis pass\n",
      "-----------------------\n",
      "mixtral_latest_50_100 mixtral_latest_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "mixtral_latest_50_100 mixtral_latest_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "llama2_70b_0_50 llama2_70b_default Analysis pass\n",
      "-----------------------\n",
      "llama2_70b_0_50 llama2_70b_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "llama2_70b_0_50 llama2_70b_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gpt_4_50_100 gpt_4_default Analysis pass\n",
      "-----------------------\n",
      "gpt_4_50_100 gpt_4_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gpt_4_50_100 gpt_4_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gemini_pro_0_50 gemini_pro_default Analysis pass\n",
      "-----------------------\n",
      "gemini_pro_0_50 gemini_pro_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gemini_pro_0_50 gemini_pro_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gpt_4_0_50 gpt_4_default Analysis pass\n",
      "-----------------------\n",
      "gpt_4_0_50 gpt_4_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "gpt_4_0_50 gpt_4_100perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "mixtral_instruct_50_100 mixtral_instruct_default Analysis pass\n",
      "-----------------------\n",
      "mixtral_instruct_50_100 mixtral_instruct_50perc_contaminated Analysis pass\n",
      "-----------------------\n",
      "mixtral_instruct_50_100 mixtral_instruct_100perc_contaminated Analysis pass\n",
      "-----------------------\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "processed_files = glob('data/GO_term_analysis/model_compare/LLM_processed_model_compare*.tsv')\n",
    "# processed_files\n",
    "# check any with None in the analysis column\n",
    "for file in processed_files:\n",
    "    model_names = '_'.join(file.split('/')[-1].split('.')[0].split('_')[-4:])\n",
    "    \n",
    "    df = pd.read_csv(file, sep='\\t')\n",
    "    # column names end with Analysis\n",
    "    analysis_cols = [col for col in df.columns if col.endswith('Analysis')]\n",
    "    for col in analysis_cols:\n",
    "        if df[col].isna().sum() > 0:\n",
    "            n_none = df[col].isna().sum()\n",
    "            print(f'{model_names} {col} has {n_none} None in the analysis column')\n",
    "        else:\n",
    "            print(f'{model_names} {col} pass')\n",
    "        print('-----------------------')\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mixtral_instruct\n",
      "['data/GO_term_analysis/model_compare/LLM_processed_model_compare_mixtral_instruct_0_50.tsv', 'data/GO_term_analysis/model_compare/LLM_processed_model_compare_mixtral_instruct_50_100.tsv']\n",
      "(100, 14)\n",
      "------------saved--------------\n"
     ]
    }
   ],
   "source": [
    "## combine the 0-50 and 50-100 files together\n",
    "from glob import glob\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "processed_files = glob('data/GO_term_analysis/model_compare/LLM_processed_model_compare*.tsv')\n",
    "\n",
    "# model_names = ['mixtral_instruct']\n",
    "for file in processed_files:\n",
    "    model_name = '_'.join(file.split('/')[-1].split('.')[0].split('_')[-4:-2])\n",
    "    model_names.append(model_name)\n",
    "model_names = list(set(model_names))\n",
    "\n",
    "for model in model_names:\n",
    "    print(model)\n",
    "    files = [file for file in processed_files if model in file]\n",
    "    print(files)\n",
    "    df = pd.concat([pd.read_csv(file, sep='\\t', index_col='GO') for file in files])\n",
    "    \n",
    "    # add the toy example in as well \n",
    "    toy_file = f'data/GO_term_analysis/LLM_processed_toy_example_w_contamination_{model}.tsv'\n",
    "    \n",
    "    df = pd.concat([df, pd.read_csv(toy_file, sep='\\t', index_col='GO')])\n",
    "    # check any with None in the analysis column\n",
    "    analysis_columns = [col for col in df.columns if col.endswith('Analysis')]\n",
    "    for col in analysis_columns:\n",
    "        if df[col].isna().sum() > 0:\n",
    "            n_none = df[col].isna().sum()\n",
    "            print(f'{model} {col} has {n_none} None in the analysis column')\n",
    "    \n",
    "    print(df.shape)\n",
    "    df.to_csv(f'data/GO_term_analysis/model_compare/LLM_processed_model_compare_100set_{model}.tsv', sep='\\t', index=True)\n",
    "    print('------------saved--------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mixtral_latest_default has 0 gene sets named with unrelated proteins, 0.00%\n",
      "mixtral_latest_50perc_contaminated has 0 gene sets named with unrelated proteins, 0.00%\n",
      "mixtral_latest_100perc_contaminated has 0 gene sets named with unrelated proteins, 0.00%\n",
      "mixtral_latest_default has 0 gene sets with score 0, 0.00%\n",
      "mixtral_latest_50perc_contaminated has 0 gene sets with score 0, 0.00%\n",
      "mixtral_latest_100perc_contaminated has 0 gene sets with score 0, 0.00%\n",
      "------------------\n",
      "gpt_4_default has 4 gene sets named with unrelated proteins, 4.00%\n",
      "gpt_4_50perc_contaminated has 28 gene sets named with unrelated proteins, 28.00%\n",
      "gpt_4_100perc_contaminated has 87 gene sets named with unrelated proteins, 87.00%\n",
      "gpt_4_default has 4 gene sets with score 0, 4.00%\n",
      "gpt_4_50perc_contaminated has 28 gene sets with score 0, 28.00%\n",
      "gpt_4_100perc_contaminated has 87 gene sets with score 0, 87.00%\n",
      "------------------\n",
      "llama2_70b_default has 0 gene sets named with unrelated proteins, 0.00%\n",
      "llama2_70b_50perc_contaminated has 0 gene sets named with unrelated proteins, 0.00%\n",
      "llama2_70b_100perc_contaminated has 0 gene sets named with unrelated proteins, 0.00%\n",
      "llama2_70b_default has 0 gene sets with score 0, 0.00%\n",
      "llama2_70b_50perc_contaminated has 0 gene sets with score 0, 0.00%\n",
      "llama2_70b_100perc_contaminated has 0 gene sets with score 0, 0.00%\n",
      "------------------\n",
      "mixtral_instruct_default has 0 gene sets named with unrelated proteins, 0.00%\n",
      "mixtral_instruct_50perc_contaminated has 0 gene sets named with unrelated proteins, 0.00%\n",
      "mixtral_instruct_100perc_contaminated has 0 gene sets named with unrelated proteins, 0.00%\n",
      "mixtral_instruct_default has 0 gene sets with score 0, 0.00%\n",
      "mixtral_instruct_50perc_contaminated has 0 gene sets with score 0, 0.00%\n",
      "mixtral_instruct_100perc_contaminated has 0 gene sets with score 0, 0.00%\n",
      "------------------\n",
      "gemini_pro_default has 20 gene sets named with unrelated proteins, 20.00%\n",
      "gemini_pro_50perc_contaminated has 41 gene sets named with unrelated proteins, 41.00%\n",
      "gemini_pro_100perc_contaminated has 59 gene sets named with unrelated proteins, 59.00%\n",
      "gemini_pro_default has 20 gene sets with score 0, 20.00%\n",
      "gemini_pro_50perc_contaminated has 41 gene sets with score 0, 41.00%\n",
      "gemini_pro_100perc_contaminated has 59 gene sets with score 0, 59.00%\n",
      "------------------\n"
     ]
    }
   ],
   "source": [
    "##check for each 100 set file, how many 'systems of unrelated proteins' are assigened to each gene set \n",
    "from glob import glob\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "files = glob('data/GO_term_analysis/model_compare/LLM_processed_model_compare_100set*.tsv')\n",
    "unnamed_dict = {}\n",
    "model_names = []\n",
    "\n",
    "for file in files:\n",
    "    model_name = '_'.join(file.split('/')[-1].split('.')[0].split('_')[-2:])\n",
    "    model_names.append(model_name)\n",
    "    df = pd.read_csv(file, sep='\\t', index_col='GO')\n",
    "    name_columns = [col for col in df.columns if col.endswith('Name')]\n",
    "    \n",
    "    for col in name_columns:\n",
    "        gene_set_type = col.split(' ')[0]\n",
    "        # print(gene_set_type)\n",
    "        #number of names contains 'unrelated proteins'\n",
    "        n_unrelated = df[col].str.contains('unrelated proteins').sum()\n",
    "        n_total = df.shape[0]\n",
    "        print(f'{gene_set_type} has {n_unrelated} gene sets named with unrelated proteins, {n_unrelated/n_total*100:.2f}%')\n",
    "        unnamed_dict[f'{gene_set_type}'] = {'n_unrelated': n_unrelated, 'n_named': n_total-n_unrelated}\n",
    "    score_columns = [col for col in df.columns if col.endswith('Score')]\n",
    "    for c in score_columns:\n",
    "        gene_set_type = c.split(' ')[0]\n",
    "        # print(gene_set_type)\n",
    "        # number of scores are 0\n",
    "        n_zero = df[c].eq(0).sum()\n",
    "        n_total = df.shape[0]\n",
    "        print(f'{gene_set_type} has {n_zero} gene sets with score 0, {n_zero/n_total*100:.2f}%')\n",
    "        \n",
    "    print('------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt_4 takes 36.49 seconds per run\n",
      "gpt_4 takes 0.048349743589743595 dollars per run\n",
      "gemini_pro takes 7.88 seconds per run\n",
      "mixtral_instruct takes 46.36 seconds per run\n",
      "llama2_70b takes 61.80 seconds per run\n",
      "gpt 3.5 takes 9.57 seconds per run\n",
      "gpt 3.5 takes 0.002813441734417344 dollars per run\n",
      "average time usage:  32.41997033481214\n"
     ]
    }
   ],
   "source": [
    "## check the time for each model \n",
    "from glob import glob\n",
    "import json\n",
    "log_files = glob('./logs/model_comparison_*.log')\n",
    "# print(log_files)\n",
    "models = ['gpt_4', 'gemini_pro', 'mixtral_instruct','llama2_70b']\n",
    "total_time_per_run = 0\n",
    "for model in models:\n",
    "    logs = [file for file in log_files if model in file]\n",
    "    time = 0\n",
    "    runs = 0\n",
    "    for log in logs:\n",
    "        with open(log, 'r') as f:\n",
    "            data = json.load(f)\n",
    "        time += data['time_taken_total']\n",
    "        runs += data['runs']\n",
    "        if  model == 'gpt_4':\n",
    "            cost = data['dollars_spent']/data['runs']\n",
    "    time_per_run = time/runs\n",
    "    if model in ['mixtral_instruct','llama2_70b']:\n",
    "        time_per_run = time_per_run/ 1e9 \n",
    "        \n",
    "    print(f'{model} takes {time_per_run :.2f} seconds per run')\n",
    "    if  model == 'gpt_4':\n",
    "        print(f'{model} takes {cost} dollars per run')\n",
    "    total_time_per_run += time_per_run\n",
    "\n",
    "with open ('/cellar/users/mhu/Projects/llm_evaluation_for_gene_set_interpretation/logs/toy_example_gpt35_.log', 'r') as f:\n",
    "    data = json.load(f)\n",
    "    time = data['time_taken_total']\n",
    "    runs = data['runs']\n",
    "    time_per_run = time/runs  \n",
    "    total_time_per_run += time_per_run \n",
    "    cost = data['dollars_spent']/runs\n",
    "print(f'gpt 3.5 takes {time_per_run:.2f} seconds per run')  \n",
    "print(f'gpt 3.5 takes {cost} dollars per run')\n",
    "print('average time usage: ', total_time_per_run/5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}