1 - Extract_Information.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41a1469b-deb1-42e9-a069-28af56d295da",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/wic053/miniforge3/envs/py312/lib/python3.12/site-packages/langchain_ollama/__init__.py:4: LangChainDeprecationWarning: As of langchain-core 0.3.0, LangChain uses pydantic v2 internally. The langchain_core.pydantic_v1 module was a compatibility shim for pydantic v1, and should no longer be used. Please update the code to import from Pydantic directly.\n",
      "\n",
      "For example, replace imports like: `from langchain_core.pydantic_v1 import BaseModel`\n",
      "with: `from pydantic import BaseModel`\n",
      "or the v1 compatibility namespace if you are working in a code base that has not been fully upgraded to pydantic 2 yet. \tfrom pydantic.v1 import BaseModel\n",
      "\n",
      "  from langchain_ollama.embeddings import OllamaEmbeddings\n",
      "/Users/wic053/miniforge3/envs/py312/lib/python3.12/site-packages/pydantic/_migration.py:283: UserWarning: `pydantic.error_wrappers:ValidationError` has been moved to `pydantic:ValidationError`.\n",
      "  warnings.warn(f'`{import_path}` has been moved to `{new_location}`.')\n"
     ]
    }
   ],
   "source": [
    "# system\n",
    "import os\n",
    "import json\n",
    "\n",
    "# langchain\n",
    "from langchain_openai import AzureChatOpenAI\n",
    "from langchain_ollama import ChatOllama\n",
    "\n",
    "# data science\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "# utils\n",
    "from utils.templates import (\n",
    "    scope_extractor_template,\n",
    "    variant_extractor_template,\n",
    "    filter_extractor_template,\n",
    "    granularity_extractor_template,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "16dc97ce-a096-42e6-95b3-0040f84f4b01",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"AZURE_OPENAI_API_KEY\"] = os.environ[\"OPENAI_API_KEY\"]\n",
    "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = os.environ[\"OPENAI_API_BASE\"]\n",
    "os.environ[\"AZURE_OPENAI_API_VERSION\"] = os.environ[\"OPENAI_API_VERSION\"]\n",
    "os.environ[\"AZURE_OPENAI_CHAT_DEPLOYMENT_NAME\"] = \"firstcontact-gpt4-turbo\"\n",
    "\n",
    "del os.environ[\"OPENAI_API_BASE\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fec79601-8565-4aea-88ae-4f18ff94285a",
   "metadata": {},
   "outputs": [],
   "source": [
    "open_models = [\n",
    "    \"mistral-nemo:12b-instruct-2407-q6_K\",\n",
    "    \"llama3.1:8b-instruct-q6_K\",\n",
    "    \"gemma2:9b-instruct-q6_K\",\n",
    "    \"qwen2:7b-instruct\",\n",
    "]\n",
    "models = {\n",
    "    \"gpt4\": AzureChatOpenAI(\n",
    "        openai_api_version=os.environ[\"OPENAI_API_VERSION\"],\n",
    "        azure_deployment=\"firstcontact-gpt4-turbo\",\n",
    "        model=\"gpt-4-128k\",\n",
    "        model_kwargs={\"response_format\": {\"type\": \"json_object\"}},\n",
    "    )\n",
    "}\n",
    "models.update(\n",
    "    {\n",
    "        model: ChatOllama(model=model, temperature=0, num_predict=1024, format=\"json\")\n",
    "        for model in open_models\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb9f153c-b624-4902-9df0-76bfe8ba5d6d",
   "metadata": {},
   "source": [
    "## Extract Scope"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "04b1651f-ab69-47d1-9eb2-9a28ffc75944",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"name\": \"gpt4\",\n",
      "    \"query\": \"What are the genomic variations found at position 123456 on chromosome 7?\",\n",
      "    \"answer\": \"genomic_variants\",\n",
      "    \"predition\": {\n",
      "        \"scope\": \"g_variants\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"mistral-nemo:12b-instruct-2407-q6_K\",\n",
      "    \"query\": \"What are the genomic variations found at position 123456 on chromosome 7?\",\n",
      "    \"answer\": \"genomic_variants\",\n",
      "    \"predition\": {\n",
      "        \"scope\": \"g_variants\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"llama3.1:8b-instruct-q6_K\",\n",
      "    \"query\": \"What are the genomic variations found at position 123456 on chromosome 7?\",\n",
      "    \"answer\": \"genomic_variants\",\n",
      "    \"predition\": {\n",
      "        \"scope\": \"g_variants\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"gemma2:9b-instruct-q6_K\",\n",
      "    \"query\": \"What are the genomic variations found at position 123456 on chromosome 7?\",\n",
      "    \"answer\": \"genomic_variants\",\n",
      "    \"predition\": {\n",
      "        \"scope\": \"g_variants\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"qwen2:7b-instruct\",\n",
      "    \"query\": \"What are the genomic variations found at position 123456 on chromosome 7?\",\n",
      "    \"answer\": \"genomic_variants\",\n",
      "    \"predition\": {\n",
      "        \"scope\": \"g_variants\"\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "for name, model in models.items():\n",
    "    test_data = pd.read_excel(\"./Queries-and-answers.xlsx\", sheet_name=0, index_col=None)\n",
    "    \n",
    "    for row in test_data.iterrows():\n",
    "        index, series = row\n",
    "        entry = dict(series)\n",
    "        query = entry['question']\n",
    "        answer = entry['scope']\n",
    "        template = scope_extractor_template.format(query=query)\n",
    "        predition = model.invoke(template).content\n",
    "        print(json.dumps({\"name\": name, \"query\": query, \"answer\": answer, \"predition\": json.loads(predition)}, indent=4))\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03cd27b6",
   "metadata": {},
   "source": [
    "## Extract Filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ad7f2a41-e04d-476c-972e-34389f1063a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"name\": \"gpt4\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"breast cancer (Individuals)\",\n",
      "    \"scope_prediction\": {\n",
      "        \"scope\": \"cohorts\"\n",
      "    },\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"breast cancer\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"mistral-nemo:12b-instruct-2407-q6_K\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"breast cancer (Individuals)\",\n",
      "    \"scope_prediction\": {\n",
      "        \"scope\": \"individuals\"\n",
      "    },\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"disease=breast cancer\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"llama3.1:8b-instruct-q6_K\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"breast cancer (Individuals)\",\n",
      "    \"scope_prediction\": {\n",
      "        \"scope\": \"individuals\"\n",
      "    },\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"breast cancer\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"gemma2:9b-instruct-q6_K\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"breast cancer (Individuals)\",\n",
      "    \"scope_prediction\": {\n",
      "        \"scope\": \"individuals\"\n",
      "    },\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"breast cancer\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"qwen2:7b-instruct\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"breast cancer (Individuals)\",\n",
      "    \"scope_prediction\": {\n",
      "        \"scope\": \"cohorts\"\n",
      "    },\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"breast cancer\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "for name, model in models.items():\n",
    "    test_data = pd.read_excel(\n",
    "        \"./Queries-and-answers.xlsx\", sheet_name=2, index_col=None\n",
    "    )\n",
    "\n",
    "    for row in test_data.iterrows():\n",
    "        index, series = row\n",
    "        entry = dict(series)\n",
    "        query = entry[\"question\"]\n",
    "        answer_scope = entry[\"scope\"]\n",
    "        answer_filters = entry[\"filters_and_scope\"]\n",
    "        scope_prediction = model.invoke(\n",
    "            scope_extractor_template.format(query=query)\n",
    "        ).content\n",
    "        filters_prediction = model.invoke(\n",
    "            filter_extractor_template.format(query=query)\n",
    "        ).content\n",
    "        print(\n",
    "            json.dumps(\n",
    "                {\n",
    "                    \"name\": name,\n",
    "                    \"query\": query,\n",
    "                    \"answer_scope\": answer_scope,\n",
    "                    \"answer_filters\": answer_filters,\n",
    "                    \"scope_prediction\": json.loads(scope_prediction),\n",
    "                    \"filters_prediction\": json.loads(filters_prediction),\n",
    "                },\n",
    "                indent=4,\n",
    "            )\n",
    "        )\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "203a93b8",
   "metadata": {},
   "source": [
    "## Extract Granularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "81348ae6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"name\": \"gpt4\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer\": \"count\",\n",
      "    \"prediction\": {\n",
      "        \"granularity\": \"count\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"mistral-nemo:12b-instruct-2407-q6_K\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer\": \"count\",\n",
      "    \"prediction\": {\n",
      "        \"granularity\": \"count\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"llama3.1:8b-instruct-q6_K\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer\": \"count\",\n",
      "    \"prediction\": {\n",
      "        \"granularity\": \"count\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"gemma2:9b-instruct-q6_K\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer\": \"count\",\n",
      "    \"prediction\": {\n",
      "        \"granularity\": \"count\"\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"qwen2:7b-instruct\",\n",
      "    \"query\": \"How many patients with breast cancer are in the repository?\",\n",
      "    \"answer\": \"count\",\n",
      "    \"prediction\": {\n",
      "        \"granularity\": \"count\"\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "for name, model in models.items():\n",
    "    test_data = pd.read_excel(\n",
    "        \"./Queries-and-answers.xlsx\", sheet_name=3, index_col=None\n",
    "    )\n",
    "\n",
    "    for row in test_data.iterrows():\n",
    "        index, series = row\n",
    "        entry = dict(series)\n",
    "        query = entry[\"question\"]\n",
    "        answer = entry[\"granularity\"]\n",
    "        prediction = model.invoke(\n",
    "            granularity_extractor_template.format(query=query)\n",
    "        ).content\n",
    "        print(\n",
    "            json.dumps(\n",
    "                {\n",
    "                    \"name\": name,\n",
    "                    \"query\": query,\n",
    "                    \"answer\": answer,\n",
    "                    \"prediction\": json.loads(prediction),\n",
    "                },\n",
    "                indent=4,\n",
    "            )\n",
    "        )\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8378ee3f",
   "metadata": {},
   "source": [
    "## Extract Variants and Filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "54a3ee08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"name\": \"gpt4\",\n",
      "    \"query\": \"I want genomic variations from 550k to 650k from second chromosome, from patients having renal failure\",\n",
      "    \"answer_var\": \"2:550000.0-650000.0\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"renal failure\",\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"renal failure\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"variant_prediction\": {\n",
      "        \"success\": true,\n",
      "        \"assembly_id\": \"unknown\",\n",
      "        \"chromosome\": \"2\",\n",
      "        \"start\": [\n",
      "            550000\n",
      "        ],\n",
      "        \"end\": [\n",
      "            650000\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"mistral-nemo:12b-instruct-2407-q6_K\",\n",
      "    \"query\": \"I want genomic variations from 550k to 650k from second chromosome, from patients having renal failure\",\n",
      "    \"answer_var\": \"2:550000.0-650000.0\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"renal failure\",\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"renal failure\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"variant_prediction\": {\n",
      "        \"success\": true,\n",
      "        \"assembly_id\": \"unknown\",\n",
      "        \"chromosome\": \"2\",\n",
      "        \"start\": [\n",
      "            550000\n",
      "        ],\n",
      "        \"end\": [\n",
      "            650000\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"llama3.1:8b-instruct-q6_K\",\n",
      "    \"query\": \"I want genomic variations from 550k to 650k from second chromosome, from patients having renal failure\",\n",
      "    \"answer_var\": \"2:550000.0-650000.0\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"renal failure\",\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"renal failure\",\n",
      "                \"scope\": \"individuals\"\n",
      "            },\n",
      "            {\n",
      "                \"term\": \"chr2\",\n",
      "                \"scope\": \"biosamples\"\n",
      "            },\n",
      "            {\n",
      "                \"term\": \"550k-650k\",\n",
      "                \"scope\": \"runs\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"variant_prediction\": {\n",
      "        \"success\": true,\n",
      "        \"assembly_id\": \"grch38\",\n",
      "        \"chromosome\": \"2\",\n",
      "        \"start\": [\n",
      "            550000\n",
      "        ],\n",
      "        \"end\": [\n",
      "            650000\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"gemma2:9b-instruct-q6_K\",\n",
      "    \"query\": \"I want genomic variations from 550k to 650k from second chromosome, from patients having renal failure\",\n",
      "    \"answer_var\": \"2:550000.0-650000.0\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"renal failure\",\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"renal failure\",\n",
      "                \"scope\": \"individuals\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"variant_prediction\": {\n",
      "        \"success\": true,\n",
      "        \"assembly_id\": \"unknown\",\n",
      "        \"chromosome\": \"2\",\n",
      "        \"start\": [\n",
      "            550000,\n",
      "            650000\n",
      "        ],\n",
      "        \"end\": [\n",
      "            550000,\n",
      "            650000\n",
      "        ]\n",
      "    }\n",
      "}\n",
      "{\n",
      "    \"name\": \"qwen2:7b-instruct\",\n",
      "    \"query\": \"I want genomic variations from 550k to 650k from second chromosome, from patients having renal failure\",\n",
      "    \"answer_var\": \"2:550000.0-650000.0\",\n",
      "    \"answer_scope\": \"Individuals\",\n",
      "    \"answer_filters\": \"renal failure\",\n",
      "    \"filters_prediction\": {\n",
      "        \"filters\": [\n",
      "            {\n",
      "                \"term\": \"chromosome == '2'\",\n",
      "                \"scope\": \"biosamples\"\n",
      "            },\n",
      "            {\n",
      "                \"term\": \"position >= 550000 AND position <= 650000\",\n",
      "                \"scope\": \"genomic_variations\"\n",
      "            }\n",
      "        ]\n",
      "    },\n",
      "    \"variant_prediction\": {\n",
      "        \"success\": true,\n",
      "        \"assembly_id\": \"unknown\",\n",
      "        \"chromosome\": \"2\",\n",
      "        \"start\": [\n",
      "            550000,\n",
      "            650000\n",
      "        ],\n",
      "        \"end\": [\n",
      "            650000\n",
      "        ]\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "for name, model in models.items():\n",
    "    test_data = pd.read_excel(\n",
    "        \"./Queries-and-answers.xlsx\", sheet_name=1, index_col=None\n",
    "    )\n",
    "\n",
    "    for row in test_data.iterrows():\n",
    "        index, series = row\n",
    "        entry = dict(series)\n",
    "        query = entry[\"question\"]\n",
    "        answer_var = f'{entry[\"chromosome\"]}:{entry[\"start\"]}-{entry[\"end\"]}'\n",
    "        answer_filters = entry[\"filters\"]\n",
    "        filters_prediction = model.invoke(\n",
    "            filter_extractor_template.format(query=query)\n",
    "        ).content\n",
    "        variant_prediction = model.invoke(\n",
    "            variant_extractor_template.format(query=query)\n",
    "        ).content\n",
    "        print(\n",
    "            json.dumps(\n",
    "                {\n",
    "                    \"name\": name,\n",
    "                    \"query\": query,\n",
    "                    \"answer_var\": answer_var,\n",
    "                    \"answer_scope\": answer_scope,\n",
    "                    \"answer_filters\": answer_filters,\n",
    "                    \"filters_prediction\": json.loads(filters_prediction),\n",
    "                    \"variant_prediction\": json.loads(variant_prediction),\n",
    "                },\n",
    "                indent=4,\n",
    "            )\n",
    "        )\n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py312",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}