medimagma_test_inference.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model\n",
    "\n",
    "#### Fixes\n",
    "[x] Magma currently still loads model from hugginface --> adjust in magma.py\n",
    "\n",
    "[x] Figure out how to load model trained with zero stage 3 --> three checkpoints, one per GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/vbernhard/medi-magma/magma_venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-06-20 15:12:13,420] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n",
      "Loading stanford-crfm/BioMedLM language model...\n",
      "loading magma checkpoint from: model/medimagma_firstfullmimic/global_step1000_model.bin\n",
      "magma successfully loaded\n"
     ]
    }
   ],
   "source": [
    "from magma import Magma\n",
    "import os\n",
    "\n",
    "model = Magma.from_checkpoint(\n",
    "    config_path = \"configs/MAGMA_medi_biomedlm.yml\",\n",
    "    checkpoint_path = \"model/medimagma_firstfullmimic/global_step1000_model.bin\",\n",
    "    device = 'cuda:5'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from magma.image_input import ImageInput\n",
    "# try on random iux image\n",
    "image = 'data/2440_IM-0978-2001.dcm.png'\n",
    "gold_report = \"Hyperexpanded lungs with flattened hemidiaphragms, consistent with emphysema. There is streaky airspace opacities in the left suprahilar and lingular regions. No pneumothorax or effusions. Mild bilateral costophrenic XXXX blunting XXXX represents pleural thickening and scarring. Degenerative changes of the thoracic spine.\"\n",
    "inputs =[\n",
    "    ## supports urls and path/to/image\n",
    "    ImageInput(image),\n",
    "    'Write a radiology report: '\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Sampling Progress:  42%|████▏     | 42/100 [00:01<00:01, 36.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "___, compared to previous study, there is persistent removal of the lungs are clear without focal consolidation. The cardiac and mediastinal silhouettes are stable. There are no pleural effusion with mild pulmonary edema.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "## returns a tensor of shape: (1, 149, 4096)\n",
    "embeddings = model.preprocess_inputs(inputs)  \n",
    "\n",
    "## returns a list of length embeddings.shape[0] (batch size)\n",
    "output = model.generate(\n",
    "    embeddings = embeddings,\n",
    "    max_steps = 100,\n",
    "    temperature = 0.7,\n",
    "    top_k = 0,\n",
    "    single_gpu = True,\n",
    ")  \n",
    "\n",
    "print(output[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "magma_venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}