DocumentQAusing_DeepsetRobert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPlzwvStq3QdC9snnd6JSYs",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/arssite/GENAi-Assessment/blob/main/DocumentQAusing_DeepsetRobert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gradio transformers torch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VgVnjXS4UfHW",
        "outputId": "bff7ced9-7503-48f0-bd3f-4ea808a4db7e"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gradio\n",
            "  Downloading gradio-4.44.1-py3-none-any.whl.metadata (15 kB)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.4.1+cu121)\n",
            "Collecting aiofiles<24.0,>=22.0 (from gradio)\n",
            "  Downloading aiofiles-23.2.1-py3-none-any.whl.metadata (9.7 kB)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.7.1)\n",
            "Collecting fastapi<1.0 (from gradio)\n",
            "  Downloading fastapi-0.115.0-py3-none-any.whl.metadata (27 kB)\n",
            "Collecting ffmpy (from gradio)\n",
            "  Downloading ffmpy-0.4.0-py3-none-any.whl.metadata (2.9 kB)\n",
            "Collecting gradio-client==1.3.0 (from gradio)\n",
            "  Downloading gradio_client-1.3.0-py3-none-any.whl.metadata (7.1 kB)\n",
            "Collecting httpx>=0.24.1 (from gradio)\n",
            "  Downloading httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.24.7)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.4.5)\n",
            "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.1.4)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.1.5)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.7.1)\n",
            "Requirement already satisfied: numpy<3.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.26.4)\n",
            "Collecting orjson~=3.0 (from gradio)\n",
            "  Downloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m666.9 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio) (24.1)\n",
            "Requirement already satisfied: pandas<3.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.2.2)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (10.4.0)\n",
            "Requirement already satisfied: pydantic>=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.9.2)\n",
            "Collecting pydub (from gradio)\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
            "Collecting python-multipart>=0.0.9 (from gradio)\n",
            "  Downloading python_multipart-0.0.12-py3-none-any.whl.metadata (1.9 kB)\n",
            "Requirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.0.2)\n",
            "Collecting ruff>=0.2.2 (from gradio)\n",
            "  Downloading ruff-0.6.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (25 kB)\n",
            "Collecting semantic-version~=2.0 (from gradio)\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl.metadata (9.7 kB)\n",
            "Collecting tomlkit==0.12.0 (from gradio)\n",
            "  Downloading tomlkit-0.12.0-py3-none-any.whl.metadata (2.7 kB)\n",
            "Requirement already satisfied: typer<1.0,>=0.12 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.12.5)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.12.2)\n",
            "Requirement already satisfied: urllib3~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.2.3)\n",
            "Collecting uvicorn>=0.14.0 (from gradio)\n",
            "  Downloading uvicorn-0.31.0-py3-none-any.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==1.3.0->gradio) (2024.6.1)\n",
            "Collecting websockets<13.0,>=10.0 (from gradio-client==1.3.0->gradio)\n",
            "  Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.5)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5.0,>=3.0->gradio) (3.10)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5.0,>=3.0->gradio) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5.0,>=3.0->gradio) (1.2.2)\n",
            "Collecting starlette<0.39.0,>=0.37.2 (from fastapi<1.0->gradio)\n",
            "  Downloading starlette-0.38.6-py3-none-any.whl.metadata (6.0 kB)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (2024.8.30)\n",
            "Collecting httpcore==1.* (from httpx>=0.24.1->gradio)\n",
            "  Downloading httpcore-1.0.6-py3-none-any.whl.metadata (21 kB)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx>=0.24.1->gradio)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (4.54.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.4.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (3.1.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio) (2024.2)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio) (2.23.4)\n",
            "Requirement already satisfied: click>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0,>=0.12->gradio) (8.1.7)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0,>=0.12->gradio) (1.5.4)\n",
            "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0,>=0.12->gradio) (13.9.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio) (1.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (2.18.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0,>=0.12->gradio) (0.1.2)\n",
            "Downloading gradio-4.44.1-py3-none-any.whl (18.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.1/18.1 MB\u001b[0m \u001b[31m20.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gradio_client-1.3.0-py3-none-any.whl (318 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.7/318.7 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n",
            "Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
            "Downloading fastapi-0.115.0-py3-none-any.whl (94 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.6/94.6 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpcore-1.0.6-py3-none-any.whl (78 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.0/78.0 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_multipart-0.0.12-py3-none-any.whl (23 kB)\n",
            "Downloading ruff-0.6.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.9/10.9 MB\u001b[0m \u001b[31m56.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Downloading uvicorn-0.31.0-py3-none-any.whl (63 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.7/63.7 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ffmpy-0.4.0-py3-none-any.whl (5.8 kB)\n",
            "Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading starlette-0.38.6-py3-none-any.whl (71 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pydub, websockets, tomlkit, semantic-version, ruff, python-multipart, orjson, h11, ffmpy, aiofiles, uvicorn, starlette, httpcore, httpx, fastapi, gradio-client, gradio\n",
            "Successfully installed aiofiles-23.2.1 fastapi-0.115.0 ffmpy-0.4.0 gradio-4.44.1 gradio-client-1.3.0 h11-0.14.0 httpcore-1.0.6 httpx-0.27.2 orjson-3.10.7 pydub-0.25.1 python-multipart-0.0.12 ruff-0.6.9 semantic-version-2.10.0 starlette-0.38.6 tomlkit-0.12.0 uvicorn-0.31.0 websockets-12.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install python-docx fitz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hmb8G7HoXIW1",
        "outputId": "babded12-a0a1-4d7b-90db-5e4352002620"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting python-docx\n",
            "  Downloading python_docx-1.1.2-py3-none-any.whl.metadata (2.0 kB)\n",
            "Collecting fitz\n",
            "  Downloading fitz-0.0.1.dev2-py2.py3-none-any.whl.metadata (816 bytes)\n",
            "Requirement already satisfied: lxml>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from python-docx) (4.9.4)\n",
            "Requirement already satisfied: typing-extensions>=4.9.0 in /usr/local/lib/python3.10/dist-packages (from python-docx) (4.12.2)\n",
            "Collecting configobj (from fitz)\n",
            "  Downloading configobj-5.0.9.tar.gz (101 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.5/101.5 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting configparser (from fitz)\n",
            "  Downloading configparser-7.1.0-py3-none-any.whl.metadata (5.4 kB)\n",
            "Requirement already satisfied: httplib2 in /usr/local/lib/python3.10/dist-packages (from fitz) (0.22.0)\n",
            "Requirement already satisfied: nibabel in /usr/local/lib/python3.10/dist-packages (from fitz) (5.2.1)\n",
            "Collecting nipype (from fitz)\n",
            "  Downloading nipype-1.8.6-py3-none-any.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from fitz) (1.26.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from fitz) (2.2.2)\n",
            "Collecting pyxnat (from fitz)\n",
            "  Downloading pyxnat-1.6.2-py3-none-any.whl.metadata (5.3 kB)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from fitz) (1.13.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.10/dist-packages (from httplib2->fitz) (3.1.4)\n",
            "Requirement already satisfied: packaging>=17 in /usr/local/lib/python3.10/dist-packages (from nibabel->fitz) (24.1)\n",
            "Requirement already satisfied: click>=6.6.0 in /usr/local/lib/python3.10/dist-packages (from nipype->fitz) (8.1.7)\n",
            "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.10/dist-packages (from nipype->fitz) (3.3)\n",
            "Collecting prov>=1.5.2 (from nipype->fitz)\n",
            "  Downloading prov-2.0.1-py3-none-any.whl.metadata (3.6 kB)\n",
            "Requirement already satisfied: pydot>=1.2.3 in /usr/local/lib/python3.10/dist-packages (from nipype->fitz) (3.0.2)\n",
            "Requirement already satisfied: python-dateutil>=2.2 in /usr/local/lib/python3.10/dist-packages (from nipype->fitz) (2.8.2)\n",
            "Collecting rdflib>=5.0.0 (from nipype->fitz)\n",
            "  Downloading rdflib-7.0.0-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting simplejson>=3.8.0 (from nipype->fitz)\n",
            "  Downloading simplejson-3.19.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.2 kB)\n",
            "Collecting traits!=5.0,<6.4,>=4.6 (from nipype->fitz)\n",
            "  Downloading traits-6.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (4.9 kB)\n",
            "Requirement already satisfied: filelock>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from nipype->fitz) (3.16.1)\n",
            "Collecting etelemetry>=0.2.0 (from nipype->fitz)\n",
            "  Downloading etelemetry-0.3.1-py3-none-any.whl.metadata (3.2 kB)\n",
            "Collecting looseversion (from nipype->fitz)\n",
            "  Downloading looseversion-1.3.0-py2.py3-none-any.whl.metadata (4.6 kB)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->fitz) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->fitz) (2024.2)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from pyxnat->fitz) (2.32.3)\n",
            "Requirement already satisfied: pathlib>=1.0 in /usr/local/lib/python3.10/dist-packages (from pyxnat->fitz) (1.0.1)\n",
            "Collecting ci-info>=0.2 (from etelemetry>=0.2.0->nipype->fitz)\n",
            "  Downloading ci_info-0.3.0-py3-none-any.whl.metadata (6.1 kB)\n",
            "Collecting rdflib>=5.0.0 (from nipype->fitz)\n",
            "  Downloading rdflib-6.3.2-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.2->nipype->fitz) (1.16.0)\n",
            "Collecting isodate<0.7.0,>=0.6.0 (from rdflib>=5.0.0->nipype->fitz)\n",
            "  Downloading isodate-0.6.1-py2.py3-none-any.whl.metadata (9.6 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->pyxnat->fitz) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->pyxnat->fitz) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->pyxnat->fitz) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->pyxnat->fitz) (2024.8.30)\n",
            "Downloading python_docx-1.1.2-py3-none-any.whl (244 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.3/244.3 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fitz-0.0.1.dev2-py2.py3-none-any.whl (20 kB)\n",
            "Downloading configparser-7.1.0-py3-none-any.whl (17 kB)\n",
            "Downloading nipype-1.8.6-py3-none-any.whl (3.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.2/3.2 MB\u001b[0m \u001b[31m52.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyxnat-1.6.2-py3-none-any.whl (95 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.6/95.6 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading etelemetry-0.3.1-py3-none-any.whl (6.4 kB)\n",
            "Downloading prov-2.0.1-py3-none-any.whl (421 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m421.5/421.5 kB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading rdflib-6.3.2-py3-none-any.whl (528 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m528.1/528.1 kB\u001b[0m \u001b[31m30.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading simplejson-3.19.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (137 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.9/137.9 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading traits-6.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (5.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.1/5.1 MB\u001b[0m \u001b[31m86.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading looseversion-1.3.0-py2.py3-none-any.whl (8.2 kB)\n",
            "Downloading ci_info-0.3.0-py3-none-any.whl (7.8 kB)\n",
            "Downloading isodate-0.6.1-py2.py3-none-any.whl (41 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.7/41.7 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: configobj\n",
            "  Building wheel for configobj (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for configobj: filename=configobj-5.0.9-py2.py3-none-any.whl size=35614 sha256=2321aa0806000941f9a1231e83f8d33b3faf79c2ac280ac3cc5daae10e2b3d1e\n",
            "  Stored in directory: /root/.cache/pip/wheels/a1/6c/03/6c5e3cf1a6e4b9e2fc5c4409be4abc5a8268bd9c878739cb32\n",
            "Successfully built configobj\n",
            "Installing collected packages: looseversion, traits, simplejson, python-docx, isodate, configparser, configobj, ci-info, rdflib, pyxnat, etelemetry, prov, nipype, fitz\n",
            "Successfully installed ci-info-0.3.0 configobj-5.0.9 configparser-7.1.0 etelemetry-0.3.1 fitz-0.0.1.dev2 isodate-0.6.1 looseversion-1.3.0 nipype-1.8.6 prov-2.0.1 python-docx-1.1.2 pyxnat-1.6.2 rdflib-6.3.2 simplejson-3.19.3 traits-6.3.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pymupdf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gyrlashOZDtO",
        "outputId": "4b53ce70-3fed-42af-a9fc-8c1a22afed6e"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pymupdf\n",
            "  Downloading PyMuPDF-1.24.11-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (3.4 kB)\n",
            "Downloading PyMuPDF-1.24.11-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (19.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.6/19.6 MB\u001b[0m \u001b[31m40.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pymupdf\n",
            "Successfully installed pymupdf-1.24.11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        },
        "id": "GP-BUORUT02o",
        "outputId": "76e1b3a8-b099-49a5-e923-153883c8e11d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
            "\n",
            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
            "Running on public URL: https://ad9aa3cbde7afb485f.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://ad9aa3cbde7afb485f.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "import torch\n",
        "import gradio as gr\n",
        "from transformers import pipeline\n",
        "import fitz  # PyMuPDF for PDF handling\n",
        "from docx import Document  # For handling .docx files\n",
        "\n",
        "model_path = (\"../Models/models--deepset--roberta-base-squad2/snapshots\"\n",
        "              \"/cbf50ba81465d4d8676b8bab348e31835147541b\")\n",
        "\n",
        "question_answer = pipeline(\"question-answering\",\n",
        "                           model=\"deepset/roberta-base-squad2\")\n",
        "\n",
        "def read_file_content(file_obj):\n",
        "    try:\n",
        "        # Determine the file extension\n",
        "        file_extension = file_obj.name.split('.')[-1].lower()\n",
        "\n",
        "        if file_extension == 'txt':\n",
        "            # Reading text files\n",
        "            with open(file_obj.name, 'r', encoding='utf-8') as file:\n",
        "                context = file.read()\n",
        "\n",
        "        elif file_extension == 'pdf':\n",
        "            # Reading PDF files using PyMuPDF (fitz)\n",
        "            with fitz.open(file_obj.name) as pdf:\n",
        "                context = \"\"\n",
        "                for page_num in range(pdf.page_count):\n",
        "                    page = pdf.load_page(page_num)\n",
        "                    context += page.get_text()\n",
        "\n",
        "        elif file_extension == 'docx':\n",
        "            # Reading Word documents using python-docx\n",
        "            doc = Document(file_obj.name)\n",
        "            context = \"\\n\".join([para.text for para in doc.paragraphs])\n",
        "\n",
        "        else:\n",
        "            return \"Unsupported file format. Please upload a .txt, .pdf, or .docx file.\"\n",
        "\n",
        "        return context\n",
        "\n",
        "    except Exception as e:\n",
        "        return f\"An error occurred: {e}\"\n",
        "\n",
        "def get_answer(file, question):\n",
        "    context = read_file_content(file)\n",
        "    if \"An error occurred\" in context or \"Unsupported\" in context:\n",
        "        return context  # Return error message directly if present\n",
        "    answer = question_answer(question=question, context=context)\n",
        "    return answer[\"answer\"]\n",
        "\n",
        "demo = gr.Interface(\n",
        "    fn=get_answer,\n",
        "    inputs=[\n",
        "        gr.File(label=\"Upload your file\"),\n",
        "        gr.Textbox(label=\"Input your question\", lines=1)\n",
        "    ],\n",
        "    outputs=[gr.Textbox(label=\"Answer text\", lines=1)],\n",
        "    title=\"Document Q & A\",\n",
        "    description=\"THIS APPLICATION WILL BE USED TO ANSWER QUESTIONS BASED ON CONTEXT PROVIDED.\"\n",
        ")\n",
        "\n",
        "demo.launch()\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!gradio deploy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SMxQav6xUkoc",
        "outputId": "62486518-70dd-41cb-a5ac-cd21fc6685cd"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\rmnist_train_small.csv:   0% 0.00/36.5M [00:00<?, ?B/s]\u001b[A\n",
            "\n",
            "\rmnist_test.csv:   0% 0.00/18.3M [00:00<?, ?B/s]\u001b[A\u001b[A\rUpload 2 LFS files:   0% 0/2 [00:00<?, ?it/s]\n",
            "\rmnist_train_small.csv:   1% 508k/36.5M [00:00<00:07, 4.73MB/s]\u001b[A\n",
            "\n",
            "\rmnist_test.csv:   2% 344k/18.3M [00:00<00:05, 3.13MB/s]\u001b[A\u001b[A\n",
            "\rmnist_train_small.csv:  29% 10.4M/36.5M [00:00<00:00, 58.6MB/s]\u001b[A\n",
            "\n",
            "mnist_test.csv:  39% 7.08M/18.3M [00:00<00:00, 34.4MB/s]\u001b[A\u001b[A\n",
            "\n",
            "mnist_test.csv:  76% 13.9M/18.3M [00:00<00:00, 44.1MB/s]\u001b[A\u001b[A\n",
            "mnist_train_small.csv:  45% 16.5M/36.5M [00:00<00:00, 32.5MB/s]\u001b[A\n",
            "mnist_train_small.csv: 100% 36.5M/36.5M [00:01<00:00, 32.6MB/s]\n",
            "\n",
            "\n",
            "mnist_test.csv: 100% 18.3M/18.3M [00:02<00:00, 8.40MB/s]\n",
            "Upload 2 LFS files: 100% 2/2 [00:02<00:00,  1.20s/it]\n",
            "Space available at \u001b[4;94mhttps://huggingface.co/spaces/arssite/Examine_Docs\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "m0k078u6bDW9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}