talk_n_chat_pdf.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyM1Tojr9l0vVVoudVMm6078",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ItshMoh/talk_n_chat_with_pdf/blob/main/talk_n_chat_pdf.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NO2v1M4yjree"
      },
      "outputs": [],
      "source": [
        "!pip install langchain-google-genai"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_google_genai import ChatGoogleGenerativeAI\n"
      ],
      "metadata": {
        "id": "xMO9YE__j8Jb"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import google.generativeai as genai\n",
        "from google.colab import userdata\n",
        "\n",
        "gemini_api_secret_name = 'GOOGLE_API_KEY'\n",
        "\n",
        "try:\n",
        "  GOOGLE_API_KEY=userdata.get(gemini_api_secret_name)\n",
        "  genai.configure(api_key=GOOGLE_API_KEY)\n",
        "except userdata.SecretNotFoundError as e:\n",
        "   print(f'Secret not found\\n\\nThis expects you to create a secret named {gemini_api_secret_name} in Colab\\n\\nVisit https://makersuite.google.com/app/apikey to create an API key\\n\\nStore that in the secrets section on the left side of the notebook (key icon)\\n\\nName the secret {gemini_api_secret_name}')\n",
        "   raise e\n",
        "except userdata.NotebookAccessError as e:\n",
        "  print(f'You need to grant this notebook access to the {gemini_api_secret_name} secret in order for the notebook to access Gemini on your behalf.')\n",
        "  raise e\n",
        "except Exception as e:\n",
        "  # unknown error\n",
        "  print(f\"There was an unknown error. Ensure you have a secret {gemini_api_secret_name} stored in Colab and it's a valid key from https://makersuite.google.com/app/apikey\")\n",
        "  raise e"
      ],
      "metadata": {
        "id": "pkeCkjAnj-qe"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q pypdf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Uj3nRe6kVOF",
        "outputId": "dc1a8779-0cda-41a5-8069-4c007d887838"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/286.1 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/286.1 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.4/286.1 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━\u001b[0m \u001b[32m276.5/286.1 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m286.1/286.1 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q langchain"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yxqUt3pYkYTG",
        "outputId": "7cd9e376-edb0-4822-bc15-a9f14c3e4cf6"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m812.8/812.8 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m21.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q pypdf2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "15_w5AVLkbep",
        "outputId": "5b789737-5545-4afa-d455-03b5fdbaac7d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/232.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m30.7/232.6 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━\u001b[0m \u001b[32m163.8/232.6 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m232.6/232.6 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import PyPDF2\n",
        "\n",
        "def pdf_to_string(pdf_path):\n",
        "    with open(pdf_path, 'rb') as pdf_file:\n",
        "        reader = PyPDF2.PdfReader(pdf_file)\n",
        "        text = ''\n",
        "        for page_num in range(len(reader.pages)):\n",
        "            page = reader.pages[page_num]\n",
        "            text += page.extract_text()\n",
        "        return text\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "nSKpRyKjkeK9"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_text_splitters import RecursiveCharacterTextSplitter"
      ],
      "metadata": {
        "id": "Xk0hraEyknGd"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "\n",
        "    chunk_size=1000,\n",
        "    chunk_overlap=20,\n",
        "    length_function=len,\n",
        "    is_separator_regex=False,\n",
        ")"
      ],
      "metadata": {
        "id": "gwkaGZIAkpjN"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_google_genai import GoogleGenerativeAIEmbeddings"
      ],
      "metadata": {
        "id": "skix9qKUk0ZG"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model1 = \"models/embedding-001\"\n",
        "\n",
        "# Create the embeddings object with the model argument\n",
        "embeddings = GoogleGenerativeAIEmbeddings(\n",
        "    google_api_key=\"GOOGLE_Api_key\",\n",
        "    model=model1,\n",
        "\n",
        ")\n"
      ],
      "metadata": {
        "id": "ZdEds7RFk9do"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q faiss-cpu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YMS16z8ilCgx",
        "outputId": "46bc1876-8be0-4ad5-91d5-65291caf2307"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.0/27.0 MB\u001b[0m \u001b[31m41.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_community.vectorstores import FAISS"
      ],
      "metadata": {
        "id": "v60iGRQclRke"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import getpass\n",
        "import os\n",
        "\n",
        "if \"GOOGLE_API_KEY\" not in os.environ:\n",
        "    os.environ[\"GOOGLE_API_KEY\"] = getpass.getpass(\"API_key\")\n",
        ""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z3cQ4zNIlVgb",
        "outputId": "bbc7a3b1-d899-4a56-fee9-4246579d8f2e"
      },
      "execution_count": 17,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "API_key··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "llm = ChatGoogleGenerativeAI(model=\"gemini-pro\",convert_system_message_to_human=True)"
      ],
      "metadata": {
        "id": "zllzleZvlbsa"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
        "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
        "\n",
        "SYSTEM_TEMPLATE = \"\"\"\n",
        "Answer the user's questions based on the below context.\n",
        "If the context doesn't contain any relevant information to the question, don't make something up and just say \"I don't know\":\n",
        "\n",
        "<context>\n",
        "{context}\n",
        "</context>\n",
        "\"\"\"\n",
        "\n",
        "question_answering_prompt = ChatPromptTemplate.from_messages(\n",
        "    [\n",
        "        (\n",
        "            \"system\",\n",
        "            SYSTEM_TEMPLATE,\n",
        "        ),\n",
        "        MessagesPlaceholder(variable_name=\"messages\"),\n",
        "    ]\n",
        ")\n",
        "\n",
        "document_chain = create_stuff_documents_chain(llm, question_answering_prompt)"
      ],
      "metadata": {
        "id": "kmdYC3tYle9k"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_core.messages import HumanMessage\n",
        "\n",
        "def invoke_document_chain(context, content):\n",
        "    result = document_chain.invoke(\n",
        "        {\n",
        "            \"context\": context,\n",
        "            \"messages\": [\n",
        "                HumanMessage(content=content)\n",
        "            ],\n",
        "        }\n",
        "    )\n",
        "    return result\n",
        "\n"
      ],
      "metadata": {
        "id": "hM6roeu_liZh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q git+https://github.com/openai/whisper.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oaUeVrtvlqXh",
        "outputId": "49fb735a-fcec-402e-ec82-54b5201c489f"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m48.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m63.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m59.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for openai-whisper (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q gradio\n",
        "!pip install -q gTTS"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8HbgZwSqltD4",
        "outputId": "0ec3c193-2252-4a49-a645-04503e4f5c3b"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.1/17.1 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.1/92.1 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m312.4/312.4 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.7/8.7 MB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import whisper\n",
        "import gradio as gr\n",
        "import time\n",
        "import warnings\n",
        "import os\n",
        "from gtts import gTTS"
      ],
      "metadata": {
        "id": "u_zGKaEzlu8t"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "idE_0dl7lxO4"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "from gtts import gTTS"
      ],
      "metadata": {
        "id": "aM5JN_LRlzOZ"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch"
      ],
      "metadata": {
        "id": "FlvnsB3yl08M"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.is_available()\n",
        "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "print(f\"Using torch {torch.__version__} ({DEVICE})\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jkxho-mDl2k_",
        "outputId": "62324afb-6699-4d0d-9270-2a33b1bd9387"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using torch 2.2.1+cu121 (cuda)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np"
      ],
      "metadata": {
        "id": "HP_vVAJXl4j_"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import whisper\n",
        "model = whisper.load_model(\"medium\", device=DEVICE)\n",
        "print(\n",
        "    f\"Model is {'multilingual' if model.is_multilingual else 'English-only'} \"\n",
        "    f\"and has {sum(np.prod(p.shape) for p in model.parameters()):,} parameters.\"\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zP9O7MQJl7RP",
        "outputId": "2795a142-974e-46f4-f123-1fb382830710"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|█████████████████████████████████████| 1.42G/1.42G [00:15<00:00, 96.1MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model is multilingual and has 762,321,920 parameters.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re"
      ],
      "metadata": {
        "id": "23pGLSlsl9rs"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def transcribe(audio):\n",
        "\n",
        "    # Check if the audio input is None or empty\n",
        "    if audio is None or audio == '':\n",
        "        return ('','',None)  # Return empty strings and None audio file\n",
        "\n",
        "    # language = 'en'\n",
        "\n",
        "    audio = whisper.load_audio(audio)\n",
        "    audio = whisper.pad_or_trim(audio)\n",
        "\n",
        "    mel = whisper.log_mel_spectrogram(audio).to(model.device)\n",
        "\n",
        "    _, probs = model.detect_language(mel)\n",
        "\n",
        "    options = whisper.DecodingOptions()\n",
        "    result = whisper.decode(model, mel, options)\n",
        "    result_text = result.text\n",
        "\n",
        "    return result_text"
      ],
      "metadata": {
        "id": "xQbTobc0l_eu"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def text_to_speech(text, file_path):\n",
        "    language = 'en'\n",
        "\n",
        "    audioobj = gTTS(text = text,\n",
        "                    lang = language,\n",
        "                    slow = False)\n",
        "\n",
        "    audioobj.save(file_path)\n",
        "\n",
        "    return file_path"
      ],
      "metadata": {
        "id": "QPn6lSO9mDcb"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 10 -q:a 9 -acodec libmp3lame Temp.mp3"
      ],
      "metadata": {
        "id": "mVgqPATcmGu_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def processed_pdf(pdf_path,audio):\n",
        "  pdf_text= pdf_to_string(pdf_path)\n",
        "  texts=text_splitter.create_documents([pdf_text])\n",
        "  db = FAISS.from_documents(texts,embeddings)\n",
        "  query=transcribe(audio)\n",
        "  docs=db.similarity_search(query)\n",
        "  result=invoke_document_chain(docs,query)\n",
        "\n",
        "  return result\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "XVO8ULZsmJeB"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import gradio as gr\n",
        "import base64\n",
        "import os\n",
        "\n",
        "# A function to handle audio and pdf inputs\n",
        "def process_inputs(pdf_path,audio_path):\n",
        "    # Process the audio file (assuming this is handled by a function called 'transcribe')\n",
        "    speech_to_text_output = transcribe(audio_path)\n",
        "\n",
        "    # Handle the image input\n",
        "    if pdf_path:\n",
        "        model_answer = processed_pdf(pdf_path, audio_path)\n",
        "    else:\n",
        "        model_answer = \"No audio provided.\"\n",
        "\n",
        "    # Assuming 'transcribe' also returns the path to a processed audio file\n",
        "    processed_audio_path = text_to_speech(model_answer, \"Temp3.mp3\")  # Replace with actual path if different\n",
        "\n",
        "    return speech_to_text_output, model_answer,processed_audio_path\n",
        "\n",
        "# Create the interface\n",
        "iface = gr.Interface(\n",
        "    fn=process_inputs,\n",
        "    inputs=[\n",
        "        gr.File(type=\"filepath\"),\n",
        "        gr.Audio(sources=[\"microphone\"], type=\"filepath\")\n",
        "\n",
        "    ],\n",
        "    outputs=[\n",
        "        gr.Textbox(label=\"Speech to Text\"),\n",
        "        gr.Textbox(label=\"Model Answer\"),\n",
        "        gr.Audio(\"Temp.mp3\")\n",
        "    ],\n",
        "    title=\"talk_n_chat_with_pdf\",\n",
        "    description=\"Upload a pdf and ask the model some question via voice.\"\n",
        ")\n",
        "\n",
        "# Launch the interface\n",
        "iface.launch(debug=\n",
        "             True)"
      ],
      "metadata": {
        "id": "4nQD9rRXmP5d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "1MKLVbV4nI9s"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}