Pinecone_VectorDB.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CK9A4pX8F-5c"
      },
      "outputs": [],
      "source": [
        "!pip install unstructured tiktoken pinecone-client pypdf pandas openai langchain numpy python-dotenv"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "import langchain\n",
        "import pinecone\n",
        "from langchain.document_loaders import PyPDFDirectoryLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.vectorstores import Pinecone\n",
        "from langchain.llms import OpenAI"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oBuyALZaGcEJ",
        "outputId": "a89669e5-bc5a-44d6-f960-78fb6a1123d8"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/pinecone/index.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import tqdm\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import userdata\n",
        "OPENAI_API_KEY = userdata.get('OPENAI_API_KEY')\n",
        "HF_API_KEY = userdata.get('HF_API_KEY')\n",
        "PINECONE_API_KEY = userdata.get('PINECONE_API_KEY')"
      ],
      "metadata": {
        "id": "_5tN3jTOHjQl"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY\n",
        "os.environ['HUGGINGFACEHUB_API_TOKEN'] = HF_API_KEY\n",
        "os.environ['PINECONE_API_KEY'] = PINECONE_API_KEY"
      ],
      "metadata": {
        "id": "sX_24k62Hvch"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Read the document:"
      ],
      "metadata": {
        "id": "3w0dzjdLH1hA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def read_doc(directory):\n",
        "  file_loader = PyPDFDirectoryLoader(directory)\n",
        "  documents = file_loader.load()\n",
        "  return documents"
      ],
      "metadata": {
        "id": "BGxrVdTLHzU9"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "doc = read_doc('/content/drive/MyDrive/Colab Notebooks/documents/')\n",
        "doc"
      ],
      "metadata": {
        "id": "sNKZCtEYIRpr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(doc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8tvDGJ6mJJYI",
        "outputId": "b2f5ed46-21e1-499f-bb98-69c95e624a09"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "58"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Divide document into text chunks:"
      ],
      "metadata": {
        "id": "quKRixJuJGPa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def chunk_data(docs, chunk_size=800, chunk_overlap=50):\n",
        "  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)\n",
        "  doc = text_splitter.split_documents(docs)\n",
        "  return docs"
      ],
      "metadata": {
        "id": "hI44C8k2JTQ_"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "documents = chunk_data(doc)\n",
        "documents"
      ],
      "metadata": {
        "id": "CQHmIQh7J7IB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(documents)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KuQllfPkKJS7",
        "outputId": "3e374d86-902c-4ad1-9f87-ee93599f2e65"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "58"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Embedding techniques of OpenAI:"
      ],
      "metadata": {
        "id": "9BulxU1qKQ-Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "embeddings = OpenAIEmbeddings(api_key=os.environ['OPENAI_API_KEY'])\n",
        "embeddings"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bmltn145KPHP",
        "outputId": "d5b89a9b-8ebd-4c10-cff8-a606728387b5"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "OpenAIEmbeddings(client=<openai.resources.embeddings.Embeddings object at 0x7cdbba5321a0>, async_client=<openai.resources.embeddings.AsyncEmbeddings object at 0x7cdbba4d6bf0>, model='text-embedding-ada-002', deployment='text-embedding-ada-002', openai_api_version='', openai_api_base=None, openai_api_type='', openai_proxy='', embedding_ctx_length=8191, openai_api_key='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', openai_organization=None, allowed_special=set(), disallowed_special='all', chunk_size=1000, max_retries=2, request_timeout=None, headers=None, tiktoken_model_name=None, show_progress_bar=False, model_kwargs={}, skip_empty=False, default_headers=None, default_query=None, http_client=None)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vectors = embeddings.embed_query(\"How are you?\")\n",
        "len(vectors)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ga9Q5BqaKqfb",
        "outputId": "fd38e77a-a8d1-4eaf-f9e9-04007ea6fd6f"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1536"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Vector Seach DB in Pinecone:"
      ],
      "metadata": {
        "id": "Eb4J8sEXK26j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pinecone.init(\n",
        "    api_key=os.environ['PINECONE_API_KEY'],\n",
        "    environment=\"gcp-starter\"\n",
        ")\n",
        "\n",
        "# Database name\n",
        "index_name = \"langchainvector\""
      ],
      "metadata": {
        "id": "tHj_FbDLK2l6"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Store docs in Pinecone:"
      ],
      "metadata": {
        "id": "TJ7za3DlQ0lk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# from langchain.vectorstores import Pinecone\n",
        "index = Pinecone.from_documents(doc, embeddings, index_name=index_name)"
      ],
      "metadata": {
        "id": "MNW9s1QkPtTH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Cosine similarity to retrieve results from vectorDB:"
      ],
      "metadata": {
        "id": "acVTL9QdQ4ll"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def retrieve_query(query, k=2):\n",
        "  matching_results = index.similarity_search(query, k=k)\n",
        "  return matching_results"
      ],
      "metadata": {
        "id": "FZlHpEoFQlxn"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.chains.question_answering import load_qa_chain\n",
        "from langchain import OpenAI"
      ],
      "metadata": {
        "id": "yi-NQohpRYWG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "llm = OpenAI(model_name=\"text-davinci-003\", temperature=0.3)\n",
        "chain = load_qa_chain(llm, chain_type=\"stuff\")"
      ],
      "metadata": {
        "id": "Ua2s6aZlRrvc"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# search answers froom VectorDB\n",
        "def retrieve_answers(query):\n",
        "  doc_search = retrieve_query(query)\n",
        "  print(doc_search)\n",
        "  response = chain.run(input_documents=doc_search, question=query)\n",
        "  return response"
      ],
      "metadata": {
        "id": "RhTKDj35SLbj"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "our_query = \"How much the agriculture target will be increased by how many crore?\"\n",
        "answer = retrieve_answers(our_query)\n",
        "print(answer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gn3-QBM3S9YI",
        "outputId": "a3c7e283-b5d6-48dc-b516-3adda555bb8a"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content=\"7 \\n \\n \\n farmers in contributing to the health of fellow citizens by growing these \\n‘Shree Anna’.   \\n22. Now to make India a global hub for ' Shree Anna' , the Indian Institute \\nof Millet Research, Hyderabad  will be supported as the Centre of Excellence \\nfor sharing best practices, research and technologies at the international \\nlevel.    \\nAgriculture Credit  \\n23. The agriculture credit target will be increased  \\nto ` 20 lakh crore with focus on animal husbandry, dairy and fisheries.  \\nFisheries \\n24. We will launch a new sub-scheme of PM Matsya Sampada Yojana \\nwith targeted investment of ` 6,000 crore to further enable activities of \\nfishermen, fish vendors, and micro & small enterprises, improve value chain \\nefficiencies, and expand the market. \\nCooperation \\n25. For farmers, especially small and marginal farmers, and other \\nmarginalised sections, the government is promoting cooperative-based \\neconomic development model. A new Ministry of Cooperation was formed \\nwith a mandate to realise the vision of ‘Sahakar Se Samriddhi’ . To realise \\nthis vision, the government has already initiated computerisation of 63,000 \\nPrimary Agricultural Credit Societies (PACS) with an investment of ` 2,516 \\ncrore. In consultation with all stakeholders and states, model bye-laws for \\nPACS were formulated enabling them to become multipurpose PACS. A \\nnational cooperative database is being prepared for country-wide mapping \\nof cooperative societies.  \\n26. With this backdrop, we will implement a plan to set up massive \\ndecentralised storage capacity. This will help farmers store their produce \\nand realize remunerative prices through sale at appropriate times. The \\ngovernment will also facilitate setting up of a large number of multipurpose \", metadata={'page': 10.0, 'source': '/content/drive/MyDrive/Colab Notebooks/documents/budget_speech.pdf'}), Document(page_content=\"6 \\n \\n \\n inclusive, farmer-centric solutions through relevant information services for \\ncrop planning and health, improved access to farm inputs, credit, and \\ninsurance, help for crop estimation, market intelligence, and support for \\ngrowth of agri-tech industry and start-ups.  \\nAgriculture Accelerator Fund  \\n17. An Agriculture Accelerator Fund will be set-up to encourage agri-\\nstartups by young entrepreneurs in rural areas. The Fund will aim at \\nbringing innovative and affordable solutions for challenges faced by \\nfarmers. It will also bring in modern technologies to transform agricultural \\npractices, increase productivity and profitability. \\nEnhancing productivity of cotton crop  \\n18. To enhance the productivity of extra-long staple cotton, we will \\nadopt a cluster-based and value chain approach through Public Private \\nPartnerships (PPP). This will mean collaboration between farmers, state and \\nindustry for input supplies, extension services, and market linkages. \\nAtmanirbhar Horticulture Clean Plant Program  \\n19. We will launch an Atmanirbhar Clean Plant Program to boost \\navailability of disease-free, quality planting material for high value \\nhorticultural crops at an outlay of ` 2,200 crore. \\nGlobal Hub for Millets: ‘Shree Anna’  \\n20. “India is at the forefront of popularizing Millets, whose consumption \\nfurthers nutrition, food security and welfare of farmers,” said Hon’ble Prime \\nMinister. \\n21.  We are the largest producer and second largest exporter of ‘Shree \\nAnna’  in the world. We grow several types of ' Shree Anna'  such as  jowar, \\nragi, bajra, kuttu, ramdana, kangni, kutki, kodo, cheena, and sama. These \\nhave a number of health benefits, and have been an integral part of our \\nfood for centuries. I acknowledge with pride the huge service done by small \", metadata={'page': 9.0, 'source': '/content/drive/MyDrive/Colab Notebooks/documents/budget_speech.pdf'})]\n",
            " The agriculture credit target will be increased to ` 20 lakh crore with an investment of ` 2,516 crore.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "our_query = \"How much the agriculture doing?\"\n",
        "answer2 = retrieve_answers(our_query)\n",
        "print(answer2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0kFlzc3JTmJt",
        "outputId": "dc94f4fb-faaf-4c8d-9673-8768cafe20c1"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content=\"7 \\n \\n \\n farmers in contributing to the health of fellow citizens by growing these \\n‘Shree Anna’.   \\n22. Now to make India a global hub for ' Shree Anna' , the Indian Institute \\nof Millet Research, Hyderabad  will be supported as the Centre of Excellence \\nfor sharing best practices, research and technologies at the international \\nlevel.    \\nAgriculture Credit  \\n23. The agriculture credit target will be increased  \\nto ` 20 lakh crore with focus on animal husbandry, dairy and fisheries.  \\nFisheries \\n24. We will launch a new sub-scheme of PM Matsya Sampada Yojana \\nwith targeted investment of ` 6,000 crore to further enable activities of \\nfishermen, fish vendors, and micro & small enterprises, improve value chain \\nefficiencies, and expand the market. \\nCooperation \\n25. For farmers, especially small and marginal farmers, and other \\nmarginalised sections, the government is promoting cooperative-based \\neconomic development model. A new Ministry of Cooperation was formed \\nwith a mandate to realise the vision of ‘Sahakar Se Samriddhi’ . To realise \\nthis vision, the government has already initiated computerisation of 63,000 \\nPrimary Agricultural Credit Societies (PACS) with an investment of ` 2,516 \\ncrore. In consultation with all stakeholders and states, model bye-laws for \\nPACS were formulated enabling them to become multipurpose PACS. A \\nnational cooperative database is being prepared for country-wide mapping \\nof cooperative societies.  \\n26. With this backdrop, we will implement a plan to set up massive \\ndecentralised storage capacity. This will help farmers store their produce \\nand realize remunerative prices through sale at appropriate times. The \\ngovernment will also facilitate setting up of a large number of multipurpose \", metadata={'page': 10.0, 'source': '/content/drive/MyDrive/Colab Notebooks/documents/budget_speech.pdf'}), Document(page_content=\"6 \\n \\n \\n inclusive, farmer-centric solutions through relevant information services for \\ncrop planning and health, improved access to farm inputs, credit, and \\ninsurance, help for crop estimation, market intelligence, and support for \\ngrowth of agri-tech industry and start-ups.  \\nAgriculture Accelerator Fund  \\n17. An Agriculture Accelerator Fund will be set-up to encourage agri-\\nstartups by young entrepreneurs in rural areas. The Fund will aim at \\nbringing innovative and affordable solutions for challenges faced by \\nfarmers. It will also bring in modern technologies to transform agricultural \\npractices, increase productivity and profitability. \\nEnhancing productivity of cotton crop  \\n18. To enhance the productivity of extra-long staple cotton, we will \\nadopt a cluster-based and value chain approach through Public Private \\nPartnerships (PPP). This will mean collaboration between farmers, state and \\nindustry for input supplies, extension services, and market linkages. \\nAtmanirbhar Horticulture Clean Plant Program  \\n19. We will launch an Atmanirbhar Clean Plant Program to boost \\navailability of disease-free, quality planting material for high value \\nhorticultural crops at an outlay of ` 2,200 crore. \\nGlobal Hub for Millets: ‘Shree Anna’  \\n20. “India is at the forefront of popularizing Millets, whose consumption \\nfurthers nutrition, food security and welfare of farmers,” said Hon’ble Prime \\nMinister. \\n21.  We are the largest producer and second largest exporter of ‘Shree \\nAnna’  in the world. We grow several types of ' Shree Anna'  such as  jowar, \\nragi, bajra, kuttu, ramdana, kangni, kutki, kodo, cheena, and sama. These \\nhave a number of health benefits, and have been an integral part of our \\nfood for centuries. I acknowledge with pride the huge service done by small \", metadata={'page': 9.0, 'source': '/content/drive/MyDrive/Colab Notebooks/documents/budget_speech.pdf'})]\n",
            " The government is promoting cooperative-based economic development model and has initiated computerization of 63,000 Primary Agricultural Credit Societies (PACS) with an investment of `2,516 crore. It is also launching a new sub-scheme of PM Matsya Sampada Yojana with targeted investment of `6,000 crore to further enable activities of fishermen, fish vendors, and micro & small enterprises, improve value chain efficiencies, and expand the market. The agriculture credit target will also be increased to `20 lakh crore with focus on animal husbandry, dairy and fisheries.\n"
          ]
        }
      ]
    }
  ]
}