c02_chainlit_chroma_local_llama2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 初始環境設定\n",
    "import os\n",
    "from pathlib import Path\n",
    "HOME = str(Path.home())\n",
    "Add_Binarry_Path=HOME+'/.local/bin:/usr/ubuntu_bin'\n",
    "os.environ['PATH']=os.environ['PATH']+':'+Add_Binarry_Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tas1NRCPl4QS",
    "outputId": "4ab41d7f-3600-4e8d-86a1-a4157e6d88b1"
   },
   "outputs": [],
   "source": [
    "## 安裝套件\n",
    "!pip install sentence-transformers -q\n",
    "!pip install pypdf pyngrok  openai gdown chainlit langchain chromadb pdfplumber tiktoken  cohere -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# read pdf\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "pdf_file='./data/1Q23-EPR-with-Tables-FINAL.pdf'\n",
    "loader= PyPDFLoader(pdf_file)\n",
    "documents = loader.load()\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=500, chunk_overlap=100\n",
    ")\n",
    "docs = text_splitter.split_documents(documents)\n",
    "\n",
    "for i, doc in enumerate(docs):\n",
    "    doc.metadata[\"source\"] = f\"source_{i}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# embedding\n",
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "\n",
    "Embeddings_ID=\"/work/u00cjz00/slurm_jobs/github/models/embedding/all-MiniLM-L6-v2\"\n",
    "embeddings=HuggingFaceEmbeddings(model_name=Embeddings_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# DB\n",
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "persist_directory=\"./chromadb/\"\n",
    "search_engine = Chroma.from_documents(\n",
    "    documents=docs, # text data that you want to embed and store\n",
    "    embedding=embeddings, # used to convert the documents into embeddings\n",
    "    #persist_directory=persist_directory, # this tells Chroma where to store its data\n",
    "    collection_name=\"mydb\" #  gives a name to the collection of embeddings, which will be helpful for retrieving specific groups of embeddings later.\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# 3. Download model\n",
    "mkdir -p Llama-7B-Chat-GPTQ\n",
    "HF_HUB_ENABLE_HF_TRANSFER=1 huggingface-cli download TheBloke/Llama-2-7B-Chat-GPTQ --local-dir Llama-7B-Chat-GPTQ --local-dir-use-symlinks False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# llm model\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline\n",
    "from langchain.llms.huggingface_pipeline import HuggingFacePipeline\n",
    "\n",
    "MODEL_ID = \"./Llama-7B-Chat-GPTQ\"\n",
    "model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map=\"cuda\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)\n",
    "\n",
    "pipe = pipeline(\n",
    "    \"text-generation\",\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    max_new_tokens=512,\n",
    "    do_sample=True,\n",
    "    temperature=0.7,\n",
    "    top_p=0.95,\n",
    "    top_k=40,\n",
    "    repetition_penalty=1.1\n",
    ")\n",
    "\n",
    "llm = HuggingFacePipeline(pipeline=pipe, model_kwargs={\"temperature\": 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# RAG Chain\n",
    "from langchain.chains import RetrievalQAWithSourcesChain\n",
    "from c01_prompts import EXAMPLE_PROMPT, PROMPT, WELCOME_MESSAGE\n",
    "chain = RetrievalQAWithSourcesChain.from_chain_type(\n",
    "    llm=llm,\n",
    "    chain_type=\"stuff\",\n",
    "    retriever=search_engine.as_retriever(max_tokens_limit=4097),\n",
    "    return_source_documents=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## DO QA Search\n",
    "query = \"What's the results for the reporter quarter? Please describe in the following order using bullet points - revenue, gross margin, opex, op margin, net income, and EPS. INclude both gaap and non-gaap numbers. Please also include quarter over quarter changes.\"\n",
    "llm_response = chain(query)\n",
    "llm_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## DO QA Search\n",
    "query = \"What's the results for the reporter quarter? Please describe in the following order using bullet points - revenue, gross margin, opex, op margin, net income, and EPS. INclude both gaap and non-gaap numbers. Please also include quarter over quarter changes.\"\n",
    "llm_response = chain(query)\n",
    "print(llm_response['question'].strip())\n",
    "print('\\n\\n\\n')\n",
    "print(llm_response['answer'].strip())\n",
    "print('\\n\\n\\n')\n",
    "print(llm_response['sources'].strip())\n",
    "print('\\n\\n\\n')\n",
    "print(llm_response['source_documents'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DEMO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!chainlit run c01_app.py -w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fKdRdCbhu8Nx",
    "outputId": "11bc1844-1a97-4b1d-ca0e-664dcdafb6a4"
   },
   "outputs": [],
   "source": [
    "!ngrok config add-authtoken xxxxxxxxxxxxx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9sXq3FVajO77",
    "outputId": "325b3acb-88a6-403e-b725-be4d5d8fea16"
   },
   "outputs": [],
   "source": [
    "from pyngrok import ngrok\n",
    "ngrok_tunnel = ngrok.connect(8000)\n",
    "print('Public URL:', ngrok_tunnel.public_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UjECiecOjPHv"
   },
   "outputs": [],
   "source": [
    "ngrok.kill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GfFTVIWKkhgt",
    "outputId": "a9a7c98e-231f-4dd8-c2c0-dfe80f070dc4"
   },
   "outputs": [],
   "source": [
    "!ps -ef |grep chainlit | awk '{print $2}' | xargs kill -9\n",
    "!ps -ef |grep ngrok | awk '{print $2}' | xargs kill -9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KFoxUSPGkjb1"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Image_S_work-demo01_pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen",
   "language": "python",
   "name": "s_work-demo01_pytorch_2.1.0-cuda11.8-cudnn8-runtime_textgen"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}