qa_w_rag_Ingest_Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "04db3079-bc7a-4c2a-a85c-d7e409ee53f5",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Install libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7b998b23-f203-4cf2-9703-e59260dc64ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting boto3>=1.28.57\n",
      "  Obtaining dependency information for boto3>=1.28.57 from https://files.pythonhosted.org/packages/ff/e9/8ece7607d288c1de22638a7223ae1dc41a34e8cd1511ebfc8171ac24db9d/boto3-1.28.73-py3-none-any.whl.metadata\n",
      "  Downloading boto3-1.28.73-py3-none-any.whl.metadata (6.7 kB)\n",
      "Collecting awscli>=1.29.57\n",
      "  Obtaining dependency information for awscli>=1.29.57 from https://files.pythonhosted.org/packages/f9/af/ac25b9236912890e5476b1153afdec2a0f4b95f26609db6dc541b93792a0/awscli-1.29.73-py3-none-any.whl.metadata\n",
      "  Downloading awscli-1.29.73-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting botocore>=1.31.57\n",
      "  Obtaining dependency information for botocore>=1.31.57 from https://files.pythonhosted.org/packages/fa/f3/c369381bf6b2913e201d91be68d34e1ad3dfe9163ae7e8203606d09ee07c/botocore-1.31.73-py3-none-any.whl.metadata\n",
      "  Downloading botocore-1.31.73-py3-none-any.whl.metadata (6.1 kB)\n",
      "Collecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.28.57)\n",
      "  Using cached jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
      "Collecting s3transfer<0.8.0,>=0.7.0 (from boto3>=1.28.57)\n",
      "  Obtaining dependency information for s3transfer<0.8.0,>=0.7.0 from https://files.pythonhosted.org/packages/5a/4b/fec9ce18f8874a96c5061422625ba86c3ee1e6587ccd92ff9f5bf7bd91b2/s3transfer-0.7.0-py3-none-any.whl.metadata\n",
      "  Using cached s3transfer-0.7.0-py3-none-any.whl.metadata (1.8 kB)\n",
      "Collecting docutils<0.17,>=0.10 (from awscli>=1.29.57)\n",
      "  Using cached docutils-0.16-py2.py3-none-any.whl (548 kB)\n",
      "Collecting PyYAML<6.1,>=3.10 (from awscli>=1.29.57)\n",
      "  Obtaining dependency information for PyYAML<6.1,>=3.10 from https://files.pythonhosted.org/packages/29/61/bf33c6c85c55bc45a29eee3195848ff2d518d84735eb0e2d8cb42e0d285e/PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Using cached PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.1 kB)\n",
      "Collecting colorama<0.4.5,>=0.2.5 (from awscli>=1.29.57)\n",
      "  Using cached colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
      "Collecting rsa<4.8,>=3.1.2 (from awscli>=1.29.57)\n",
      "  Using cached rsa-4.7.2-py3-none-any.whl (34 kB)\n",
      "Collecting python-dateutil<3.0.0,>=2.1 (from botocore>=1.31.57)\n",
      "  Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)\n",
      "Collecting urllib3<2.1,>=1.25.4 (from botocore>=1.31.57)\n",
      "  Obtaining dependency information for urllib3<2.1,>=1.25.4 from https://files.pythonhosted.org/packages/d2/b2/b157855192a68541a91ba7b2bbcb91f1b4faa51f8bae38d8005c034be524/urllib3-2.0.7-py3-none-any.whl.metadata\n",
      "  Using cached urllib3-2.0.7-py3-none-any.whl.metadata (6.6 kB)\n",
      "Collecting six>=1.5 (from python-dateutil<3.0.0,>=2.1->botocore>=1.31.57)\n",
      "  Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
      "Collecting pyasn1>=0.1.3 (from rsa<4.8,>=3.1.2->awscli>=1.29.57)\n",
      "  Using cached pyasn1-0.5.0-py2.py3-none-any.whl (83 kB)\n",
      "Downloading boto3-1.28.73-py3-none-any.whl (135 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.8/135.8 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading awscli-1.29.73-py3-none-any.whl (4.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.3/4.3 MB\u001b[0m \u001b[31m25.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hDownloading botocore-1.31.73-py3-none-any.whl (11.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.3/11.3 MB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hUsing cached PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (705 kB)\n",
      "Using cached s3transfer-0.7.0-py3-none-any.whl (79 kB)\n",
      "Using cached urllib3-2.0.7-py3-none-any.whl (124 kB)\n",
      "Installing collected packages: urllib3, six, PyYAML, pyasn1, jmespath, docutils, colorama, rsa, python-dateutil, botocore, s3transfer, boto3, awscli\n",
      "  Attempting uninstall: urllib3\n",
      "    Found existing installation: urllib3 1.26.18\n",
      "    Uninstalling urllib3-1.26.18:\n",
      "      Successfully uninstalled urllib3-1.26.18\n",
      "  Attempting uninstall: six\n",
      "    Found existing installation: six 1.16.0\n",
      "    Uninstalling six-1.16.0:\n",
      "      Successfully uninstalled six-1.16.0\n",
      "  Attempting uninstall: PyYAML\n",
      "    Found existing installation: PyYAML 6.0.1\n",
      "    Uninstalling PyYAML-6.0.1:\n",
      "      Successfully uninstalled PyYAML-6.0.1\n",
      "  Attempting uninstall: pyasn1\n",
      "    Found existing installation: pyasn1 0.5.0\n",
      "    Uninstalling pyasn1-0.5.0:\n",
      "      Successfully uninstalled pyasn1-0.5.0\n",
      "  Attempting uninstall: jmespath\n",
      "    Found existing installation: jmespath 1.0.1\n",
      "    Uninstalling jmespath-1.0.1:\n",
      "      Successfully uninstalled jmespath-1.0.1\n",
      "  Attempting uninstall: docutils\n",
      "    Found existing installation: docutils 0.16\n",
      "    Uninstalling docutils-0.16:\n",
      "      Successfully uninstalled docutils-0.16\n",
      "  Attempting uninstall: colorama\n",
      "    Found existing installation: colorama 0.4.4\n",
      "    Uninstalling colorama-0.4.4:\n",
      "      Successfully uninstalled colorama-0.4.4\n",
      "  Attempting uninstall: rsa\n",
      "    Found existing installation: rsa 4.7.2\n",
      "    Uninstalling rsa-4.7.2:\n",
      "      Successfully uninstalled rsa-4.7.2\n",
      "  Attempting uninstall: python-dateutil\n",
      "    Found existing installation: python-dateutil 2.8.2\n",
      "    Uninstalling python-dateutil-2.8.2:\n",
      "      Successfully uninstalled python-dateutil-2.8.2\n",
      "  Attempting uninstall: botocore\n",
      "    Found existing installation: botocore 1.31.69\n",
      "    Uninstalling botocore-1.31.69:\n",
      "      Successfully uninstalled botocore-1.31.69\n",
      "  Attempting uninstall: s3transfer\n",
      "    Found existing installation: s3transfer 0.7.0\n",
      "    Uninstalling s3transfer-0.7.0:\n",
      "      Successfully uninstalled s3transfer-0.7.0\n",
      "  Attempting uninstall: boto3\n",
      "    Found existing installation: boto3 1.28.69\n",
      "    Uninstalling boto3-1.28.69:\n",
      "      Successfully uninstalled boto3-1.28.69\n",
      "  Attempting uninstall: awscli\n",
      "    Found existing installation: awscli 1.29.69\n",
      "    Uninstalling awscli-1.29.69:\n",
      "      Successfully uninstalled awscli-1.29.69\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "spyder 5.3.3 requires pyqt5<5.16, which is not installed.\n",
      "spyder 5.3.3 requires pyqtwebengine<5.16, which is not installed.\n",
      "distributed 2022.7.0 requires tornado<6.2,>=6.0.3, but you have tornado 6.3.3 which is incompatible.\n",
      "jupyterlab 3.4.4 requires jupyter-server~=1.16, but you have jupyter-server 2.7.3 which is incompatible.\n",
      "jupyterlab-server 2.10.3 requires jupyter-server~=1.4, but you have jupyter-server 2.7.3 which is incompatible.\n",
      "notebook 6.5.6 requires jupyter-client<8,>=5.3.4, but you have jupyter-client 8.4.0 which is incompatible.\n",
      "notebook 6.5.6 requires pyzmq<25,>=17, but you have pyzmq 25.1.1 which is incompatible.\n",
      "opensearch-py 2.3.1 requires urllib3<2,>=1.21.1, but you have urllib3 2.0.7 which is incompatible.\n",
      "panel 0.13.1 requires bokeh<2.5.0,>=2.4.0, but you have bokeh 3.3.0 which is incompatible.\n",
      "pyasn1-modules 0.2.8 requires pyasn1<0.5.0,>=0.4.6, but you have pyasn1 0.5.0 which is incompatible.\n",
      "sagemaker-datawrangler 0.4.3 requires sagemaker-data-insights==0.4.0, but you have sagemaker-data-insights 0.3.3 which is incompatible.\n",
      "spyder 5.3.3 requires ipython<8.0.0,>=7.31.1, but you have ipython 8.16.1 which is incompatible.\n",
      "spyder 5.3.3 requires pylint<3.0,>=2.5.0, but you have pylint 3.0.1 which is incompatible.\n",
      "spyder-kernels 2.3.3 requires ipython<8,>=7.31.1; python_version >= \"3\", but you have ipython 8.16.1 which is incompatible.\n",
      "spyder-kernels 2.3.3 requires jupyter-client<8,>=7.3.4; python_version >= \"3\", but you have jupyter-client 8.4.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed PyYAML-6.0.1 awscli-1.29.73 boto3-1.28.73 botocore-1.31.73 colorama-0.4.4 docutils-0.16 jmespath-1.0.1 pyasn1-0.5.0 python-dateutil-2.8.2 rsa-4.7.2 s3transfer-0.7.0 six-1.16.0 urllib3-2.0.7\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: opensearch-py==2.3.1 in /opt/conda/lib/python3.10/site-packages (2.3.1)\n",
      "Requirement already satisfied: langchain==0.0.309 in /opt/conda/lib/python3.10/site-packages (0.0.309)\n",
      "Requirement already satisfied: pypdf<4,>=3.8 in /opt/conda/lib/python3.10/site-packages (3.16.4)\n",
      "Collecting pypdf<4,>=3.8\n",
      "  Obtaining dependency information for pypdf<4,>=3.8 from https://files.pythonhosted.org/packages/74/a9/5ccde1312650dd03e65350224fea85d9a430c182a01f056599cbb76f7390/pypdf-3.17.0-py3-none-any.whl.metadata\n",
      "  Downloading pypdf-3.17.0-py3-none-any.whl.metadata (7.5 kB)\n",
      "Requirement already satisfied: apache-beam in /opt/conda/lib/python3.10/site-packages (2.51.0)\n",
      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.14.6)\n",
      "Requirement already satisfied: tiktoken in /opt/conda/lib/python3.10/site-packages (0.5.1)\n",
      "Collecting urllib3<2,>=1.21.1 (from opensearch-py==2.3.1)\n",
      "  Obtaining dependency information for urllib3<2,>=1.21.1 from https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl.metadata\n",
      "  Using cached urllib3-1.26.18-py2.py3-none-any.whl.metadata (48 kB)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.4.0 in /opt/conda/lib/python3.10/site-packages (from opensearch-py==2.3.1) (2.31.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from opensearch-py==2.3.1) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.10/site-packages (from opensearch-py==2.3.1) (2.8.2)\n",
      "Requirement already satisfied: certifi>=2022.12.07 in /opt/conda/lib/python3.10/site-packages (from opensearch-py==2.3.1) (2023.7.22)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (1.4.39)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (3.8.6)\n",
      "Requirement already satisfied: anyio<4.0 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (3.5.0)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (4.0.3)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (0.6.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (1.33)\n",
      "Requirement already satisfied: langsmith<0.1.0,>=0.0.40 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (0.0.44)\n",
      "Requirement already satisfied: numpy<2,>=1 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (1.24.4)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (2.4.2)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.309) (8.2.3)\n",
      "Requirement already satisfied: crcmod<2.0,>=1.7 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (1.7)\n",
      "Requirement already satisfied: orjson<4,>=3.9.7 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (3.9.9)\n",
      "Requirement already satisfied: dill<0.3.2,>=0.3.1.1 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (0.3.1.1)\n",
      "Requirement already satisfied: cloudpickle~=2.2.1 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (2.2.1)\n",
      "Requirement already satisfied: fastavro<2,>=0.23.6 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (1.8.4)\n",
      "Requirement already satisfied: fasteners<1.0,>=0.3 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (0.19)\n",
      "Requirement already satisfied: grpcio!=1.48.0,<2,>=1.33.1 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (1.59.0)\n",
      "Requirement already satisfied: hdfs<3.0.0,>=2.1.0 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (2.7.3)\n",
      "Requirement already satisfied: httplib2<0.23.0,>=0.8 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (0.22.0)\n",
      "Requirement already satisfied: js2py<1,>=0.74 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (0.74)\n",
      "Requirement already satisfied: objsize<0.7.0,>=0.6.1 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (0.6.1)\n",
      "Requirement already satisfied: packaging>=22.0 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (23.2)\n",
      "Requirement already satisfied: pymongo<5.0.0,>=3.8.0 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (4.5.0)\n",
      "Requirement already satisfied: proto-plus<2,>=1.7.1 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (1.22.3)\n",
      "Requirement already satisfied: protobuf!=4.0.*,!=4.21.*,!=4.22.0,!=4.23.*,!=4.24.0,!=4.24.1,!=4.24.2,<4.25.0,>=3.20.3 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (4.24.4)\n",
      "Requirement already satisfied: pydot<2,>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (1.4.2)\n",
      "Requirement already satisfied: pytz>=2018.3 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (2022.1)\n",
      "Requirement already satisfied: regex>=2020.6.8 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (2022.7.9)\n",
      "Requirement already satisfied: typing-extensions>=3.7.0 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (4.8.0)\n",
      "Requirement already satisfied: zstandard<1,>=0.18.0 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (0.21.0)\n",
      "Requirement already satisfied: pyarrow<12.0.0,>=3.0.0 in /opt/conda/lib/python3.10/site-packages (from apache-beam) (11.0.0)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (1.4.4)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.64.1)\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.9)\n",
      "Requirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2023.6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.309) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.309) (2.0.4)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.309) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.309) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.309) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.309) (1.3.1)\n",
      "Requirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.10/site-packages (from anyio<4.0->langchain==0.0.309) (3.3)\n",
      "Requirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.10/site-packages (from anyio<4.0->langchain==0.0.309) (1.2.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain==0.0.309) (3.20.1)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain==0.0.309) (0.9.0)\n",
      "Requirement already satisfied: docopt in /opt/conda/lib/python3.10/site-packages (from hdfs<3.0.0,>=2.1.0->apache-beam) (0.6.2)\n",
      "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /opt/conda/lib/python3.10/site-packages (from httplib2<0.23.0,>=0.8->apache-beam) (3.0.9)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.6.0)\n",
      "Requirement already satisfied: tzlocal>=1.2 in /opt/conda/lib/python3.10/site-packages (from js2py<1,>=0.74->apache-beam) (5.1)\n",
      "Requirement already satisfied: pyjsparser>=2.5.1 in /opt/conda/lib/python3.10/site-packages (from js2py<1,>=0.74->apache-beam) (2.7.1)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.10/site-packages (from jsonpatch<2.0,>=1.33->langchain==0.0.309) (2.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from pydantic<3,>=1->langchain==0.0.309) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.10.1 in /opt/conda/lib/python3.10/site-packages (from pydantic<3,>=1->langchain==0.0.309) (2.10.1)\n",
      "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /opt/conda/lib/python3.10/site-packages (from pymongo<5.0.0,>=3.8.0->apache-beam) (2.4.2)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /opt/conda/lib/python3.10/site-packages (from SQLAlchemy<3,>=1.4->langchain==0.0.309) (1.1.1)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain==0.0.309) (0.4.3)\n",
      "Downloading pypdf-3.17.0-py3-none-any.whl (277 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m277.4/277.4 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n",
      "Installing collected packages: urllib3, pypdf\n",
      "  Attempting uninstall: urllib3\n",
      "    Found existing installation: urllib3 2.0.7\n",
      "    Uninstalling urllib3-2.0.7:\n",
      "      Successfully uninstalled urllib3-2.0.7\n",
      "  Attempting uninstall: pypdf\n",
      "    Found existing installation: pypdf 3.16.4\n",
      "    Uninstalling pypdf-3.16.4:\n",
      "      Successfully uninstalled pypdf-3.16.4\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "distributed 2022.7.0 requires tornado<6.2,>=6.0.3, but you have tornado 6.3.3 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed pypdf-3.17.0 urllib3-1.26.18\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install --no-build-isolation --force-reinstall \\\n",
    "    \"boto3>=1.28.57\" \\\n",
    "    \"awscli>=1.29.57\" \\\n",
    "    \"botocore>=1.31.57\"\n",
    "\n",
    "%pip install -U opensearch-py==2.3.1 langchain==0.0.309 \"pypdf>=3.8,<4\" \\\n",
    "    apache-beam \\\n",
    "    datasets \\\n",
    "    tiktoken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3160b4b-2fc1-40a0-aef8-af19d943b093",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6a8a6c48-d944-45b9-ba69-0d7946c4b14a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create new client\n",
      "  Using region: us-east-1\n",
      "boto3 Bedrock client successfully created!\n",
      "bedrock-runtime(https://bedrock-runtime.us-east-1.amazonaws.com)\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import json\n",
    "import os\n",
    "import sys\n",
    "\n",
    "import boto3\n",
    "\n",
    "module_path = \"..\"\n",
    "sys.path.append(os.path.abspath(module_path))\n",
    "from utils import bedrock, print_ww\n",
    "\n",
    "\n",
    "# ---- ⚠️ Un-comment and edit the below lines as needed for your AWS setup ⚠️ ----\n",
    "\n",
    "# os.environ[\"AWS_DEFAULT_REGION\"] = \"<REGION_NAME>\"  # E.g. \"us-east-1\"\n",
    "# os.environ[\"AWS_PROFILE\"] = \"<YOUR_PROFILE>\"\n",
    "# os.environ[\"BEDROCK_ASSUME_ROLE\"] = \"<YOUR_ROLE_ARN>\"  # E.g. \"arn:aws:...\"\n",
    "\n",
    "os.environ[\"AWS_DEFAULT_REGION\"] = \"us-east-1\" \n",
    "\n",
    "boto3_bedrock = bedrock.get_bedrock_client(\n",
    "    #assumed_role=os.environ.get(\"BEDROCK_ASSUME_ROLE\", None),\n",
    "    region=os.environ.get(\"AWS_DEFAULT_REGION\", None)\n",
    ")\n",
    "\n",
    "\n",
    "from langchain.embeddings import BedrockEmbeddings\n",
    "from langchain.llms.bedrock import Bedrock\n",
    "from langchain.load.dump import dumps\n",
    "\n",
    "# - create the Anthropic Model\n",
    "llm = Bedrock(\n",
    "    model_id=\"anthropic.claude-v2\", client=boto3_bedrock, model_kwargs={\"max_tokens_to_sample\": 200}\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adccde0f-f639-4774-b2d8-e12259f5857a",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8e6a255a-7f8f-4d92-9345-7b3cbca0d1d8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# DATA PREPARATION\n",
    "\n",
    "import numpy as np\n",
    "from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter\n",
    "from langchain.document_loaders import PyPDFLoader, PyPDFDirectoryLoader\n",
    "\n",
    "loader = PyPDFDirectoryLoader(\"./vndata/\")\n",
    "\n",
    "documents = loader.load()\n",
    "# - in our testing Character split works better with this PDF data set\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    # Set a really small chunk size, just to show.\n",
    "    chunk_size=2000,\n",
    "    chunk_overlap=200,\n",
    ")\n",
    "docs = text_splitter.split_documents(documents)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16a13184-6584-49de-871a-698278ec636a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b433edd7-2816-46db-ac1e-8d26e9e0f097",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average length among 3 documents loaded is 1557 characters.\n",
      "After the split we have 4 documents more than the original 3.\n",
      "Average length among 4 documents (after split) is 1165 characters.\n"
     ]
    }
   ],
   "source": [
    "avg_doc_length = lambda documents: sum([len(doc.page_content) for doc in documents]) // len(\n",
    "    documents\n",
    ")\n",
    "avg_char_count_pre = avg_doc_length(documents)\n",
    "avg_char_count_post = avg_doc_length(docs)\n",
    "print(f\"Average length among {len(documents)} documents loaded is {avg_char_count_pre} characters.\")\n",
    "print(f\"After the split we have {len(docs)} documents more than the original {len(documents)}.\")\n",
    "print(\n",
    "    f\"Average length among {len(docs)} documents (after split) is {avg_char_count_post} characters.\"\n",
    ")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a40c4886-49f0-4c6d-ad46-84659a595264",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "48da74f8-8199-4496-a3e3-f535ceebe1aa",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Opensearch set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f677b0e-a2d4-4f31-9f77-8f65fef563a9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "import time\n",
    "vector_store_name = 'llm-rag'\n",
    "index_name = \"rag-sbert\"\n",
    "\n",
    "encryption_policy_name = \"llm-rag-sp\"\n",
    "network_policy_name = \"llm-rag-np\"\n",
    "access_policy_name = 'llm-rag-ap'\n",
    "identity = boto3.client('sts').get_caller_identity()['Arn']\n",
    "\n",
    "aoss_client = boto3.client('opensearchserverless')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "075bbc64-52a8-468f-b1ad-a4c98e0d3552",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Todo: Replace your OpenSearch endpoint here: \n",
    "\n",
    "host = 'https://1n3li4pv4s7jhgykpmie.us-east-1.aoss.amazonaws.com:443'\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aacdb12f-c4dc-4457-8736-8f164415bbb2",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## SageMaker embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "670ef343-f54d-4792-b75e-15a16d34f28d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Todo: \n",
    "# Change endpoint_name to your Sagemaker sbert deployed model.\n",
    "\n",
    "## Langchain Doc for SM Embedding Class Customization\n",
    "\n",
    "from typing import Dict, List\n",
    "from langchain.embeddings import SagemakerEndpointEmbeddings\n",
    "from langchain.embeddings.sagemaker_endpoint import EmbeddingsContentHandler\n",
    "import json\n",
    "\n",
    "\n",
    "class ContentHandler(EmbeddingsContentHandler):\n",
    "    content_type = \"application/json\"\n",
    "    accepts = \"application/json\"\n",
    "\n",
    "    def transform_input(self, inputs: list[str], model_kwargs: Dict) -> bytes:\n",
    "        \"\"\"\n",
    "        Transforms the input into bytes that can be consumed by SageMaker endpoint.\n",
    "        Args:\n",
    "            inputs: List of input strings.\n",
    "            model_kwargs: Additional keyword arguments to be passed to the endpoint.\n",
    "        Returns:\n",
    "            The transformed bytes input.\n",
    "        \"\"\"\n",
    "        # Example: inference.py expects a JSON string with a \"inputs\" key:\n",
    "        input_str = json.dumps({\"inputs\": inputs, **model_kwargs})  \n",
    "        return input_str.encode(\"utf-8\")\n",
    "\n",
    "    def transform_output(self, output: bytes) -> List[List[float]]:\n",
    "        \"\"\"\n",
    "        Transforms the bytes output from the endpoint into a list of embeddings.\n",
    "        Args:\n",
    "            output: The bytes output from SageMaker endpoint.\n",
    "        Returns:\n",
    "            The transformed output - list of embeddings\n",
    "        Note:\n",
    "            The length of the outer list is the number of input strings.\n",
    "            The length of the inner lists is the embedding dimension.\n",
    "        \"\"\"\n",
    "        # Example: inference.py returns a JSON string with the list of\n",
    "        # embeddings in a \"vectors\" key:\n",
    "        response_json = json.loads(output.read().decode(\"utf-8\"))\n",
    "        return response_json[\"vectors\"]\n",
    "\n",
    "\n",
    "content_handler = ContentHandler()\n",
    "\n",
    "\n",
    "\n",
    "sbert_batch_embeddings = SagemakerEndpointEmbeddings(\n",
    "    # credentials_profile_name=\"credentials-profile-name\",\n",
    "    endpoint_name=\"huggingface-pytorch-inference-2023-10-20-04-45-11-397\", # change this to your own sbert endpoint\n",
    "    region_name=\"us-east-1\",  # change this to your sagemaker deployed sbert endpoint Region \n",
    "    content_handler=content_handler,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f64769c9-69f2-4bb9-81bd-9ee2c44e6c24",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "768\n",
      "1\n",
      "768\n"
     ]
    }
   ],
   "source": [
    "res_1 = sbert_batch_embeddings.embed_query(\"foo\")\n",
    "print(len(res_1)) # expect 768\n",
    "\n",
    "res_2 = sbert_batch_embeddings.embed_documents([\"foo\"])\n",
    "print(len(res_2) ) # expect 1\n",
    "print(len(res_2[0]) ) # expect 768"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cb177bd-671d-4a92-8707-e17eb650dae6",
   "metadata": {},
   "source": [
    "## Ingest to AOSS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a1ae0e2-1b46-4eec-ad1f-465745dd7dc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth\n",
    "from langchain.vectorstores import OpenSearchVectorSearch\n",
    "\n",
    "import langchain \n",
    "\n",
    "\n",
    "service = 'aoss'\n",
    "credentials = boto3.Session().get_credentials()\n",
    "auth = AWSV4SignerAuth(credentials, os.environ.get(\"AWS_DEFAULT_REGION\", None), service)\n",
    "\n",
    "docsearch = OpenSearchVectorSearch.from_documents(\n",
    "    docs,\n",
    "    sbert_batch_embeddings,\n",
    "    bulk_size=1000,\n",
    "    opensearch_url=host,\n",
    "    http_auth=auth,\n",
    "    timeout = 100,\n",
    "    use_ssl = True,\n",
    "    verify_certs = True,\n",
    "    connection_class = RequestsHttpConnection,\n",
    "    index_name=index_name,\n",
    "    engine=\"faiss\",\n",
    ")\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science 3.0)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-310-v1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}