LaVague.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "57cUBU_kW8YI"
      },
      "source": [
        "# LaVague"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tvfCFTct28SD"
      },
      "source": [
        "## Installing pre-requisites"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7slo7I6RpeOg"
      },
      "source": [
        "### Selenium headless Chrome driver\n",
        "\n",
        "We will pilot a Selenium browser and have to install the headless version to be run in this notebook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xL3v_HV7UIRi",
        "outputId": "8eeef932-5f37-42bc-fd53-29ef1fa233d0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Update the repository and any packages...\n",
            "Get:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Get:3 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]\n",
            "Hit:4 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]\n",
            "Hit:6 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n",
            "Hit:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Hit:8 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
            "Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,951 kB]\n",
            "Hit:11 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,349 kB]\n",
            "Get:13 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,561 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,076 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,994 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,843 kB]\n",
            "Fetched 10.0 MB in 2s (4,392 kB/s)\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "40 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Calculating upgrade... Done\n",
            "The following packages were automatically installed and are no longer required:\n",
            "  linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic\n",
            "Use 'sudo apt autoremove' to remove them.\n",
            "The following NEW packages will be installed:\n",
            "  linux-headers-5.15.0-100 linux-headers-5.15.0-100-generic\n",
            "The following packages have been kept back:\n",
            "  libcudnn8 libcudnn8-dev libnccl-dev libnccl2\n",
            "The following packages will be upgraded:\n",
            "  base-files binutils binutils-common binutils-x86-64-linux-gnu coreutils\n",
            "  cuda-compat-12-2 cuda-keyring cuda-toolkit-12-config-common\n",
            "  cuda-toolkit-config-common dpkg dpkg-dev libbinutils libc-bin libctf-nobfd0\n",
            "  libctf0 libcups2 libdpkg-perl libgnutls30 libldap-2.5-0 libpam-modules\n",
            "  libpam-modules-bin libpam-runtime libpam0g libperl5.34 libprocps8 libudev1\n",
            "  linux-headers-generic linux-libc-dev login openssl passwd perl perl-base\n",
            "  perl-modules-5.34 procps tar\n",
            "36 upgraded, 2 newly installed, 0 to remove and 4 not upgraded.\n",
            "Need to get 72.8 MB of archives.\n",
            "After this operation, 103 MB of additional disk space will be used.\n",
            "Get:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-compat-12-2 535.161.07-1 [33.6 MB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 base-files amd64 12ubuntu4.6 [62.5 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 coreutils amd64 8.32-4.1ubuntu1.1 [1,436 kB]\n",
            "Get:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-keyring 1.1-1 [4,332 B]\n",
            "Get:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-toolkit-12-config-common 12.4.99-1 [16.5 kB]\n",
            "Get:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-toolkit-config-common 12.4.99-1 [16.5 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 tar amd64 1.34+dfsg-1ubuntu0.1.22.04.2 [295 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dpkg amd64 1.21.1ubuntu2.3 [1,239 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 login amd64 1:4.8.1-2ubuntu2.2 [188 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libperl5.34 amd64 5.34.0-3ubuntu1.3 [4,820 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 perl amd64 5.34.0-3ubuntu1.3 [232 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 perl-base amd64 5.34.0-3ubuntu1.3 [1,762 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 perl-modules-5.34 all 5.34.0-3ubuntu1.3 [2,976 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libc-bin amd64 2.35-0ubuntu3.6 [706 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam0g amd64 1.4.0-11ubuntu2.4 [60.2 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam-modules-bin amd64 1.4.0-11ubuntu2.4 [37.6 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam-modules amd64 1.4.0-11ubuntu2.4 [280 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam-runtime all 1.4.0-11ubuntu2.4 [40.3 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libudev1 amd64 249.11-0ubuntu3.12 [78.2 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 passwd amd64 1:4.8.1-2ubuntu2.2 [768 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgnutls30 amd64 3.7.3-4ubuntu1.4 [969 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libprocps8 amd64 2:3.3.17-6ubuntu2.1 [36.2 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 procps amd64 2:3.3.17-6ubuntu2.1 [378 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 openssl amd64 3.0.2-0ubuntu1.15 [1,186 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libctf0 amd64 2.38-4ubuntu2.6 [103 kB]\n",
            "Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libctf-nobfd0 amd64 2.38-4ubuntu2.6 [108 kB]\n",
            "Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.38-4ubuntu2.6 [2,326 kB]\n",
            "Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libbinutils amd64 2.38-4ubuntu2.6 [662 kB]\n",
            "Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils amd64 2.38-4ubuntu2.6 [3,200 B]\n",
            "Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils-common amd64 2.38-4ubuntu2.6 [222 kB]\n",
            "Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dpkg-dev all 1.21.1ubuntu2.3 [922 kB]\n",
            "Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libdpkg-perl all 1.21.1ubuntu2.3 [237 kB]\n",
            "Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libcups2 amd64 2.4.1op1-1ubuntu4.8 [264 kB]\n",
            "Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libldap-2.5-0 amd64 2.5.16+dfsg-0ubuntu0.22.04.2 [183 kB]\n",
            "Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-100 all 5.15.0-100.110 [12.3 MB]\n",
            "Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-100-generic amd64 5.15.0-100.110 [2,858 kB]\n",
            "Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.100.97 [2,292 B]\n",
            "Get:38 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-libc-dev amd64 5.15.0-100.110 [1,348 kB]\n",
            "Fetched 72.8 MB in 2s (43.6 MB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 38.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../base-files_12ubuntu4.6_amd64.deb ...\n",
            "Unpacking base-files (12ubuntu4.6) over (12ubuntu4.4) ...\n",
            "Setting up base-files (12ubuntu4.6) ...\n",
            "Installing new version of config file /etc/issue ...\n",
            "Installing new version of config file /etc/issue.net ...\n",
            "Installing new version of config file /etc/lsb-release ...\n",
            "Installing new version of config file /etc/update-motd.d/10-help-text ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../coreutils_8.32-4.1ubuntu1.1_amd64.deb ...\n",
            "Unpacking coreutils (8.32-4.1ubuntu1.1) over (8.32-4.1ubuntu1) ...\n",
            "Setting up coreutils (8.32-4.1ubuntu1.1) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../tar_1.34+dfsg-1ubuntu0.1.22.04.2_amd64.deb ...\n",
            "Unpacking tar (1.34+dfsg-1ubuntu0.1.22.04.2) over (1.34+dfsg-1ubuntu0.1.22.04.1) ...\n",
            "Setting up tar (1.34+dfsg-1ubuntu0.1.22.04.2) ...\n",
            "update-alternatives: warning: forcing reinstallation of alternative /usr/sbin/rmt-tar because link group rmt is broken\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../dpkg_1.21.1ubuntu2.3_amd64.deb ...\n",
            "Unpacking dpkg (1.21.1ubuntu2.3) over (1.21.1ubuntu2.2) ...\n",
            "Setting up dpkg (1.21.1ubuntu2.3) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../login_1%3a4.8.1-2ubuntu2.2_amd64.deb ...\n",
            "Unpacking login (1:4.8.1-2ubuntu2.2) over (1:4.8.1-2ubuntu2.1) ...\n",
            "Setting up login (1:4.8.1-2ubuntu2.2) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../libperl5.34_5.34.0-3ubuntu1.3_amd64.deb ...\n",
            "Unpacking libperl5.34:amd64 (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Preparing to unpack .../perl_5.34.0-3ubuntu1.3_amd64.deb ...\n",
            "Unpacking perl (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Preparing to unpack .../perl-base_5.34.0-3ubuntu1.3_amd64.deb ...\n",
            "Unpacking perl-base (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Setting up perl-base (5.34.0-3ubuntu1.3) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../perl-modules-5.34_5.34.0-3ubuntu1.3_all.deb ...\n",
            "Unpacking perl-modules-5.34 (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Preparing to unpack .../libc-bin_2.35-0ubuntu3.6_amd64.deb ...\n",
            "Unpacking libc-bin (2.35-0ubuntu3.6) over (2.35-0ubuntu3.4) ...\n",
            "Setting up libc-bin (2.35-0ubuntu3.6) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam0g_1.4.0-11ubuntu2.4_amd64.deb ...\n",
            "Unpacking libpam0g:amd64 (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam0g:amd64 (1.4.0-11ubuntu2.4) ...\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78.)\n",
            "debconf: falling back to frontend: Readline\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam-modules-bin_1.4.0-11ubuntu2.4_amd64.deb ...\n",
            "Unpacking libpam-modules-bin (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam-modules-bin (1.4.0-11ubuntu2.4) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam-modules_1.4.0-11ubuntu2.4_amd64.deb ...\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78.)\n",
            "debconf: falling back to frontend: Readline\n",
            "Unpacking libpam-modules:amd64 (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam-modules:amd64 (1.4.0-11ubuntu2.4) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam-runtime_1.4.0-11ubuntu2.4_all.deb ...\n",
            "Unpacking libpam-runtime (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam-runtime (1.4.0-11ubuntu2.4) ...\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78.)\n",
            "debconf: falling back to frontend: Readline\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../libudev1_249.11-0ubuntu3.12_amd64.deb ...\n",
            "Unpacking libudev1:amd64 (249.11-0ubuntu3.12) over (249.11-0ubuntu3.10) ...\n",
            "Setting up libudev1:amd64 (249.11-0ubuntu3.12) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../passwd_1%3a4.8.1-2ubuntu2.2_amd64.deb ...\n",
            "Unpacking passwd (1:4.8.1-2ubuntu2.2) over (1:4.8.1-2ubuntu2.1) ...\n",
            "Setting up passwd (1:4.8.1-2ubuntu2.2) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../libgnutls30_3.7.3-4ubuntu1.4_amd64.deb ...\n",
            "Unpacking libgnutls30:amd64 (3.7.3-4ubuntu1.4) over (3.7.3-4ubuntu1.2) ...\n",
            "Setting up libgnutls30:amd64 (3.7.3-4ubuntu1.4) ...\n",
            "(Reading database ... 121749 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libprocps8_2%3a3.3.17-6ubuntu2.1_amd64.deb ...\n",
            "Unpacking libprocps8:amd64 (2:3.3.17-6ubuntu2.1) over (2:3.3.17-6ubuntu2) ...\n",
            "Preparing to unpack .../01-procps_2%3a3.3.17-6ubuntu2.1_amd64.deb ...\n",
            "Unpacking procps (2:3.3.17-6ubuntu2.1) over (2:3.3.17-6ubuntu2) ...\n",
            "Preparing to unpack .../02-openssl_3.0.2-0ubuntu1.15_amd64.deb ...\n",
            "Unpacking openssl (3.0.2-0ubuntu1.15) over (3.0.2-0ubuntu1.12) ...\n",
            "Preparing to unpack .../03-libctf0_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking libctf0:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../04-libctf-nobfd0_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking libctf-nobfd0:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../05-binutils-x86-64-linux-gnu_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking binutils-x86-64-linux-gnu (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../06-libbinutils_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking libbinutils:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../07-binutils_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking binutils (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../08-binutils-common_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking binutils-common:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../09-cuda-compat-12-2_535.161.07-1_amd64.deb ...\n",
            "Unpacking cuda-compat-12-2 (535.161.07-1) over (535.129.03-1) ...\n",
            "Preparing to unpack .../10-cuda-keyring_1.1-1_all.deb ...\n",
            "Unpacking cuda-keyring (1.1-1) over (1.0-1) ...\n",
            "Preparing to unpack .../11-cuda-toolkit-12-config-common_12.4.99-1_all.deb ...\n",
            "Unpacking cuda-toolkit-12-config-common (12.4.99-1) over (12.3.52-1) ...\n",
            "Preparing to unpack .../12-cuda-toolkit-config-common_12.4.99-1_all.deb ...\n",
            "Unpacking cuda-toolkit-config-common (12.4.99-1) over (12.3.52-1) ...\n",
            "Preparing to unpack .../13-dpkg-dev_1.21.1ubuntu2.3_all.deb ...\n",
            "Unpacking dpkg-dev (1.21.1ubuntu2.3) over (1.21.1ubuntu2.2) ...\n",
            "Preparing to unpack .../14-libdpkg-perl_1.21.1ubuntu2.3_all.deb ...\n",
            "Unpacking libdpkg-perl (1.21.1ubuntu2.3) over (1.21.1ubuntu2.2) ...\n",
            "Preparing to unpack .../15-libcups2_2.4.1op1-1ubuntu4.8_amd64.deb ...\n",
            "Unpacking libcups2:amd64 (2.4.1op1-1ubuntu4.8) over (2.4.1op1-1ubuntu4.7) ...\n",
            "Preparing to unpack .../16-libldap-2.5-0_2.5.16+dfsg-0ubuntu0.22.04.2_amd64.deb ...\n",
            "Unpacking libldap-2.5-0:amd64 (2.5.16+dfsg-0ubuntu0.22.04.2) over (2.5.16+dfsg-0ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package linux-headers-5.15.0-100.\n",
            "Preparing to unpack .../17-linux-headers-5.15.0-100_5.15.0-100.110_all.deb ...\n",
            "Unpacking linux-headers-5.15.0-100 (5.15.0-100.110) ...\n",
            "Selecting previously unselected package linux-headers-5.15.0-100-generic.\n",
            "Preparing to unpack .../18-linux-headers-5.15.0-100-generic_5.15.0-100.110_amd64.deb ...\n",
            "Unpacking linux-headers-5.15.0-100-generic (5.15.0-100.110) ...\n",
            "Preparing to unpack .../19-linux-headers-generic_5.15.0.100.97_amd64.deb ...\n",
            "Unpacking linux-headers-generic (5.15.0.100.97) over (5.15.0.97.92) ...\n",
            "Preparing to unpack .../20-linux-libc-dev_5.15.0-100.110_amd64.deb ...\n",
            "Unpacking linux-libc-dev:amd64 (5.15.0-100.110) over (5.15.0-88.98) ...\n",
            "Setting up cuda-toolkit-config-common (12.4.99-1) ...\n",
            "Setting up cuda-compat-12-2 (535.161.07-1) ...\n",
            "Setting up binutils-common:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up linux-libc-dev:amd64 (5.15.0-100.110) ...\n",
            "Setting up libctf-nobfd0:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up perl-modules-5.34 (5.34.0-3ubuntu1.3) ...\n",
            "Setting up linux-headers-5.15.0-100 (5.15.0-100.110) ...\n",
            "Setting up libldap-2.5-0:amd64 (2.5.16+dfsg-0ubuntu0.22.04.2) ...\n",
            "Setting up linux-headers-5.15.0-100-generic (5.15.0-100.110) ...\n",
            "Setting up cuda-keyring (1.1-1) ...\n",
            "Setting up libcups2:amd64 (2.4.1op1-1ubuntu4.8) ...\n",
            "Setting up libbinutils:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up openssl (3.0.2-0ubuntu1.15) ...\n",
            "Setting up cuda-toolkit-12-config-common (12.4.99-1) ...\n",
            "Setting up libprocps8:amd64 (2:3.3.17-6ubuntu2.1) ...\n",
            "Setting up libctf0:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up libperl5.34:amd64 (5.34.0-3ubuntu1.3) ...\n",
            "Setting up linux-headers-generic (5.15.0.100.97) ...\n",
            "Setting up perl (5.34.0-3ubuntu1.3) ...\n",
            "Setting up libdpkg-perl (1.21.1ubuntu2.3) ...\n",
            "Setting up procps (2:3.3.17-6ubuntu2.1) ...\n",
            "Setting up binutils-x86-64-linux-gnu (2.38-4ubuntu2.6) ...\n",
            "Setting up binutils (2.38-4ubuntu2.6) ...\n",
            "Setting up dpkg-dev (1.21.1ubuntu2.3) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.6) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "Install prerequisite system packages...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "wget is already the newest version (1.21.2-2ubuntu1).\n",
            "curl is already the newest version (7.81.0-1ubuntu1.15).\n",
            "unzip is already the newest version (6.0-26ubuntu3.2).\n",
            "The following packages were automatically installed and are no longer required:\n",
            "  linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic\n",
            "Use 'sudo apt autoremove' to remove them.\n",
            "The following additional packages will be installed:\n",
            "  libjq1 libonig5\n",
            "The following NEW packages will be installed:\n",
            "  jq libjq1 libonig5\n",
            "0 upgraded, 3 newly installed, 0 to remove and 4 not upgraded.\n",
            "Need to get 357 kB of archives.\n",
            "After this operation, 1,087 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB]\n",
            "Fetched 357 kB in 1s (290 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 3.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libonig5:amd64.\n",
            "(Reading database ... 150014 files and directories currently installed.)\n",
            "Preparing to unpack .../libonig5_6.9.7.1-2build1_amd64.deb ...\n",
            "Unpacking libonig5:amd64 (6.9.7.1-2build1) ...\n",
            "Selecting previously unselected package libjq1:amd64.\n",
            "Preparing to unpack .../libjq1_1.6-2.1ubuntu3_amd64.deb ...\n",
            "Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ...\n",
            "Selecting previously unselected package jq.\n",
            "Preparing to unpack .../jq_1.6-2.1ubuntu3_amd64.deb ...\n",
            "Unpacking jq (1.6-2.1ubuntu3) ...\n",
            "Setting up libonig5:amd64 (6.9.7.1-2build1) ...\n",
            "Setting up libjq1:amd64 (1.6-2.1ubuntu3) ...\n",
            "Setting up jq (1.6-2.1ubuntu3) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.6) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "--2024-03-10 12:15:09--  https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.94/linux64/chrome-linux64.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.197.207, 74.125.135.207, 173.194.203.207, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.197.207|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 149157879 (142M) [application/zip]\n",
            "Saving to: ‘chrome-linux64.zip’\n",
            "\n",
            "chrome-linux64.zip  100%[===================>] 142.25M   247MB/s    in 0.6s    \n",
            "\n",
            "2024-03-10 12:15:10 (247 MB/s) - ‘chrome-linux64.zip’ saved [149157879/149157879]\n",
            "\n",
            "Install Chrome dependencies...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Note, selecting 'libgcc-s1' instead of 'libgcc1'\n",
            "fonts-liberation is already the newest version (1:1.07.4-11).\n",
            "libasound2 is already the newest version (1.2.6.1-1ubuntu1).\n",
            "libasound2 set to manually installed.\n",
            "libatk-bridge2.0-0 is already the newest version (2.38.0-3).\n",
            "libatk-bridge2.0-0 set to manually installed.\n",
            "libatk1.0-0 is already the newest version (2.36.0-3build1).\n",
            "libatk1.0-0 set to manually installed.\n",
            "libcairo2 is already the newest version (1.16.0-5ubuntu2).\n",
            "libcairo2 set to manually installed.\n",
            "libfontconfig1 is already the newest version (2.13.1-4.2ubuntu5).\n",
            "libfontconfig1 set to manually installed.\n",
            "libnspr4 is already the newest version (2:4.32-3build1).\n",
            "libnspr4 set to manually installed.\n",
            "libxcb1 is already the newest version (1.14-3ubuntu3).\n",
            "libxcb1 set to manually installed.\n",
            "libxcomposite1 is already the newest version (1:0.4.5-1build2).\n",
            "libxcomposite1 set to manually installed.\n",
            "libxcursor1 is already the newest version (1:1.2.0-2build4).\n",
            "libxcursor1 set to manually installed.\n",
            "libxdamage1 is already the newest version (1:1.1.5-2build2).\n",
            "libxdamage1 set to manually installed.\n",
            "libxext6 is already the newest version (2:1.3.4-1build1).\n",
            "libxfixes3 is already the newest version (1:6.0.0-1).\n",
            "libxfixes3 set to manually installed.\n",
            "libxi6 is already the newest version (2:1.8-1build1).\n",
            "libxi6 set to manually installed.\n",
            "libxrandr2 is already the newest version (2:1.5.2-1build1).\n",
            "libxrandr2 set to manually installed.\n",
            "libxrender1 is already the newest version (1:0.9.10-1build4).\n",
            "libxrender1 set to manually installed.\n",
            "libxss1 is already the newest version (1:1.2.3-1build2).\n",
            "libxss1 set to manually installed.\n",
            "lsb-release is already the newest version (11.1.0ubuntu4).\n",
            "lsb-release set to manually installed.\n",
            "wget is already the newest version (1.21.2-2ubuntu1).\n",
            "ca-certificates is already the newest version (20230311ubuntu0.22.04.1).\n",
            "libc6 is already the newest version (2.35-0ubuntu3.6).\n",
            "libcups2 is already the newest version (2.4.1op1-1ubuntu4.8).\n",
            "libcups2 set to manually installed.\n",
            "libdbus-1-3 is already the newest version (1.12.20-2ubuntu4.1).\n",
            "libdbus-1-3 set to manually installed.\n",
            "libexpat1 is already the newest version (2.4.7-1ubuntu0.2).\n",
            "libexpat1 set to manually installed.\n",
            "libgbm1 is already the newest version (23.2.1-1ubuntu3.1~22.04.2).\n",
            "libgbm1 set to manually installed.\n",
            "libgcc-s1 is already the newest version (12.3.0-1ubuntu1~22.04).\n",
            "libglib2.0-0 is already the newest version (2.72.4-0ubuntu2.2).\n",
            "libglib2.0-0 set to manually installed.\n",
            "libgtk-3-0 is already the newest version (3.24.33-1ubuntu2).\n",
            "libgtk-3-0 set to manually installed.\n",
            "libnss3 is already the newest version (2:3.68.2-0ubuntu1.2).\n",
            "libnss3 set to manually installed.\n",
            "libpango-1.0-0 is already the newest version (1.50.6+ds-2ubuntu1).\n",
            "libpango-1.0-0 set to manually installed.\n",
            "libpangocairo-1.0-0 is already the newest version (1.50.6+ds-2ubuntu1).\n",
            "libpangocairo-1.0-0 set to manually installed.\n",
            "libstdc++6 is already the newest version (12.3.0-1ubuntu1~22.04).\n",
            "libx11-6 is already the newest version (2:1.7.5-1ubuntu0.3).\n",
            "libx11-6 set to manually installed.\n",
            "libx11-xcb1 is already the newest version (2:1.7.5-1ubuntu0.3).\n",
            "libx11-xcb1 set to manually installed.\n",
            "xdg-utils is already the newest version (1.1.3-4.1ubuntu3~22.04.1).\n",
            "xdg-utils set to manually installed.\n",
            "The following packages were automatically installed and are no longer required:\n",
            "  linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic\n",
            "Use 'sudo apt autoremove' to remove them.\n",
            "The following additional packages will be installed:\n",
            "  libdbusmenu-glib4 libdbusmenu-gtk3-4\n",
            "Suggested packages:\n",
            "  indicator-application\n",
            "The following NEW packages will be installed:\n",
            "  libappindicator3-1 libdbusmenu-glib4 libdbusmenu-gtk3-4 libxtst6\n",
            "0 upgraded, 4 newly installed, 0 to remove and 4 not upgraded.\n",
            "Need to get 113 kB of archives.\n",
            "After this operation, 432 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 libdbusmenu-glib4 amd64 16.04.1+18.10.20180917-0ubuntu8 [45.4 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libdbusmenu-gtk3-4 amd64 16.04.1+18.10.20180917-0ubuntu8 [31.0 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libappindicator3-1 amd64 12.10.1+20.10.20200706.1-0ubuntu1 [23.2 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxtst6 amd64 2:1.2.3-1build4 [13.4 kB]\n",
            "Fetched 113 kB in 1s (134 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 4.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libdbusmenu-glib4:amd64.\n",
            "(Reading database ... 150031 files and directories currently installed.)\n",
            "Preparing to unpack .../libdbusmenu-glib4_16.04.1+18.10.20180917-0ubuntu8_amd64.deb ...\n",
            "Unpacking libdbusmenu-glib4:amd64 (16.04.1+18.10.20180917-0ubuntu8) ...\n",
            "Selecting previously unselected package libdbusmenu-gtk3-4:amd64.\n",
            "Preparing to unpack .../libdbusmenu-gtk3-4_16.04.1+18.10.20180917-0ubuntu8_amd64.deb ...\n",
            "Unpacking libdbusmenu-gtk3-4:amd64 (16.04.1+18.10.20180917-0ubuntu8) ...\n",
            "Selecting previously unselected package libappindicator3-1.\n",
            "Preparing to unpack .../libappindicator3-1_12.10.1+20.10.20200706.1-0ubuntu1_amd64.deb ...\n",
            "Unpacking libappindicator3-1 (12.10.1+20.10.20200706.1-0ubuntu1) ...\n",
            "Selecting previously unselected package libxtst6:amd64.\n",
            "Preparing to unpack .../libxtst6_2%3a1.2.3-1build4_amd64.deb ...\n",
            "Unpacking libxtst6:amd64 (2:1.2.3-1build4) ...\n",
            "Setting up libxtst6:amd64 (2:1.2.3-1build4) ...\n",
            "Setting up libdbusmenu-glib4:amd64 (16.04.1+18.10.20180917-0ubuntu8) ...\n",
            "Setting up libdbusmenu-gtk3-4:amd64 (16.04.1+18.10.20180917-0ubuntu8) ...\n",
            "Setting up libappindicator3-1 (12.10.1+20.10.20200706.1-0ubuntu1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.6) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "Unzip the binary file...\n",
            "Archive:  chrome-linux64.zip\n",
            "  inflating: chrome-linux64/ABOUT    \n",
            "  inflating: chrome-linux64/MEIPreload/manifest.json  \n",
            "  inflating: chrome-linux64/MEIPreload/preloaded_data.pb  \n",
            "  inflating: chrome-linux64/chrome   \n",
            "  inflating: chrome-linux64/chrome-wrapper  \n",
            "  inflating: chrome-linux64/chrome_100_percent.pak  \n",
            "  inflating: chrome-linux64/chrome_200_percent.pak  \n",
            "  inflating: chrome-linux64/chrome_crashpad_handler  \n",
            "  inflating: chrome-linux64/chrome_sandbox  \n",
            "  inflating: chrome-linux64/icudtl.dat  \n",
            "  inflating: chrome-linux64/libEGL.so  \n",
            "  inflating: chrome-linux64/libGLESv2.so  \n",
            "  inflating: chrome-linux64/libvk_swiftshader.so  \n",
            "  inflating: chrome-linux64/libvulkan.so.1  \n",
            " extracting: chrome-linux64/product_logo_48.png  \n",
            "  inflating: chrome-linux64/resources.pak  \n",
            "  inflating: chrome-linux64/v8_context_snapshot.bin  \n",
            "  inflating: chrome-linux64/vk_swiftshader_icd.json  \n",
            "  inflating: chrome-linux64/xdg-mime  \n",
            "  inflating: chrome-linux64/xdg-settings  \n",
            "   creating: chrome-linux64/locales/\n",
            "  inflating: chrome-linux64/locales/af.pak.info  \n",
            "  inflating: chrome-linux64/locales/ja.pak  \n",
            "  inflating: chrome-linux64/locales/fil.pak  \n",
            "  inflating: chrome-linux64/locales/ca.pak.info  \n",
            "  inflating: chrome-linux64/locales/es.pak.info  \n",
            "  inflating: chrome-linux64/locales/th.pak  \n",
            "  inflating: chrome-linux64/locales/en-GB.pak.info  \n",
            "  inflating: chrome-linux64/locales/ro.pak  \n",
            "  inflating: chrome-linux64/locales/fil.pak.info  \n",
            "  inflating: chrome-linux64/locales/en-US.pak.info  \n",
            "  inflating: chrome-linux64/locales/hi.pak.info  \n",
            "  inflating: chrome-linux64/locales/sw.pak.info  \n",
            "  inflating: chrome-linux64/locales/lv.pak.info  \n",
            "  inflating: chrome-linux64/locales/sw.pak  \n",
            "  inflating: chrome-linux64/locales/uk.pak  \n",
            "  inflating: chrome-linux64/locales/de.pak.info  \n",
            "  inflating: chrome-linux64/locales/sv.pak.info  \n",
            "  inflating: chrome-linux64/locales/et.pak.info  \n",
            "  inflating: chrome-linux64/locales/ko.pak  \n",
            "  inflating: chrome-linux64/locales/zh-CN.pak  \n",
            "  inflating: chrome-linux64/locales/bg.pak  \n",
            "  inflating: chrome-linux64/locales/sl.pak.info  \n",
            "  inflating: chrome-linux64/locales/id.pak.info  \n",
            "  inflating: chrome-linux64/locales/pl.pak.info  \n",
            "  inflating: chrome-linux64/locales/ru.pak.info  \n",
            "  inflating: chrome-linux64/locales/it.pak.info  \n",
            "  inflating: chrome-linux64/locales/ko.pak.info  \n",
            "  inflating: chrome-linux64/locales/mr.pak.info  \n",
            "  inflating: chrome-linux64/locales/mr.pak  \n",
            "  inflating: chrome-linux64/locales/es.pak  \n",
            "  inflating: chrome-linux64/locales/cs.pak.info  \n",
            "  inflating: chrome-linux64/locales/hr.pak.info  \n",
            "  inflating: chrome-linux64/locales/cs.pak  \n",
            "  inflating: chrome-linux64/locales/lt.pak  \n",
            "  inflating: chrome-linux64/locales/ml.pak.info  \n",
            "  inflating: chrome-linux64/locales/te.pak  \n",
            "  inflating: chrome-linux64/locales/ta.pak.info  \n",
            "  inflating: chrome-linux64/locales/ca.pak  \n",
            "  inflating: chrome-linux64/locales/gu.pak  \n",
            "  inflating: chrome-linux64/locales/ar.pak  \n",
            "  inflating: chrome-linux64/locales/ms.pak  \n",
            "  inflating: chrome-linux64/locales/uk.pak.info  \n",
            "  inflating: chrome-linux64/locales/da.pak  \n",
            "  inflating: chrome-linux64/locales/nb.pak.info  \n",
            "  inflating: chrome-linux64/locales/am.pak.info  \n",
            "  inflating: chrome-linux64/locales/ml.pak  \n",
            "  inflating: chrome-linux64/locales/ro.pak.info  \n",
            "  inflating: chrome-linux64/locales/ur.pak.info  \n",
            "  inflating: chrome-linux64/locales/sk.pak  \n",
            "  inflating: chrome-linux64/locales/es-419.pak  \n",
            "  inflating: chrome-linux64/locales/th.pak.info  \n",
            "  inflating: chrome-linux64/locales/en-GB.pak  \n",
            "  inflating: chrome-linux64/locales/he.pak  \n",
            "  inflating: chrome-linux64/locales/ta.pak  \n",
            "  inflating: chrome-linux64/locales/hu.pak.info  \n",
            "  inflating: chrome-linux64/locales/lt.pak.info  \n",
            "  inflating: chrome-linux64/locales/te.pak.info  \n",
            "  inflating: chrome-linux64/locales/et.pak  \n",
            "  inflating: chrome-linux64/locales/zh-TW.pak.info  \n",
            "  inflating: chrome-linux64/locales/af.pak  \n",
            "  inflating: chrome-linux64/locales/zh-TW.pak  \n",
            "  inflating: chrome-linux64/locales/zh-CN.pak.info  \n",
            "  inflating: chrome-linux64/locales/id.pak  \n",
            "  inflating: chrome-linux64/locales/hi.pak  \n",
            "  inflating: chrome-linux64/locales/ja.pak.info  \n",
            "  inflating: chrome-linux64/locales/kn.pak  \n",
            "  inflating: chrome-linux64/locales/bn.pak  \n",
            "  inflating: chrome-linux64/locales/el.pak  \n",
            "  inflating: chrome-linux64/locales/en-US.pak  \n",
            "  inflating: chrome-linux64/locales/el.pak.info  \n",
            "  inflating: chrome-linux64/locales/sr.pak  \n",
            "  inflating: chrome-linux64/locales/fr.pak  \n",
            "  inflating: chrome-linux64/locales/ar.pak.info  \n",
            "  inflating: chrome-linux64/locales/de.pak  \n",
            "  inflating: chrome-linux64/locales/vi.pak  \n",
            "  inflating: chrome-linux64/locales/pt-BR.pak  \n",
            "  inflating: chrome-linux64/locales/hu.pak  \n",
            "  inflating: chrome-linux64/locales/nl.pak.info  \n",
            "  inflating: chrome-linux64/locales/bg.pak.info  \n",
            "  inflating: chrome-linux64/locales/gu.pak.info  \n",
            "  inflating: chrome-linux64/locales/bn.pak.info  \n",
            "  inflating: chrome-linux64/locales/pt-PT.pak.info  \n",
            "  inflating: chrome-linux64/locales/kn.pak.info  \n",
            "  inflating: chrome-linux64/locales/hr.pak  \n",
            "  inflating: chrome-linux64/locales/fi.pak.info  \n",
            "  inflating: chrome-linux64/locales/pt-PT.pak  \n",
            "  inflating: chrome-linux64/locales/ms.pak.info  \n",
            "  inflating: chrome-linux64/locales/ur.pak  \n",
            "  inflating: chrome-linux64/locales/it.pak  \n",
            "  inflating: chrome-linux64/locales/fr.pak.info  \n",
            "  inflating: chrome-linux64/locales/pl.pak  \n",
            "  inflating: chrome-linux64/locales/es-419.pak.info  \n",
            "  inflating: chrome-linux64/locales/sr.pak.info  \n",
            "  inflating: chrome-linux64/locales/fa.pak.info  \n",
            "  inflating: chrome-linux64/locales/tr.pak  \n",
            "  inflating: chrome-linux64/locales/pt-BR.pak.info  \n",
            "  inflating: chrome-linux64/locales/fa.pak  \n",
            "  inflating: chrome-linux64/locales/vi.pak.info  \n",
            "  inflating: chrome-linux64/locales/he.pak.info  \n",
            "  inflating: chrome-linux64/locales/nb.pak  \n",
            "  inflating: chrome-linux64/locales/lv.pak  \n",
            "  inflating: chrome-linux64/locales/nl.pak  \n",
            "  inflating: chrome-linux64/locales/fi.pak  \n",
            "  inflating: chrome-linux64/locales/ru.pak  \n",
            "  inflating: chrome-linux64/locales/da.pak.info  \n",
            "  inflating: chrome-linux64/locales/tr.pak.info  \n",
            "  inflating: chrome-linux64/locales/sk.pak.info  \n",
            "  inflating: chrome-linux64/locales/sl.pak  \n",
            "  inflating: chrome-linux64/locales/am.pak  \n",
            "  inflating: chrome-linux64/locales/sv.pak  \n",
            "   creating: chrome-linux64/resources/\n",
            "   creating: chrome-linux64/resources/inspector_overlay/\n",
            "  inflating: chrome-linux64/resources/inspector_overlay/main.js  \n",
            "  inflating: chrome-linux64/resources/inspector_overlay/inspector_overlay_resources.grd  \n",
            "Downloading latest Chromedriver...\n",
            "--2024-03-10 12:15:17--  https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.94/linux64/chromedriver-linux64.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 172.253.117.207, 142.250.99.207, 142.250.107.207, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|172.253.117.207|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 8597995 (8.2M) [application/zip]\n",
            "Saving to: ‘chromedriver-linux64.zip’\n",
            "\n",
            "chromedriver-linux6 100%[===================>]   8.20M  --.-KB/s    in 0.07s   \n",
            "\n",
            "2024-03-10 12:15:17 (113 MB/s) - ‘chromedriver-linux64.zip’ saved [8597995/8597995]\n",
            "\n",
            "Unzip the binary file and make it executable...\n",
            "Archive:  chromedriver-linux64.zip\n",
            "  inflating: chromedriver-linux64/LICENSE.chromedriver  \n",
            "  inflating: chromedriver-linux64/chromedriver  \n",
            "Install Selenium...\n",
            "Collecting selenium\n",
            "  Downloading selenium-4.18.1-py3-none-any.whl (10.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.0/10.0 MB\u001b[0m \u001b[31m30.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3[socks]<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from selenium) (2.0.7)\n",
            "Collecting trio~=0.17 (from selenium)\n",
            "  Downloading trio-0.24.0-py3-none-any.whl (460 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m460.2/460.2 kB\u001b[0m \u001b[31m43.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting trio-websocket~=0.9 (from selenium)\n",
            "  Downloading trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.10/dist-packages (from selenium) (2024.2.2)\n",
            "Requirement already satisfied: typing_extensions>=4.9.0 in /usr/local/lib/python3.10/dist-packages (from selenium) (4.10.0)\n",
            "Requirement already satisfied: attrs>=20.1.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (23.2.0)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (2.4.0)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (3.6)\n",
            "Collecting outcome (from trio~=0.17->selenium)\n",
            "  Downloading outcome-1.3.0.post0-py2.py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: sniffio>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.2.0)\n",
            "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium)\n",
            "  Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
            "Collecting h11<1,>=0.9.0 (from wsproto>=0.14->trio-websocket~=0.9->selenium)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: outcome, h11, wsproto, trio, trio-websocket, selenium\n",
            "Successfully installed h11-0.14.0 outcome-1.3.0.post0 selenium-4.18.1 trio-0.24.0 trio-websocket-0.11.1 wsproto-1.2.0\n",
            "Removing archive files\n"
          ]
        }
      ],
      "source": [
        "#!/usr/bin/bash\n",
        "\n",
        "!echo \"Update the repository and any packages...\"\n",
        "!sudo apt update && sudo apt upgrade -y\n",
        "\n",
        "!echo \"Install prerequisite system packages...\"\n",
        "!sudo apt install wget curl unzip jq -y\n",
        "\n",
        "# Set metadata for Google Chrome repository...\n",
        "!wget https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.94/linux64/chrome-linux64.zip\n",
        "\n",
        "!echo \"Install Chrome dependencies...\"\n",
        "!sudo apt install ca-certificates fonts-liberation \\\n",
        "    libappindicator3-1 libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 \\\n",
        "    libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 \\\n",
        "    libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 \\\n",
        "    libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \\\n",
        "    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 \\\n",
        "    libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils -y\n",
        "\n",
        "!echo \"Unzip the binary file...\"\n",
        "!unzip chrome-linux64.zip\n",
        "\n",
        "!echo \"Downloading latest Chromedriver...\"\n",
        "!wget https://storage.googleapis.com/chrome-for-testing-public/122.0.6261.94/linux64/chromedriver-linux64.zip\n",
        "\n",
        "!echo \"Unzip the binary file and make it executable...\"\n",
        "!unzip chromedriver-linux64.zip\n",
        "\n",
        "!echo \"Install Selenium...\"\n",
        "!python3 -m pip install selenium\n",
        "\n",
        "!echo \"Removing archive files\"\n",
        "!rm chrome-linux64.zip  chromedriver-linux64.zip"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4BAtd_MKX42j"
      },
      "source": [
        "### Requirements"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gEZvvC1Ln0nH",
        "outputId": "43712b31-276e-4fe9-8781-a27acf87a71c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-03-10 12:15:26--  https://raw.githubusercontent.com/dhuynh95/LaVague/main/requirements.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 190 [text/plain]\n",
            "Saving to: ‘requirements.txt’\n",
            "\n",
            "\rrequirements.txt      0%[                    ]       0  --.-KB/s               \rrequirements.txt    100%[===================>]     190  --.-KB/s    in 0s      \n",
            "\n",
            "2024-03-10 12:15:27 (4.26 MB/s) - ‘requirements.txt’ saved [190/190]\n",
            "\n",
            "Collecting llama_index (from -r requirements.txt (line 1))\n",
            "  Downloading llama_index-0.10.18-py3-none-any.whl (5.6 kB)\n",
            "Collecting llama-index-embeddings-huggingface (from -r requirements.txt (line 2))\n",
            "  Downloading llama_index_embeddings_huggingface-0.1.4-py3-none-any.whl (7.7 kB)\n",
            "Collecting llama-index-llms-huggingface (from -r requirements.txt (line 3))\n",
            "  Downloading llama_index_llms_huggingface-0.1.3-py3-none-any.whl (7.2 kB)\n",
            "Collecting llama-index-retrievers-bm25 (from -r requirements.txt (line 4))\n",
            "  Downloading llama_index_retrievers_bm25-0.1.3-py3-none-any.whl (2.9 kB)\n",
            "Collecting tree-sitter (from -r requirements.txt (line 5))\n",
            "  Downloading tree_sitter-0.20.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (488 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m488.2/488.2 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tree-sitter-languages (from -r requirements.txt (line 6))\n",
            "  Downloading tree_sitter_languages-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m98.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: nest_asyncio in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (1.6.0)\n",
            "Requirement already satisfied: selenium in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (4.18.1)\n",
            "Collecting google-search-results (from -r requirements.txt (line 9))\n",
            "  Downloading google_search_results-2.4.2.tar.gz (18 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting llama-index-agent-openai<0.2.0,>=0.1.4 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_agent_openai-0.1.5-py3-none-any.whl (12 kB)\n",
            "Collecting llama-index-cli<0.2.0,>=0.1.2 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_cli-0.1.8-py3-none-any.whl (25 kB)\n",
            "Collecting llama-index-core<0.11.0,>=0.10.18 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_core-0.10.18.post1-py3-none-any.whl (15.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m85.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting llama-index-embeddings-openai<0.2.0,>=0.1.5 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_embeddings_openai-0.1.6-py3-none-any.whl (6.0 kB)\n",
            "Collecting llama-index-indices-managed-llama-cloud<0.2.0,>=0.1.2 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.1.3-py3-none-any.whl (6.6 kB)\n",
            "Collecting llama-index-legacy<0.10.0,>=0.9.48 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_legacy-0.9.48-py3-none-any.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m87.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting llama-index-llms-openai<0.2.0,>=0.1.5 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_llms_openai-0.1.7-py3-none-any.whl (9.3 kB)\n",
            "Collecting llama-index-multi-modal-llms-openai<0.2.0,>=0.1.3 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_multi_modal_llms_openai-0.1.4-py3-none-any.whl (5.8 kB)\n",
            "Collecting llama-index-program-openai<0.2.0,>=0.1.3 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_program_openai-0.1.4-py3-none-any.whl (4.1 kB)\n",
            "Collecting llama-index-question-gen-openai<0.2.0,>=0.1.2 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_question_gen_openai-0.1.3-py3-none-any.whl (2.9 kB)\n",
            "Collecting llama-index-readers-file<0.2.0,>=0.1.4 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_readers_file-0.1.9-py3-none-any.whl (35 kB)\n",
            "Collecting llama-index-readers-llama-parse<0.2.0,>=0.1.2 (from llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_readers_llama_parse-0.1.3-py3-none-any.whl (2.5 kB)\n",
            "Requirement already satisfied: huggingface-hub[inference]>=0.19.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (0.20.3)\n",
            "Collecting torch<3.0.0,>=2.1.2 (from llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading torch-2.2.1-cp310-cp310-manylinux1_x86_64.whl (755.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m755.5/755.5 MB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.37.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (4.38.2)\n",
            "Collecting rank-bm25<0.3.0,>=0.2.2 (from llama-index-retrievers-bm25->-r requirements.txt (line 4))\n",
            "  Downloading rank_bm25-0.2.2-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: urllib3[socks]<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from selenium->-r requirements.txt (line 8)) (2.0.7)\n",
            "Requirement already satisfied: trio~=0.17 in /usr/local/lib/python3.10/dist-packages (from selenium->-r requirements.txt (line 8)) (0.24.0)\n",
            "Requirement already satisfied: trio-websocket~=0.9 in /usr/local/lib/python3.10/dist-packages (from selenium->-r requirements.txt (line 8)) (0.11.1)\n",
            "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.10/dist-packages (from selenium->-r requirements.txt (line 8)) (2024.2.2)\n",
            "Requirement already satisfied: typing_extensions>=4.9.0 in /usr/local/lib/python3.10/dist-packages (from selenium->-r requirements.txt (line 8)) (4.10.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from google-search-results->-r requirements.txt (line 9)) (2.31.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (2023.6.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (4.66.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (6.0.1)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (23.2)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (3.9.3)\n",
            "Requirement already satisfied: pydantic<3.0,>1.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (2.6.3)\n",
            "Collecting llama-index-vector-stores-chroma<0.2.0,>=0.1.1 (from llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_index_vector_stores_chroma-0.1.6-py3-none-any.whl (4.7 kB)\n",
            "Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (2.0.28)\n",
            "Collecting dataclasses-json (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading dataclasses_json-0.6.4-py3-none-any.whl (28 kB)\n",
            "Collecting deprecated>=1.2.9.3 (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Collecting dirtyjson<2.0.0,>=1.0.8 (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading dirtyjson-1.0.8-py3-none-any.whl (25 kB)\n",
            "Collecting httpx (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting llamaindex-py-client<0.2.0,>=0.1.13 (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llamaindex_py_client-0.1.13-py3-none-any.whl (107 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.0/108.0 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (3.2.1)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (3.8.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (1.25.2)\n",
            "Collecting openai>=1.1.0 (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading openai-1.13.3-py3-none-any.whl (227 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.4/227.4 kB\u001b[0m \u001b[31m31.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (1.5.3)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (9.4.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (8.2.3)\n",
            "Collecting tiktoken>=0.3.3 (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m93.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect>=0.8.0 (from llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index->-r requirements.txt (line 1)) (4.12.3)\n",
            "Collecting bs4<0.0.3,>=0.0.2 (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading bs4-0.0.2-py2.py3-none-any.whl (1.2 kB)\n",
            "Collecting pymupdf<2.0.0,>=1.23.21 (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading PyMuPDF-1.23.26-cp310-none-manylinux2014_x86_64.whl (4.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m107.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pypdf<5.0.0,>=4.0.1 (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading pypdf-4.1.0-py3-none-any.whl (286 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m286.1/286.1 kB\u001b[0m \u001b[31m36.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting striprtf<0.0.27,>=0.0.26 (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading striprtf-0.0.26-py3-none-any.whl (6.9 kB)\n",
            "Collecting llama-parse<0.4.0,>=0.3.3 (from llama-index-readers-llama-parse<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading llama_parse-0.3.8-py3-none-any.whl (6.7 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->google-search-results->-r requirements.txt (line 9)) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->google-search-results->-r requirements.txt (line 9)) (3.6)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (1.12)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (3.1.3)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m70.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m69.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m102.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m29.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.2.0 (from torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading triton-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.99-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m86.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.37.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.37.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.37.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (0.4.2)\n",
            "Collecting accelerate>=0.21.0 (from transformers<5.0.0,>=4.37.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2))\n",
            "  Downloading accelerate-0.27.2-py3-none-any.whl (279 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m36.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=20.1.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 8)) (23.2.0)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 8)) (2.4.0)\n",
            "Requirement already satisfied: outcome in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 8)) (1.3.0.post0)\n",
            "Requirement already satisfied: sniffio>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 8)) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium->-r requirements.txt (line 8)) (1.2.0)\n",
            "Requirement already satisfied: wsproto>=0.14 in /usr/local/lib/python3.10/dist-packages (from trio-websocket~=0.9->selenium->-r requirements.txt (line 8)) (1.2.0)\n",
            "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from urllib3[socks]<3,>=1.26->selenium->-r requirements.txt (line 8)) (1.7.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->transformers<5.0.0,>=4.37.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (5.9.5)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (1.3.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (4.0.3)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-file<0.2.0,>=0.1.4->llama_index->-r requirements.txt (line 1)) (2.5)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.9.3->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (1.14.1)\n",
            "Collecting chromadb<0.5.0,>=0.4.22 (from llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading chromadb-0.4.24-py3-none-any.whl (525 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m525.5/525.5 kB\u001b[0m \u001b[31m52.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (3.7.1)\n",
            "Collecting httpcore==1.* (from httpx->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading httpcore-1.0.4-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (0.14.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (1.3.2)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (1.7.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0,>1.1->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0,>1.1->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (2.16.3)\n",
            "Collecting PyMuPDFb==1.23.22 (from pymupdf<2.0.0,>=1.23.21->llama-index-readers-file<0.2.0,>=0.1.4->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading PyMuPDFb-1.23.22-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (30.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m30.6/30.6 MB\u001b[0m \u001b[31m58.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (3.0.3)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading marshmallow-3.21.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (2.1.5)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (2023.4)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch<3.0.0,>=2.1.2->llama-index-embeddings-huggingface->-r requirements.txt (line 2)) (1.3.0)\n",
            "Requirement already satisfied: build>=1.0.3 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (1.1.1)\n",
            "Collecting chroma-hnswlib==0.7.3 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m100.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading fastapi-0.110.0-py3-none-any.whl (92 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.1/92.1 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading uvicorn-0.28.0-py3-none-any.whl (60 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting posthog>=2.4.0 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading posthog-3.5.0-py2.py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pulsar-client>=3.1.0 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading pulsar_client-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m106.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime>=1.14.1 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading onnxruntime-1.17.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m110.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-api>=1.2.0 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_api-1.23.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.4/58.4 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_exporter_otlp_proto_grpc-1.23.0-py3-none-any.whl (18 kB)\n",
            "Collecting opentelemetry-instrumentation-fastapi>=0.41b0 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_instrumentation_fastapi-0.44b0-py3-none-any.whl (11 kB)\n",
            "Collecting opentelemetry-sdk>=1.2.0 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_sdk-1.23.0-py3-none-any.whl (105 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.7/105.7 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pypika>=0.48.9 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading PyPika-0.48.9.tar.gz (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting overrides>=7.3.1 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading overrides-7.7.0-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (6.1.2)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (1.62.0)\n",
            "Collecting bcrypt>=4.0.1 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading bcrypt-4.1.2-cp39-abi3-manylinux_2_28_x86_64.whl (698 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m698.9/698.9 kB\u001b[0m \u001b[31m65.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (0.9.0)\n",
            "Collecting kubernetes>=28.1.0 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading kubernetes-29.0.0-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m92.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting mmh3>=4.0.1 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting orjson>=3.9.12 (from chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading orjson-3.9.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.5/138.5 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->llama-index-core<0.11.0,>=0.10.18->llama_index->-r requirements.txt (line 1)) (1.16.0)\n",
            "Requirement already satisfied: pyproject_hooks in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (1.0.0)\n",
            "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (2.0.1)\n",
            "Collecting starlette<0.37.0,>=0.36.3 (from fastapi>=0.95.2->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading starlette-0.36.3-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (2.27.0)\n",
            "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (1.7.0)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (1.3.1)\n",
            "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (3.2.2)\n",
            "Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (23.5.26)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (3.20.3)\n",
            "Collecting importlib-metadata<7.0,>=6.0 (from opentelemetry-api>=1.2.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading importlib_metadata-6.11.0-py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (1.62.0)\n",
            "Collecting opentelemetry-exporter-otlp-proto-common==1.23.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_exporter_otlp_proto_common-1.23.0-py3-none-any.whl (17 kB)\n",
            "Collecting opentelemetry-proto==1.23.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_proto-1.23.0-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-instrumentation-asgi==0.44b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_instrumentation_asgi-0.44b0-py3-none-any.whl (14 kB)\n",
            "Collecting opentelemetry-instrumentation==0.44b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_instrumentation-0.44b0-py3-none-any.whl (28 kB)\n",
            "Collecting opentelemetry-semantic-conventions==0.44b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_semantic_conventions-0.44b0-py3-none-any.whl (36 kB)\n",
            "Collecting opentelemetry-util-http==0.44b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading opentelemetry_util_http-0.44b0-py3-none-any.whl (6.9 kB)\n",
            "Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.44b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (67.7.2)\n",
            "Collecting asgiref~=3.0 (from opentelemetry-instrumentation-asgi==0.44b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading asgiref-3.7.2-py3-none-any.whl (24 kB)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Collecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m38.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m111.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m89.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (4.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (3.17.0)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1))\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.5.0,>=0.4.22->llama-index-vector-stores-chroma<0.2.0,>=0.1.1->llama-index-cli<0.2.0,>=0.1.2->llama_index->-r requirements.txt (line 1)) (0.5.1)\n",
            "Building wheels for collected packages: google-search-results, pypika\n",
            "  Building wheel for google-search-results (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for google-search-results: filename=google_search_results-2.4.2-py3-none-any.whl size=32003 sha256=d68ccd6a8e0dd83c320cc9958c0319094d448b897c2f5ec7fce2ecc5d2d70270\n",
            "  Stored in directory: /root/.cache/pip/wheels/d3/b2/c3/03302d12bb44a2cdff3c9371f31b72c0c4e84b8d2285eeac53\n",
            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53723 sha256=3b1d64f830e5e1b1a40abaf74e2cc06f59c4623b64446e57733652047458e056\n",
            "  Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n",
            "Successfully built google-search-results pypika\n",
            "Installing collected packages: striprtf, pypika, monotonic, mmh3, dirtyjson, websockets, uvloop, uvicorn, triton, tree-sitter, rank-bm25, python-dotenv, pypdf, PyMuPDFb, pulsar-client, overrides, orjson, opentelemetry-util-http, opentelemetry-semantic-conventions, opentelemetry-proto, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, mypy-extensions, marshmallow, importlib-metadata, humanfriendly, httptools, httpcore, deprecated, chroma-hnswlib, bcrypt, backoff, asgiref, watchfiles, typing-inspect, tree-sitter-languages, tiktoken, starlette, pymupdf, posthog, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, nvidia-cusparse-cu12, nvidia-cudnn-cu12, httpx, google-search-results, coloredlogs, bs4, opentelemetry-sdk, opentelemetry-instrumentation, openai, onnxruntime, nvidia-cusolver-cu12, llamaindex-py-client, kubernetes, fastapi, dataclasses-json, torch, opentelemetry-instrumentation-asgi, opentelemetry-exporter-otlp-proto-grpc, llama-index-legacy, llama-index-core, opentelemetry-instrumentation-fastapi, llama-parse, llama-index-retrievers-bm25, llama-index-readers-file, llama-index-llms-openai, llama-index-indices-managed-llama-cloud, llama-index-embeddings-openai, llama-index-embeddings-huggingface, accelerate, llama-index-readers-llama-parse, llama-index-multi-modal-llms-openai, llama-index-agent-openai, chromadb, llama-index-vector-stores-chroma, llama-index-program-openai, llama-index-llms-huggingface, llama-index-question-gen-openai, llama-index-cli, llama_index\n",
            "  Attempting uninstall: triton\n",
            "    Found existing installation: triton 2.1.0\n",
            "    Uninstalling triton-2.1.0:\n",
            "      Successfully uninstalled triton-2.1.0\n",
            "  Attempting uninstall: importlib-metadata\n",
            "    Found existing installation: importlib-metadata 7.0.1\n",
            "    Uninstalling importlib-metadata-7.0.1:\n",
            "      Successfully uninstalled importlib-metadata-7.0.1\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 2.1.0+cu121\n",
            "    Uninstalling torch-2.1.0+cu121:\n",
            "      Successfully uninstalled torch-2.1.0+cu121\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchaudio 2.1.0+cu121 requires torch==2.1.0, but you have torch 2.2.1 which is incompatible.\n",
            "torchdata 0.7.0 requires torch==2.1.0, but you have torch 2.2.1 which is incompatible.\n",
            "torchtext 0.16.0 requires torch==2.1.0, but you have torch 2.2.1 which is incompatible.\n",
            "torchvision 0.16.0+cu121 requires torch==2.1.0, but you have torch 2.2.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed PyMuPDFb-1.23.22 accelerate-0.27.2 asgiref-3.7.2 backoff-2.2.1 bcrypt-4.1.2 bs4-0.0.2 chroma-hnswlib-0.7.3 chromadb-0.4.24 coloredlogs-15.0.1 dataclasses-json-0.6.4 deprecated-1.2.14 dirtyjson-1.0.8 fastapi-0.110.0 google-search-results-2.4.2 httpcore-1.0.4 httptools-0.6.1 httpx-0.27.0 humanfriendly-10.0 importlib-metadata-6.11.0 kubernetes-29.0.0 llama-index-agent-openai-0.1.5 llama-index-cli-0.1.8 llama-index-core-0.10.18.post1 llama-index-embeddings-huggingface-0.1.4 llama-index-embeddings-openai-0.1.6 llama-index-indices-managed-llama-cloud-0.1.3 llama-index-legacy-0.9.48 llama-index-llms-huggingface-0.1.3 llama-index-llms-openai-0.1.7 llama-index-multi-modal-llms-openai-0.1.4 llama-index-program-openai-0.1.4 llama-index-question-gen-openai-0.1.3 llama-index-readers-file-0.1.9 llama-index-readers-llama-parse-0.1.3 llama-index-retrievers-bm25-0.1.3 llama-index-vector-stores-chroma-0.1.6 llama-parse-0.3.8 llama_index-0.10.18 llamaindex-py-client-0.1.13 marshmallow-3.21.1 mmh3-4.1.0 monotonic-1.6 mypy-extensions-1.0.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.99 nvidia-nvtx-cu12-12.1.105 onnxruntime-1.17.1 openai-1.13.3 opentelemetry-api-1.23.0 opentelemetry-exporter-otlp-proto-common-1.23.0 opentelemetry-exporter-otlp-proto-grpc-1.23.0 opentelemetry-instrumentation-0.44b0 opentelemetry-instrumentation-asgi-0.44b0 opentelemetry-instrumentation-fastapi-0.44b0 opentelemetry-proto-1.23.0 opentelemetry-sdk-1.23.0 opentelemetry-semantic-conventions-0.44b0 opentelemetry-util-http-0.44b0 orjson-3.9.15 overrides-7.7.0 posthog-3.5.0 pulsar-client-3.4.0 pymupdf-1.23.26 pypdf-4.1.0 pypika-0.48.9 python-dotenv-1.0.1 rank-bm25-0.2.2 starlette-0.36.3 striprtf-0.0.26 tiktoken-0.6.0 torch-2.2.1 tree-sitter-0.20.4 tree-sitter-languages-1.10.2 triton-2.2.0 typing-inspect-0.9.0 uvicorn-0.28.0 uvloop-0.19.0 watchfiles-0.21.0 websockets-12.0\n"
          ]
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/dhuynh95/LaVague/main/requirements.txt\n",
        "!pip install -r requirements.txt\n",
        "\n",
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0jo-qziepeOj"
      },
      "source": [
        "## Choosing inference engine\n",
        "\n",
        "LaVague works with two kind of inference:\n",
        "- Local\n",
        "- API with Hugging Face Inference API\n",
        "\n",
        "Local model allows full control over the experience, but might be slower to setup.\n",
        "Starting with Hugging Face Inference API is good for a quick start but lacks flexibility and control.\n",
        "\n",
        "Both options work, but in this Colab notebook, it might take longer to run with local model as weights download can take a while."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Hugging Face Inference API\n",
        "\n",
        "To have a fast and low-cost experience, we will use [Hugging Face Inference for PRO users](https://huggingface.co/blog/inference-pro).\n",
        "You can swap the call to the ``Nous-Hermes-2-Mixtral-8x7B-DPO``.\n",
        "\n",
        "You will need a Hugging Face Hub Token to use the ``Nous-Hermes-2-Mixtral-8x7B-DPO`` model from the Inference API. You can get one by signing up on the [Hugging Face Hub](https://huggingface.co/join).\n",
        "\n",
        "If you prefer using a local model, skip to the next section [Local model](#Local-model)."
      ],
      "metadata": {
        "id": "WgcPzmqUyGwB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.llms.huggingface import HuggingFaceInferenceAPI\n",
        "\n",
        "try:\n",
        "  from google.colab import userdata\n",
        "  HF_TOKEN = userdata.get('HF_TOKEN')\n",
        "except:\n",
        "  import os\n",
        "  HF_TOKEN = os.environ[\"HF_TOKEN\"]\n",
        "\n",
        "if not HF_TOKEN:\n",
        "  from getpass import getpass\n",
        "  HF_TOKEN = getpass('Enter your HF Token (https://huggingface.co/docs/hub/en/security-tokens): ')\n",
        "\n",
        "model_id = \"NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO\"\n",
        "max_new_tokens = 512\n",
        "\n",
        "# Monkey patch because stream_complete is not implemented in the current version of llama_index\n",
        "from llama_index.core.base.llms.types import (\n",
        "    CompletionResponse,\n",
        ")\n",
        "def stream_complete(\n",
        "    self, prompt: str, formatted: bool = False, **kwargs\n",
        "):\n",
        "  def gen():\n",
        "    text = \"\"\n",
        "    for x in self._sync_client.text_generation(\n",
        "            prompt, **{**{\"max_new_tokens\": self.num_output, \"stream\": True}, **kwargs}\n",
        "        ):\n",
        "      text += x\n",
        "      yield CompletionResponse(text=text, delta=x)\n",
        "  return gen()\n",
        "\n",
        "HuggingFaceInferenceAPI.stream_complete = stream_complete\n",
        "\n",
        "llm = HuggingFaceInferenceAPI(model_name=model_id, token=HF_TOKEN, num_output=max_new_tokens)"
      ],
      "metadata": {
        "id": "sEQIl-HYyyju"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Local model\n",
        "\n",
        "Here we will use the latest ``HuggingFaceH4/zephyr-7b-gemma-v0.1`` to do local inference."
      ],
      "metadata": {
        "id": "BXJplaLHzBgV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install accelerate"
      ],
      "metadata": {
        "id": "cZwjrYWH2NjB",
        "outputId": "fd61b1fe-71f4-4496-daea-3cd41db25ee3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.27.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.2.1)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.20.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate) (12.4.99)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -i https://pypi.org/simple/ bitsandbytes"
      ],
      "metadata": {
        "id": "1OKZCJQGq1GL",
        "outputId": "3110d52a-428f-466e-a7eb-e2206a82a18b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple/\n",
            "Collecting bitsandbytes\n",
            "  Downloading bitsandbytes-0.43.0-py3-none-manylinux_2_24_x86_64.whl (102.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.2/102.2 MB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (2.2.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (1.25.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->bitsandbytes) (12.4.99)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->bitsandbytes) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->bitsandbytes) (1.3.0)\n",
            "Installing collected packages: bitsandbytes\n",
            "Successfully installed bitsandbytes-0.43.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "398c2359c137457b94fe7ed7b8c4b939",
            "2600f072828a4abf8e8ed51cd64f2dc2",
            "15c0d6723eba4298b61ddd67c1595f66",
            "d1a4215edd20433482d339c83c9d0060",
            "c71a506bf63948c8bfb79b6811705de5",
            "984f401572a645f7861db035993c806b",
            "2ba0655ceb93417d859921bf08b5ec9d",
            "0d9126c291a04e8fb1ce30a7103197d1",
            "a36c6df55813490a9e0b42b67a698327",
            "ec5d092ea24946279f14f03570a48c98",
            "777ac9bad3974a2ab7062644bda6a3c6"
          ]
        },
        "id": "wcKFPIFe8PMp",
        "outputId": "c0fc56bb-6aae-478e-ab91-8277bfdfe45c"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "398c2359c137457b94fe7ed7b8c4b939"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from llama_index.llms.huggingface import HuggingFaceLLM\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
        "import torch\n",
        "\n",
        "model_id = \"HuggingFaceH4/zephyr-7b-gemma-v0.1\"\n",
        "\n",
        "quantization_config = BitsAndBytesConfig(\n",
        "load_in_4bit=True,\n",
        "bnb_4bit_use_double_quant=True,\n",
        "bnb_4bit_quant_type=\"nf4\",\n",
        "bnb_4bit_compute_dtype=torch.bfloat16\n",
        ")\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
        "model = AutoModelForCausalLM.from_pretrained(model_id, device_map=\"auto\", quantization_config=quantization_config)\n",
        "\n",
        "# We will stop generation as soon as the model outputs the end of Markdown to make inference faster\n",
        "stop_token_id = [tokenizer.convert_tokens_to_ids(\"```\"), tokenizer.convert_tokens_to_ids(\"``\")]\n",
        "llm = HuggingFaceLLM(model=model, tokenizer=tokenizer, max_new_tokens=1024, stopping_ids=stop_token_id)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setting up embedding model and prompt template"
      ],
      "metadata": {
        "id": "Pq5XzbJ9Pjje"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will use a ``bge-small-en-v1.5`` to perform semantic search."
      ],
      "metadata": {
        "id": "a6hECyZl04rs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
        "\n",
        "embed_model = \"BAAI/bge-small-en-v1.5\"\n",
        "embedder = HuggingFaceEmbedding(model_name=embed_model, device=\"cuda\")"
      ],
      "metadata": {
        "id": "-QJuhG_3zhW9"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will use a specific prompt template that leverages Few shot learning with Chain of Thought to ensure the model performs correctly for our use case of Selenium code generation.\n",
        "\n",
        "You can have a look at the template [here](https://github.com/dhuynh95/LaVague/blob/main/prompt_template.txt)."
      ],
      "metadata": {
        "id": "12OBKO8X1Mer"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\"\n",
        "\n",
        "!wget https://raw.githubusercontent.com/dhuynh95/LaVague/main/prompt_template.txt\n",
        "\n",
        "with open(\"prompt_template.txt\", \"r\") as file:\n",
        "  PROMPT_TEMPLATE_STR = file.read()"
      ],
      "metadata": {
        "id": "EvD6Ut-_1L-Z",
        "outputId": "3fd3a640-0a89-4ebc-ca32-cdc11333262a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-03-10 13:18:31--  https://raw.githubusercontent.com/dhuynh95/LaVague/main/prompt_template.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.109.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 5493 (5.4K) [text/plain]\n",
            "Saving to: ‘prompt_template.txt.4’\n",
            "\n",
            "\rprompt_template.txt   0%[                    ]       0  --.-KB/s               \rprompt_template.txt 100%[===================>]   5.36K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-03-10 13:18:31 (61.6 MB/s) - ‘prompt_template.txt.4’ saved [5493/5493]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I2v0gR_0peOk"
      },
      "source": [
        "## Preparing the action engine\n",
        "\n",
        "Here we will use llama_index to create an action engine that will be able to execute the instructions we will give it.\n",
        "\n",
        "It will index the HTML of the current page to retrieve the most relevant information to feed the LLM to then generate Selenium code to execute."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core import Document\n",
        "from llama_index.core.node_parser import CodeSplitter\n",
        "from llama_index.retrievers.bm25 import BM25Retriever\n",
        "from llama_index.core import VectorStoreIndex\n",
        "from llama_index.core.query_engine import RetrieverQueryEngine\n",
        "from llama_index.core import get_response_synthesizer\n",
        "from llama_index.core import PromptTemplate\n",
        "\n",
        "MAX_CHARS = 1500\n",
        "K = 3\n",
        "\n",
        "class ActionEngine:\n",
        "    def __init__(self, llm, embedding):\n",
        "        self.llm = llm\n",
        "        self.embedding = embedding\n",
        "\n",
        "    def _get_index(self, html):\n",
        "        text_list = [html]\n",
        "        documents = [Document(text=t) for t in text_list]\n",
        "\n",
        "        splitter = CodeSplitter(\n",
        "            language=\"html\",\n",
        "            chunk_lines=40,  # lines per chunk\n",
        "            chunk_lines_overlap=200,  # lines overlap between chunks\n",
        "            max_chars=MAX_CHARS,  # max chars per chunk\n",
        "        )\n",
        "        nodes = splitter.get_nodes_from_documents(documents)\n",
        "        nodes = [node for node in nodes if node.text]\n",
        "\n",
        "        index = VectorStoreIndex(nodes, embed_model=self.embedding)\n",
        "\n",
        "        return index\n",
        "\n",
        "    def get_query_engine(self, state):\n",
        "        html = state\n",
        "        index = self._get_index(html)\n",
        "\n",
        "        retriever = BM25Retriever.from_defaults(\n",
        "            index=index,\n",
        "            similarity_top_k=K,\n",
        "        )\n",
        "\n",
        "        response_synthesizer = get_response_synthesizer(streaming=True, llm=self.llm)\n",
        "\n",
        "        # assemble query engine\n",
        "        query_engine = RetrieverQueryEngine(\n",
        "            retriever=retriever,\n",
        "            response_synthesizer=response_synthesizer,\n",
        "        )\n",
        "\n",
        "        prompt_template = PromptTemplate(PROMPT_TEMPLATE_STR)\n",
        "\n",
        "        query_engine.update_prompts(\n",
        "            {\"response_synthesizer:text_qa_template\": prompt_template}\n",
        "        )\n",
        "\n",
        "        return query_engine"
      ],
      "metadata": {
        "id": "V0gqXcLjeIeN"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2cu4DWoApeOl"
      },
      "source": [
        "## Code execution in action"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gradio"
      ],
      "metadata": {
        "id": "m37SBNKBVaok",
        "outputId": "ebda8b53-be99-49fd-ece1-c4ee4fa93f7b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gradio\n",
            "  Downloading gradio-4.21.0-py3-none-any.whl (17.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m50.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiofiles<24.0,>=22.0 (from gradio)\n",
            "  Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.2.2)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio) (0.110.0)\n",
            "Collecting ffmpy (from gradio)\n",
            "  Downloading ffmpy-0.3.2.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gradio-client==0.12.0 (from gradio)\n",
            "  Downloading gradio_client-0.12.0-py3-none-any.whl (310 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m310.7/310.7 kB\u001b[0m \u001b[31m36.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: httpx>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.27.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.20.3)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.1.2)\n",
            "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.1.3)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.1.5)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.7.1)\n",
            "Requirement already satisfied: numpy~=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.25.2)\n",
            "Requirement already satisfied: orjson~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.9.15)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio) (23.2)\n",
            "Requirement already satisfied: pandas<3.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.5.3)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (9.4.0)\n",
            "Requirement already satisfied: pydantic>=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.6.3)\n",
            "Collecting pydub (from gradio)\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Collecting python-multipart>=0.0.9 (from gradio)\n",
            "  Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.0.1)\n",
            "Collecting ruff>=0.2.2 (from gradio)\n",
            "  Downloading ruff-0.3.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m93.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting semantic-version~=2.0 (from gradio)\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting tomlkit==0.12.0 (from gradio)\n",
            "  Downloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typer[all]<1.0,>=0.9 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.9.0)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.10.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.28.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.12.0->gradio) (2023.6.0)\n",
            "Collecting websockets<12.0,>=10.0 (from gradio-client==0.12.0->gradio)\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (4.19.2)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.12.1)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (2024.2.2)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (1.0.4)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (3.6)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.1->gradio) (0.14.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (3.13.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (4.66.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (4.49.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (3.1.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio) (2023.4)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio) (2.16.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio) (8.1.7)\n",
            "Collecting colorama<0.5.0,>=0.4.3 (from typer[all]<1.0,>=0.9->gradio)\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Collecting shellingham<2.0.0,>=1.3.0 (from typer[all]<1.0,>=0.9->gradio)\n",
            "  Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n",
            "Requirement already satisfied: rich<14.0.0,>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio) (13.7.1)\n",
            "Requirement already satisfied: starlette<0.37.0,>=0.36.3 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio) (0.36.3)\n",
            "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (23.2.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (2023.12.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.33.0)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.18.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio) (1.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (2.16.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.24.1->gradio) (1.2.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.3->gradio) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.3->gradio) (2.0.7)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (0.1.2)\n",
            "Building wheels for collected packages: ffmpy\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.2-py3-none-any.whl size=5584 sha256=8449cb9bae66a2cdb6f4eddde5e8a0ea975a0d1a3d5570ef508fe35311fbabfb\n",
            "  Stored in directory: /root/.cache/pip/wheels/bd/65/9a/671fc6dcde07d4418df0c592f8df512b26d7a0029c2a23dd81\n",
            "Successfully built ffmpy\n",
            "Installing collected packages: pydub, ffmpy, websockets, tomlkit, shellingham, semantic-version, ruff, python-multipart, colorama, aiofiles, gradio-client, gradio\n",
            "  Attempting uninstall: websockets\n",
            "    Found existing installation: websockets 12.0\n",
            "    Uninstalling websockets-12.0:\n",
            "      Successfully uninstalled websockets-12.0\n",
            "Successfully installed aiofiles-23.2.1 colorama-0.4.6 ffmpy-0.3.2 gradio-4.21.0 gradio-client-0.12.0 pydub-0.25.1 python-multipart-0.0.9 ruff-0.3.2 semantic-version-2.10.0 shellingham-1.5.4 tomlkit-0.12.0 websockets-11.0.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "action_engine = ActionEngine(llm, embedder)"
      ],
      "metadata": {
        "id": "GsxcI4EjWeXf"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import gradio as gr\n",
        "from selenium import webdriver\n",
        "from selenium.webdriver.chrome.service import Service\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.chrome.options import Options\n",
        "from selenium.webdriver.common.keys import Keys\n",
        "\n",
        "MAX_CHARS = 1500\n",
        "\n",
        "action_engine = ActionEngine(llm, embedder)\n",
        "\n",
        "## Setup chrome options\n",
        "chrome_options = Options()\n",
        "chrome_options.add_argument(\"--headless\") # Ensure GUI is off\n",
        "chrome_options.add_argument(\"--no-sandbox\")\n",
        "chrome_options.add_argument(\"--window-size=1600,900\")\n",
        "\n",
        "# Set path to chrome/chromedriver as per your configuration\n",
        "\n",
        "try:\n",
        "    import google.colab\n",
        "    chrome_options.binary_location = \"/content/chrome-linux64/chrome\"\n",
        "    webdriver_service = Service(\"/content/chromedriver-linux64/chromedriver\")\n",
        "except:\n",
        "    import os.path\n",
        "    homedir = os.path.expanduser(\"~\")\n",
        "    chrome_options.binary_location = f\"{homedir}/chrome-linux64/chrome\"\n",
        "    webdriver_service = Service(f\"{homedir}/chromedriver-linux64/chromedriver\")\n",
        "\n",
        "\n",
        "title = \"\"\"\n",
        "<div align=\"center\">\n",
        "  <h1>🌊 Welcome to LaVague</h1>\n",
        "  <p>Redefining internet surfing by transforming natural language instructions into seamless browser interactions.</p>\n",
        "</div>\n",
        "\"\"\"\n",
        "\n",
        "# Choose Chrome Browser\n",
        "driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)\n",
        "\n",
        "# action_engine = ActionEngine(llm, embedder)\n",
        "\n",
        "def process_url(url):\n",
        "    driver.get(url)\n",
        "    driver.save_screenshot(\"screenshot.png\")\n",
        "    # This function is supposed to fetch and return the image from the URL.\n",
        "    # Placeholder function: replace with actual image fetching logic.\n",
        "    return \"screenshot.png\"\n",
        "\n",
        "def process_instruction(query):\n",
        "    state = driver.page_source\n",
        "    query_engine = action_engine.get_query_engine(state)\n",
        "    streaming_response = query_engine.query(query)\n",
        "\n",
        "    source_nodes = streaming_response.get_formatted_sources(MAX_CHARS)\n",
        "\n",
        "    response = \"\"\n",
        "\n",
        "    for text in streaming_response.response_gen:\n",
        "    # do something with text as they arrive.\n",
        "        response += text\n",
        "        yield response, source_nodes\n",
        "\n",
        "def exec_code(code):\n",
        "    code = code.split(\"```\")[0]\n",
        "    try:\n",
        "        exec(code)\n",
        "        return \"Successful code execution\", code\n",
        "    except Exception as e:\n",
        "        output = f\"Error in code execution: {str(e)}\"\n",
        "        return output, code\n",
        "\n",
        "def update_image_display(img):\n",
        "    driver.save_screenshot(\"screenshot.png\")\n",
        "    url = driver.current_url\n",
        "    return \"screenshot.png\", url\n",
        "\n",
        "def create_demo(base_url, instructions):\n",
        "  with gr.Blocks() as demo:\n",
        "      with gr.Row():\n",
        "          gr.HTML(title)\n",
        "      with gr.Row():\n",
        "          url_input = gr.Textbox(value=base_url, label=\"Enter URL and press 'Enter' to load the page.\")\n",
        "\n",
        "      with gr.Row():\n",
        "          with gr.Column(scale=8):\n",
        "              image_display = gr.Image(label=\"Browser\", interactive=False)\n",
        "\n",
        "          with gr.Column(scale=2):\n",
        "              text_area = gr.Textbox(label=\"Instructions\")\n",
        "              gr.Examples(examples=instructions, inputs=text_area,\n",
        "\n",
        "              )\n",
        "              generate_btn = gr.Button(value=\"Execute\")\n",
        "              code_display = gr.Code(label=\"Generated code\", language=\"python\",\n",
        "                                    lines=5, interactive=False)\n",
        "              with gr.Accordion(label=\"Logs\", open=False) as log_accordion:\n",
        "                  log_display = gr.Textbox(interactive=False)\n",
        "                  source_display = gr.Textbox(label=\"Retrieved nodes\", interactive=False)\n",
        "      # Linking components\n",
        "      url_input.submit(process_url, inputs=url_input, outputs=image_display)\n",
        "      generate_btn.click(process_instruction, inputs=text_area, outputs=[code_display, source_display]).then(\n",
        "          exec_code, inputs=code_display, outputs=[log_display, code_display]\n",
        "      ).then(\n",
        "          update_image_display, inputs=image_display, outputs=[image_display, url_input]\n",
        "      )\n",
        "  demo.launch(share=True)\n",
        ""
      ],
      "metadata": {
        "id": "e5k-Tw5EVZuA"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "base_url = \"https://huggingface.co/\"\n",
        "\n",
        "instructions = [\"Click on the Datasets item on the menu, between Models and Spaces\",\n",
        "                \"Click on the search bar 'Filter by name', type 'The Stack', and press 'Enter'\",\n",
        "                \"Scroll by 500 pixels\",]\n",
        "\n",
        "create_demo(base_url, instructions)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 591
        },
        "id": "9kPolvYHfFmA",
        "outputId": "828680f7-bd0e-456c-ec87-a308a0bb5454"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
            "Running on public URL: https://4b591cac74e73f8c34.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://4b591cac74e73f8c34.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 591
        },
        "id": "MrQ82tgr4_zt",
        "outputId": "0c1f0678-93cd-4cbf-db75-88d9988cdaf8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
            "Running on public URL: https://1de7fa38a8df824289.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://1de7fa38a8df824289.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "base_url = \"https://www.irs.gov/\"\n",
        "\n",
        "instructions = [\"Click on the 'Pay' item on the menu, between 'File' and 'Refunds'\",\n",
        "                \"Click on 'Pay Now with Direct Pay' just below 'Pay from your Bank Account'\",\n",
        "                \"Click on 'Make a Payment', just above 'Answers to common questions'\",]\n",
        "\n",
        "create_demo(base_url, instructions)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "398c2359c137457b94fe7ed7b8c4b939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2600f072828a4abf8e8ed51cd64f2dc2",
              "IPY_MODEL_15c0d6723eba4298b61ddd67c1595f66",
              "IPY_MODEL_d1a4215edd20433482d339c83c9d0060"
            ],
            "layout": "IPY_MODEL_c71a506bf63948c8bfb79b6811705de5"
          }
        },
        "2600f072828a4abf8e8ed51cd64f2dc2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_984f401572a645f7861db035993c806b",
            "placeholder": "​",
            "style": "IPY_MODEL_2ba0655ceb93417d859921bf08b5ec9d",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "15c0d6723eba4298b61ddd67c1595f66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0d9126c291a04e8fb1ce30a7103197d1",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a36c6df55813490a9e0b42b67a698327",
            "value": 4
          }
        },
        "d1a4215edd20433482d339c83c9d0060": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec5d092ea24946279f14f03570a48c98",
            "placeholder": "​",
            "style": "IPY_MODEL_777ac9bad3974a2ab7062644bda6a3c6",
            "value": " 4/4 [00:12&lt;00:00,  2.89s/it]"
          }
        },
        "c71a506bf63948c8bfb79b6811705de5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "984f401572a645f7861db035993c806b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ba0655ceb93417d859921bf08b5ec9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0d9126c291a04e8fb1ce30a7103197d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a36c6df55813490a9e0b42b67a698327": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ec5d092ea24946279f14f03570a48c98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "777ac9bad3974a2ab7062644bda6a3c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}