experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shwu/prayog/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "MODEL = \"sarvamai/OpenHathi-7B-Hi-v0.1-Base\"\n",
    "DATASET = \"sarvamai/samvaad-hi-v1\"\n",
    "\n",
    "# paths\n",
    "CUSTOM_DATA = Path(\"custom_data\")\n",
    "\n",
    "MODELS = CUSTOM_DATA / \"models\"\n",
    "DATA = CUSTOM_DATA / \"datasets\"\n",
    "\n",
    "MODEL_PATH = MODELS / \"openhathi\"\n",
    "DATA_PATH = DATA / \"samvaad\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL: sarvamai/OpenHathi-7B-Hi-v0.1-Base\n",
      "DATASET: sarvamai/samvaad-hi-v1\n",
      "CUSTOM_DATA: /home/shwu/prayog/custom_data\n",
      "MODELS: /home/shwu/prayog/custom_data/models\n",
      "DATA: /home/shwu/prayog/custom_data/datasets\n",
      "MODEL_PATH: /home/shwu/prayog/custom_data/models/openhathi\n",
      "DATA_PATH: /home/shwu/prayog/custom_data/datasets/samvaad\n"
     ]
    }
   ],
   "source": [
    "print(f\"MODEL: {MODEL}\")\n",
    "print(f\"DATASET: {DATASET}\")\n",
    "print(f\"CUSTOM_DATA: {CUSTOM_DATA.absolute()}\")\n",
    "print(f\"MODELS: {MODELS.absolute()}\")\n",
    "print(f\"DATA: {DATA.absolute()}\")\n",
    "print(f\"MODEL_PATH: {MODEL_PATH.absolute()}\")\n",
    "print(f\"DATA_PATH: {DATA_PATH.absolute()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset\n",
    "\n",
    "Explore and format dataset for usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['messages'],\n",
       "        num_rows: 101476\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = load_dataset(DATASET); ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "role_dict = []\n",
    "\n",
    "for messages in ds[\"train\"][\"messages\"]:\n",
    "    convo_stats = {}\n",
    "    for message in messages:\n",
    "        convo_stats[message[\"role\"]] = convo_stats.get(message[\"role\"], 0) + 1\n",
    "    role_dict.append(convo_stats)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAESCAYAAAD38s6aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAn7ElEQVR4nO3deVwU9f8H8NdyLZvC4sWxioJHHiCHoaZ8S/BGvPNbmrdppQYimkalBipopZFiKKbgt69n3lnSN8kjU/MAr59HliuignhwiMQhzO8PH05toDnLLLusr+fjMY8H85mZnfdu9GL87MznoxAEQQAREZkdC2MXQEREhsGAJyIyUwx4IiIzxYAnIjJTDHgiIjPFgCciMlMMeCIiM2Vl7AIMrby8HDdu3ICdnR0UCoWxyyEiqjJBEHDv3j1oNBpYWDz+Ot3sA/7GjRtwdXU1dhlERLLLyMhAo0aNHrvd7APezs4OwMMPwt7e3sjVEBFVXX5+PlxdXcV8exyzD/hH3TL29vYMeCIyK//U7cwvWYmIzBQDnojITDHgiYjMlNn3wROR4ZSXl6OkpMTYZZgda2trWFpaVvl1GPBEpJeSkhJotVqUl5cbuxSz5ODgAGdn5yo9v2PUgD9w4AA++eQTnDhxApmZmdi2bRsGDhwIACgtLcWHH36I7777DpcvX4ZarUb37t2xYMECaDQaY5ZN9MwTBAGZmZmwtLSEq6vrEx+2IWkEQUBhYSGys7MBAC4uLnq/llED/v79+/D29sa4ceMwePBgnW2FhYVITU3FrFmz4O3tjZycHEyZMgX9+/fH8ePHjVQxEQHAgwcPUFhYCI1Gg+eee87Y5ZgdlUoFAMjOzoajo6Pe3TVGDfigoCAEBQVVuk2tVuOHH37QaYuLi0OHDh1w9epVNG7cuNLjiouLUVxcLK7n5+fLVzARAQDKysoAADY2NkauxHw9+sNZWlpaMwNeqry8PCgUCjg4ODx2n5iYGERGRlZfUc+gYVFDjV2CjvWzNxi7hGdWTR3f6fKN341dgqipplml7XJ8tjWm46yoqAgzZ87EsGHDnvhEakREBPLy8sQlIyOjGqskIjIdNeIKvrS0FK+++ioEQUB8fPwT91UqlVAqldVUGRGR6TL5gH8U7unp6fjxxx85ngyRCbt0IbVaz9eiVbtqPV9NY9IB/yjcL126hL1796JevXrGLomIqMYwasAXFBTgt99+E9e1Wi1OnjyJunXrwsXFBUOGDEFqaip27dqFsrIyZGVlAQDq1q3Lb++JiP6BUb9kPX78OHx9feHr6wsACA8Ph6+vL2bPno3r169j586duHbtGnx8fODi4iIuhw4dMmbZRFRDBQQEICQkBHNnz4Nvm3bo4N0RG9ZuQGFhIWZMnQmv570R6N8V+37cLx5z8cKvGDtiHNq28EIH746YFjINd+/eFbfv37sfrw58DT6tffGChx/Gj5qA9Cvp4vZrGdfQrGFzfP/d93h9yHB4NPNEcPe+SD1u+O4sowZ8QEAABEGosCQlJcHNza3SbYIgICAgwJhlE1ENtmbNGtSpWwdbd23FqLGjMDtiDt55KwTt/HyxI3kHXnr5X5geOh1//PEH8vPyMeLVEfDwaIPtu7chce1q3L59GyFvhYqv90fhH3jjzXHY/t02fLXxP7CwUGDi+EkVhnBYtHAxJrw9Hrv+9w3cm7ohbPJUPHjwwKDv1aT74ImI5Obt7Y13wiYDACaGvI0Vy1agTp06GDr84fMdIVNDsPY/63Dh3AX8/NMheHi2wfSI6eLxCxYtwL/avwTt71q4N3NH7+DeOq+/YPECtG/bAZd+/Q0tWz0vto9/+w0Edg8EAEyZPgW9A4OQfiUdzzduabD3yoAnomeKl5eX+LOlpSUc6jigZes/Q7Z+g/oAgDt37uDCufM4cugXtG3hVeF10tOvwr2ZO7SXryD201icSjuFnLt3UV4uAABuXL+hE/CtWrcSf3Z0dHx4jtt35H1zf8OAJ6JnirW1tc66QqGAtZWVzjoAlJcLuF9YiK49umLG++9WeB1Hp4ch/eaYN9GwkQbzP54PJ2dHlJeXI6hrH5SWlursb1XpOQw7EicDnojoMTw8PfD9d9+jkWsjnYB+JOduDi7/fhnRn8xH+47tAQDHj5rOYIg1ZqgCIqLqNnLMCOTm5iJsUhhOnzyN9CvpOLDvAGZMnYmysjKoHdSoU6cONvx3A65or+DQwcOYHxlt7LJFvIInItmY25OlTs5O2LR9Ez6O/hijXx+DkuISNGzUEC8HvAQLCwsoFAp8/kUsomZHIahbHzRt2hSz587C60OGG7t0AIBCEATB2EUYUn5+PtRqNfLy8jjMgUw4miQVFRVBq9XC3d0dtra2xi5HspowmuSTPuOnzTV20RARmSkGPBGRmWLAExGZKQY8EZGZYsATEZkpBjwRkZliwBMRmSkGPBGRmWLAExE9xpFDR9CsYXPk5+UbuxS9cKgCIpLNf3d+Uq3nG9G/4iiPcmrn1w5H0g7Dzt7OoOcxFAY8EdFj2NjYoIFjA2OXoTd20RDRM0PqnKx/76LZvHELfFr74sC+A+jZpRfatvDCmOFjkX0z25hv67EY8ET0TJEyJ2tliv4owpfLV+HTJZ9i/dZ1yLx+AzFzF1Tzu3g6DHgieqY8mpPVvakbJoa8DaVSKc7J6t7UDSFTQ5CTk4ML5y5UenxpaSnmLoiCl3dbeLb1xMgxI3Ho4KFqfhdPhwFPRM8UKXOyVkalUqGJWxNxvYGTo8HnVtUXA56InilS5mStjJW17r0pCoUCpjqtBgOeiMhMMeCJiMyUUQP+wIED6NevHzQaDRQKBbZv366zXRAEzJ49Gy4uLlCpVOjevTsuXbpknGKJiGoYoz7odP/+fXh7e2PcuHEYPHhwhe0ff/wxlixZgjVr1sDd3R2zZs1Cr169cO7cuRo5DySRuTP0k6VVtW/fPgC6c7Ie+GV/hf1+v/5bpT8Pee0VDHntFZ19e/buobOPKTFqwAcFBSEoKKjSbYIgIDY2Fh9++CEGDBgAAPjPf/4DJycnbN++HUOHmtbEz0REpsZk++C1Wi2ysrLQvXt3sU2tVqNjx444fPjwY48rLi5Gfn6+zkJE9Cwy2YDPysoCADg5Oem0Ozk5idsqExMTA7VaLS6urq4GrZOIyFSZbMDrKyIiAnl5eeKSkZFh7JKIiIzCZAPe2dkZAHDz5k2d9ps3b4rbKqNUKmFvb6+zEBE9iyQHfNOmTSt9hDc3NxdNmzaVpSgAcHd3h7OzM1JSUsS2/Px8/PLLL+jUqZNs5yEiMleS76K5cuUKysrKKrQXFxfj+vXrkl6roKAAv/325+1FWq0WJ0+eRN26ddG4cWOEhYVh3rx5aNGihXibpEajwcCBA6WWTUT0zHnqgN+5c6f48/fffw+1Wi2ul5WVISUlBW5ubpJOfvz4cQQGBorr4eHhAIDRo0cjKSkJM2bMwP379/Hmm28iNzcX//rXv5CcnMx74ImInsJTB/yjq2aFQoHRo0frbLO2toabmxsWLVok6eQBAQFPHKRHoVAgKioKUVFRkl6XiIgkBHx5eTmAh33jx44dQ/369Q1WFBERVZ3kPnitVmuIOojIDAyLqt4nzNfP3lCt56tp9BqqICUlBSkpKcjOzhav7B9ZvXq1LIUREVHVSL5NMjIyEj179kRKSgpu376NnJwcnYWIyFRJnXS7rKwM7017D11eDECbZh7o/lIPJH6ZJL5ecVExegf2xvszPhDb0q+kw+t5b3y94evqfnsVSL6CX758OZKSkjBy5EhD1ENEZFBr1qzB+LfHY+uurfh257eYHTEH/0v+AT1798DEkIlIXLka00On46djB2BlZQVnF2csXbEUdeo4IPV4Kj6Y8SEcHRsguH8wlLZKLF66GK/0ewWB3QLQtXtXTAuZBv+X/fHvof829luVHvAlJSXo3LmzIWohIjK4R5NuA8DEkLexYtkKcdJtAAiZGoK1/1mHC+cuwPcFX4RNDxOPdW3sirQTafjum+8Q3D8YANDGsw2mzgjH+9PfR/CAvrh+/QZWrllZ7e+rMpK7aMaPH49169YZohYiIoOTOun2V0lfoX/vAWjftj3atvDChrUbceNGps5rjn/rDbg1dcdXiV9hwaIY1KlbpxreyT+TfAVfVFSEhIQE7NmzB15eXhUmsF28eLFsxRERyU3KpNvf7NiFmLkL8P6sCPj6+aJWrVpYGf8lTqWd0nmNO7fv4MplLSwtLXFFm44ugTAJkgP+9OnT8PHxAQCcPXtWZ9ujD4aIyBycOHYC7V5ohxFjRohtV9OvVthv5rT38Hyrlnh12L/x/rsfwP+lzmjeonl1llopyQG/d+9eQ9RBRGRy3NzdsG3zNhzYdwCurq7YtmU7Tp86rTPPxFdJXyHtRBq+/WEXNA012JuyD+HvhGPzN5thY2NjxOpNeLhgIiJjGzZiKHoF9ULoxCkY3O8V5ObkYMTo4eL233/7HQvmLkRkdCQ0DTUAgKjoSNy9m4PPPok1UtV/UghPGgymEoGBgU/sivnxxx+rXJSc8vPzoVarkZeXx7HhZVLdTyv+Ez7NWP2Kioqg1Wrh7u5eIwf/++uk28bWVNOs0vYnfcZPm2uSu2ge9b8/UlpaipMnT+Ls2bMVBiEjIiLjkRzwn332WaXtH330EQoKCqpcEBERyUO2PvgRI0ZwHBoiIhMiW8AfPny4RvbFERGZK8ldNIMHD9ZZFwQBmZmZOH78OGbNmiVbYURk+iTeo0ESyPHZSg74v07VBwAWFhZo2bIloqKi0LNnzyoXRESmz9LSEsDDsalUKpWRqzFPhYWFACo+eSuF5IBPTEzU+2REZB6srKzw3HPP4datW7C2toaFRc16pKbsQZmxSxAVFRXprAuCgMLCQmRnZ8PBwUH8Y6oPvSb8AIATJ07g/PnzAAAPDw/4+vrqXQQR1SwKhQIuLi7QarVIT083djmS3cq9ZewSRA/uV/7HxsHBAc7OzlV6bckBn52djaFDh2Lfvn1wcHAAAOTm5iIwMBAbNmxAgwYNqlQQEdUMNjY2aNGiBUpKSoxdimRxy5YauwTRoskVB2i0trau0pX7I5IDPiQkBPfu3cP//d//oXXr1gCAc+fOYfTo0QgNDcX69eurXBQR1QwWFhY18u65u/fvGrsEkSE/P8kBn5ycjD179ojhDgBt2rTBsmXL+CUrEZEJkfzNSHl5eaXf6lpbW1eYgJuIiIxHcsB37doVU6ZMwY0bN8S269evY+rUqejWrZusxRERkf4kB3xcXBzy8/Ph5uaGZs2aoVmzZnB3d0d+fj6WLjWdLy6IiJ51kvvgXV1dkZqaij179uDChQsAgNatW6N79+6yF1dWVoaPPvoI//3vf5GVlQWNRoMxY8bgww8/5OxRRET/QK/74BUKBXr06IEePXrIXY+OhQsXIj4+HmvWrIGHhweOHz+OsWPHQq1WIzQ01KDnJiKq6SR30YSGhmLJkiUV2uPi4hAWFiZHTaJDhw5hwIABCA4OhpubG4YMGYKePXvi6NGjsp6HiMgcSQ74LVu2wN/fv0J7586dsXnzZlmK+utrpqSk4NdffwUAnDp1CgcPHkRQUNBjjykuLkZ+fr7OQkT0LJLcRXPnzp0KA44BgL29PW7fvi1LUY+89957yM/PR6tWrWBpaYmysjLMnz8fw4cPf+wxMTExiIyMlLUOIqKaSPIVfPPmzZGcnFyhfffu3WjatKksRT2yadMmrF27FuvWrUNqairWrFmDTz/9FGvWrHnsMREREcjLyxOXjIwMWWsiIqopJF/Bh4eH45133sGtW7fQtWtXAEBKSgoWLVqE2NhYWYt799138d5772Ho0IeTPLdt2xbp6emIiYl57PyvSqUSSqVS1jqIiGoiyQE/btw4FBcXY/78+Zg7dy4AwM3NDfHx8Rg1apSsxRUWFlYYhtTS0pJPzBIRPQW9bpOcOHEiJk6ciFu3bkGlUqF27dpy1wUA6NevH+bPn4/GjRvDw8MDaWlpWLx4McaNG2eQ8xERmRO9x4MHYPChgZcuXYpZs2Zh0qRJyM7OhkajwVtvvYXZs2cb9LxEROagSgFvaHZ2doiNjZW9b5+I6FlQs+bZIiKip8aAJyIyUwx4IiIzpVcffEpKClJSUpCdnV3hlsXVq1fLUhgREVWN5ICPjIxEVFQU/Pz84OLiwmF7iYhMlOSAX758OZKSkjBy5EhD1ENERDKR3AdfUlKCzp07G6IWIiKSkeSAHz9+PNatW2eIWoiISEaSu2iKioqQkJCAPXv2wMvLC9bW1jrbFy9eLFtxRESkP8kBf/r0afj4+AAAzp49q7ONX7gSEZkOyQG/d+9eQ9RBREQyq9KDTteuXcO1a9fkqoWIiGQkOeDLy8sRFRUFtVqNJk2aoEmTJnBwcMDcuXM5TjsRkQmR3EXzwQcfYNWqVViwYIE4+fbBgwfx0UcfoaioCPPnz5e9SCIikk5ywK9ZswZffvkl+vfvL7Z5eXmhYcOGmDRpEgOeiMhESO6iuXv3Llq1alWhvVWrVrh7964sRRERUdVJDnhvb2/ExcVVaI+Li4O3t7csRRERUdVJ7qL5+OOPERwcjD179qBTp04AgMOHDyMjIwPfffed7AUSEZF+JF/Bd+nSBb/++isGDRqE3Nxc5ObmYvDgwbh48SJeeuklQ9RIRER60Gs8eI1Gwy9TiYhM3FMF/OnTp+Hp6QkLCwucPn36ift6eXnJUhgREVXNUwW8j48PsrKy4OjoCB8fHygUCgiCUGE/hUKBsrIy2YskIiLpnirgtVotGjRoIP5MRESm76kCvkmTJuLP6enp6Ny5M6ysdA998OABDh06pLMvEREZj+S7aAIDAyt9oCkvLw+BgYGyFEVERFUnOeAFQah03Pc7d+6gVq1ashRFRERV99S3SQ4ePBjAwy9Sx4wZA6VSKW4rKyvD6dOnDTJX6/Xr1zFz5kzs3r0bhYWFaN68ORITE+Hn5yf7uYiIzMlTB7xarQbw8Arezs4OKpVK3GZjY4MXX3wREyZMkLW4nJwc+Pv7IzAwELt370aDBg1w6dIl1KlTR9bzEBGZo6cO+MTERACAm5sbpk+fXi3dMQsXLoSrq6t4bgBwd3d/4jHFxcUoLi4W1/Pz8w1WHxGRKZPcBz9nzpxq62vfuXMn/Pz88O9//xuOjo7w9fXFypUrn3hMTEwM1Gq1uLi6ulZLrUREpkavoQo2b96MTZs24erVqygpKdHZlpqaKkthAHD58mXEx8cjPDwc77//Po4dO4bQ0FDY2Nhg9OjRlR4TERGB8PBwcT0/P58hT0TPJMlX8EuWLMHYsWPh5OSEtLQ0dOjQAfXq1cPly5cRFBQka3Hl5eVo164doqOj4evrizfffBMTJkzA8uXLH3uMUqmEvb29zkJE9CySHPBffPEFEhISsHTpUtjY2GDGjBn44YcfEBoairy8PFmLc3FxQZs2bXTaWrdujatXr8p6HiIicyQ54K9evSreDqlSqXDv3j0AwMiRI7F+/XpZi/P398fFixd12n799Vc+LUtE9BQkB7yzs7P4JGvjxo1x5MgRAA/HqKlsALKqmDp1Ko4cOYLo6Gj89ttvWLduHRISEjB58mRZz0NEZI4kB3zXrl2xc+dOAMDYsWMxdepU9OjRA6+99hoGDRoka3Ht27fHtm3bsH79enh6emLu3LmIjY3F8OHDZT0PEZE5knwXTUJCAsrLywEAkydPRr169XDo0CH0798fb731luwF9u3bF3379pX9dYmIzJ3kgLewsICFxZ8X/kOHDsXQoUNlLYqIiKpOchdNcnIyDh48KK4vW7YMPj4+eP3115GTkyNrcUREpD/JAf/uu++Kj/+fOXMG4eHh6NOnD7Rarc4DRkREZFySu2i0Wq14b/qWLVvQr18/REdHIzU1FX369JG9QCIi0o/kK3gbGxsUFhYCAPbs2YOePXsCAOrWrcuBvYiITIjkK3h/f3+Eh4fD398fR48excaNGwE8fACpUaNGshdIRET6kXwFv2zZMlhbW2Pz5s2Ij49Hw4YNAQC7d+9G7969ZS+QiIj0I+kK/sGDB9i3bx9WrlwJZ2dnnW2fffaZrIUREVHVSLqCt7Kywttvv60zoQYREZkmyV00HTp0QFpamiFqISIiGUn+knXSpEmYNm0arl27hhdeeKHC7E5eXl6yFUdERPqTHPCPhiUIDQ0V2xQKBQRBgEKhQFlZmXzVERGR3vR60ImIiEyf5IDnZBtERDWD5C9ZAeCrr76Cv78/NBoN0tPTAQCxsbHYsWOHrMUREZH+JAd8fHy8OMBYbm6u2Ofu4OCA2NhYuesjIiI9SQ74pUuXYuXKlfjggw9gaWkptvv5+eHMmTOyFkdERPqTHPBarRa+vr4V2pVKJe7fvy9LUUREVHWSA97d3R0nT56s0J6cnIzWrVvLURMREclA8l004eHhmDx5MoqKiiAIAo4ePYr169cjJiYGX375pSFqJCIiPUgO+PHjx0OlUuHDDz9EYWEhXn/9dWg0Gnz++eecm5WIyIRIDngAGD58OIYPH47CwkIUFBTA0dFR7rqIiKiKJPfBz5s3T3ya9bnnnmO4ExGZKMkB//XXX6N58+bo3LkzvvjiC9y+fdsQdRERURVJDvhTp07h9OnTCAgIwKeffgqNRoPg4GCsW7dOnKuViIiMT6+hCjw8PBAdHY3Lly9j7969cHNzQ1hYWIVZnuS2YMECKBQKhIWFGfQ8RETmQK+A/6tatWpBpVLBxsYGpaWlctRUqWPHjmHFihUcb56I6CnpFfBarRbz58+Hh4cH/Pz8kJaWhsjISGRlZcldHwCgoKAAw4cPx8qVK1GnTh2DnIOIyNxIvk3yxRdfxLFjx+Dl5YWxY8di2LBhaNiwoSFqE02ePBnBwcHo3r075s2b98R9i4uLdeaMzc/PN2htRESmSnLAd+vWDatXr0abNm0MUU8FGzZsQGpqKo4dO/ZU+8fExCAyMtLAVRERmT7JXTTz58+vtnDPyMjAlClTsHbtWtja2j7VMREREcjLyxOXjIwMA1dJRGSaJF/Bl5WVISkpCSkpKcjOzkZ5ebnO9h9//FG24k6cOIHs7Gy0a9dO5/wHDhxAXFwciouLdYYsBh6OaqlUKmWrgYioppIc8FOmTEFSUhKCg4Ph6ekJhUJhiLoAPOwO+vsY82PHjkWrVq0wc+bMCuFORER/khzwGzZswKZNm9CnTx9D1KPDzs4Onp6eOm21atVCvXr1KrQTEZEuyX3wNjY2aN68uSFqISIiGUm+gp82bRo+//xzxMXFGbR75nH27dtX7eckIqqJJAf8wYMHsXfvXuzevRseHh6wtrbW2b5161bZiiMiIv1JDngHBwcMGjTIELWYnEsXUo1dgqhFq3b/vBPRX/D3lyQHfGJioiHqICIimek1oxMA3Lp1CxcvXgQAtGzZEg0aNJCtKCIiqjrJd9Hcv38f48aNg4uLC15++WW8/PLL0Gg0eOONNzgePBGRCZEc8OHh4di/fz+++eYb5ObmIjc3Fzt27MD+/fsxbdo0Q9RIRER6kNxFs2XLFmzevBkBAQFiW58+faBSqfDqq68iPj5ezvqIiEhPkq/gCwsL4eTkVKHd0dGRXTRERCZEcsB36tQJc+bMQVFRkdj2xx9/IDIyEp06dZK1OCIi0p/kLprPP/8cvXr1QqNGjeDt7Q3g4UTctra2+P7772UvkIiI9CM54D09PXHp0iWsXbsWFy5cAAAMGzYMw4cPh0qlkr1AIiLSj173wT/33HOYMGGC3LUQEZGMJPfBx8TEYPXq1RXaV69ejYULF8pSFBERVZ3kgF+xYgVatWpVod3DwwPLly+XpSgiIqo6yQGflZUFFxeXCu0NGjRAZmamLEUREVHVSQ54V1dX/PzzzxXaf/75Z2g0GlmKIiKiqpP8JeuECRMQFhaG0tJSdO3aFQCQkpKCGTNmcKgCIiITIjng3333Xdy5cweTJk1CSUkJAMDW1hYzZ85ERESE7AUSEZF+JAe8QqHAwoULMWvWLJw/fx4qlQotWrSAUqk0RH1ERKQnvceDr127Ntq3by9nLUREJCPJX7ISEVHNwIAnIjJTDHgiIjPFgCciMlMMeCIiM8WAJyIyUyYd8DExMWjfvj3s7Ozg6OiIgQMH4uLFi8Yui4ioRjDpgN+/fz8mT56MI0eO4IcffkBpaSl69uyJ+/fvG7s0IiKTp/eDTtUhOTlZZz0pKQmOjo44ceIEXn75ZSNVRURUM5h0wP9dXl4eAKBu3bqP3ae4uBjFxcXien5+vsHrIiIyRSbdRfNX5eXlCAsLg7+/Pzw9PR+7X0xMDNRqtbi4urpWY5VERKajxgT85MmTcfbsWWzYsOGJ+0VERCAvL09cMjIyqqlCIiLTUiO6aN555x3s2rULBw4cQKNGjZ64r1Kp5MiWREQw8YAXBAEhISHYtm0b9u3bB3d3d2OXRERUY5h0wE+ePBnr1q3Djh07YGdnh6ysLACAWq2GSqUycnVERKbNpPvg4+PjkZeXh4CAALi4uIjLxo0bjV0aEZHJM+kreEEQjF0CEVGNZdJX8EREpD8GPBGRmWLAExGZKQY8EZGZYsATEZkpBjwRkZliwBMRmSkGPBGRmWLAExGZKQY8EZGZYsATEZkpkx6Lhv70352fGLsEIr3x99c4eAVPRGSmGPBERGaKAU9EZKYY8EREZooBT0RkphjwRERmigFPRGSmGPBERGaKAU9EZKYY8EREZooBT0RkphjwRERmigFPRGSmGPBERGaqRgT8smXL4ObmBltbW3Ts2BFHjx41dklERCbP5AN+48aNCA8Px5w5c5Camgpvb2/06tUL2dnZxi6NiMikmfyEH4sXL8aECRMwduxYAMDy5cvx7bffYvXq1Xjvvfcq7F9cXIzi4mJxPS8vDwCQn58v+dwFBQV6Vi2/PwqLjF2CqLSo1Ngl6NDnv+2zgL+/j2dKv8P6/P4+OkYQhCfvKJiw4uJiwdLSUti2bZtO+6hRo4T+/ftXesycOXMEAFy4cOFi9ktGRsYTM9Skr+Bv376NsrIyODk56bQ7OTnhwoULlR4TERGB8PBwcb28vBx3795FvXr1oFAoDFqvqcvPz4erqysyMjJgb29v7HKIJOHv758EQcC9e/eg0WieuJ9JB7w+lEollEqlTpuDg4NxijFR9vb2z/z/IFRz8ff3IbVa/Y/7mPSXrPXr14elpSVu3ryp037z5k04OzsbqSoioprBpAPexsYGL7zwAlJSUsS28vJypKSkoFOnTkasjIjI9Jl8F014eDhGjx4NPz8/dOjQAbGxsbh//754Vw09PaVSiTlz5lTowiKqCfj7K51CEP7pPhvji4uLwyeffIKsrCz4+PhgyZIl6Nixo7HLIiIyaTUi4ImISDqT7oMnIiL9MeCJiMwUA56IyEwx4InIpCgUCmzfvt3YZZgFBnwNFBAQgLCwsArtSUlJfGqXqtXhw4dhaWmJ4OBg2V4zMzMTQUFBT7Wvof4YXLlyBQqFAidPnpT9tasTA56eSmmp6Yy+R6Zj1apVCAkJwYEDB3Djxg1ZXtPZ2Zn3usuEAW+m9u3bhw4dOqBWrVpwcHCAv78/0tPTxe07duxAu3btYGtri6ZNmyIyMhIPHjwQtysUCsTHx6N///6oVasW5s+fb4y3QSasoKAAGzduxMSJExEcHIykpCRxW05ODoYPH44GDRpApVKhRYsWSExMBACUlJTgnXfegYuLC2xtbdGkSRPExMSIx/71qvxJ+7q5uQEABg0aBIVCIa7//vvvGDBgAJycnFC7dm20b98ee/bs0andzc0N0dHRGDduHOzs7NC4cWMkJCSI293d3QEAvr6+UCgUCAgIkPGTqz4MeDP04MEDDBw4EF26dMHp06dx+PBhvPnmm+Jomj/99BNGjRqFKVOm4Ny5c1ixYgWSkpIqhPhHH32EQYMG4cyZMxg3bpwx3gqZsE2bNqFVq1Zo2bIlRowYgdWrV4vjk8+aNQvnzp3D7t27cf78ecTHx6N+/foAgCVLlmDnzp3YtGkTLl68iLVr14rh/HdP2vfYsWMAgMTERGRmZorrBQUF6NOnD1JSUpCWlobevXujX79+uHr1qs5rL1q0CH5+fkhLS8OkSZMwceJEXLx4EQDEWeP27NmDzMxMbN26VdbPrtpUfdR2qm5dunQRpkyZUqE9MTFRUKvVwp07dwQAwr59+yo9vlu3bkJ0dLRO21dffSW4uLiI6wCEsLAwWesm89K5c2chNjZWEARBKC0tFerXry/s3btXEARB6NevnzB27NhKjwsJCRG6du0qlJeXV7odgDgHhJR9n8TDw0NYunSpuN6kSRNhxIgR4np5ebng6OgoxMfHC4IgCFqtVgAgpKWl/eNrmzJewZuhunXrYsyYMejVqxf69euHzz//HJmZmeL2U6dOISoqCrVr1xaXCRMmIDMzE4WFheJ+fn5+xiifaoCLFy/i6NGjGDZsGADAysoKr732GlatWgUAmDhxIjZs2AAfHx/MmDEDhw4dEo8dM2YMTp48iZYtWyI0NBT/+9//HnseKfs+UlBQgOnTp6N169ZwcHBA7dq1cf78+QpX8F5eXuLPCoUCzs7OZjcVKAO+BrK3txenIvyr3NxccYzoxMREHD58GJ07d8bGjRvx/PPP48iRIwAe/g8QGRmJkydPisuZM2dw6dIl2Nraiq9Xq1at6nlDVOOsWrUKDx48gEajgZWVFaysrBAfH48tW7YgLy8PQUFBSE9Px9SpU3Hjxg1069YN06dPBwC0a9cOWq0Wc+fOxR9//IFXX30VQ4YMqfQ8UvZ9ZPr06di2bRuio6Px008/4eTJk2jbti1KSkp09rO2ttZZVygUKC8vr8KnYoKM/U8Ikm769OmCl5dXhfaRI0cK3bt3r/SYF198UQgJCREE4eE/rceNG/fEc+Ap/+lLz57S0lLByclJWLRokXDmzBmdpVmzZmI3x18tX75csLOzq/T1kpOTBQDCnTt3BEF48u/e3/e1trYWNm/erLOPp6enEBUVJa7fu3dPUKvVOt2aTZo0ET777DOd47y9vYU5c+YIgiAI169fFwAIx48ff9JHYfJMfrhgqmjixImIi4tDaGgoxo8fD6VSiW+//Rbr16/HN998A61Wi4SEBPTv3x8ajQYXL17EpUuXMGrUKADA7Nmz0bdvXzRu3BhDhgyBhYUFTp06hbNnz2LevHlGfndk6nbt2oWcnBy88cYbFWYVeuWVV7Bq1SrcuHEDL7zwAjw8PFBcXIxdu3ahdevWAIDFixfDxcUFvr6+sLCwwNdffw1nZ+dKn+H4p33d3NyQkpICf39/KJVK1KlTBy1atMDWrVvRr18/KBQKzJo1S/KVuaOjI1QqFZKTk9GoUSPY2to+1QxKJsfYf2FIP0ePHhV69OghNGjQQFCr1ULHjh3Fq56srCxh4MCBgouLi2BjYyM0adJEmD17tlBWViYen5ycLHTu3FlQqVSCvb290KFDByEhIUHcDl7B02P07dtX6NOnT6XbfvnlFwGAEBkZKbRu3VpQqVRC3bp1hQEDBgiXL18WBEEQEhISBB8fH6FWrVqCvb290K1bNyE1NVV8jb/+7v3Tvjt37hSaN28uWFlZCU2aNBEE4eEXpIGBgYJKpRJcXV2FuLi4Cjcm/NMVvCAIwsqVKwVXV1fBwsJC6NKli96flzFxuGAiIjPFL1mJiMwUA56IyEwx4ImIzBQDnojITDHgiYjMFAOeiMhMMeCJiMwUA56IyEwx4ImIzBQDnojITDHgiYjM1P8DybmOh8Gznp8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 400x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df = pd.DataFrame(role_dict)\n",
    "\n",
    "# Calculating mean, min, and max conversation exchanges between user and assistant\n",
    "user_mean = df['user'].mean()\n",
    "user_min = df['user'].min()\n",
    "user_max = df['user'].max()\n",
    "\n",
    "assistant_mean = df['assistant'].mean()\n",
    "assistant_min = df['assistant'].min()\n",
    "assistant_max = df['assistant'].max()\n",
    "\n",
    "# Plotting\n",
    "labels = ['User', 'Assistant']\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "width = 0.2  # the width of the bars\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(4,3))\n",
    "\n",
    "# colors - https://coolors.co/palette/dad7cd-a3b18a-588157-3a5a40-344e41\n",
    "bars_mean = ax.bar(x - width, [user_mean, assistant_mean], width, label='mean', color='#DAD7CD')\n",
    "bars_min = ax.bar(x, [user_min, assistant_min], width, label='min', color='#A3B18A')\n",
    "bars_max = ax.bar(x + width, [user_max, assistant_max], width, label='max', color='#588157')\n",
    "\n",
    "ax.set_ylabel('conversation count')\n",
    "# ax.set_title('mean, min, and max conversation counts between User and Assistant')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(labels)\n",
    "ax.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare DataSets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We prepare two datasets for testing purposes; One following the tulu format and the other follows alpaca format (both are chat/instruction formats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHAT = \"chat\"\n",
    "TULU = \"tulu\"\n",
    "EOL = \"\\n\"\n",
    "SPACE = \" \"\n",
    "\n",
    "ROLE_TOS = {\n",
    "    CHAT: {\n",
    "        \"user\": f\"user:{SPACE}\",\n",
    "        \"system\": f\"system:{SPACE}\",\n",
    "        \"assistant\": f\"assistant:{SPACE}\",\n",
    "    },\n",
    "    TULU: {\n",
    "        \"user\": f\"<|user|>{EOL}\",\n",
    "        \"system\": f\"<|system|>{EOL}\",\n",
    "        \"assistant\": f\"<|assistant|>{EOL}\",\n",
    "    }\n",
    "}\n",
    "\n",
    "# 1. Tulu format\n",
    "# Adapted from AI4Bharat/IndicInstruct - https://github.com/AI4Bharat/IndicInstruct/blob/0d22aa33f6322d917bb83876e5fa877fb9edb2f2/eval/templates.py#L1\n",
    "\n",
    "def create_prompt_with_tulu_chat_format(messages, bos=\"<s>\", eos=\"</s>\", add_bos=True):\n",
    "    formatted_text = \"\"\n",
    "    for message in messages:\n",
    "        if message[\"role\"] == \"system\":\n",
    "            formatted_text += ROLE_TOS[TULU][message[\"role\"]] + message[\"content\"] + EOL\n",
    "        elif message[\"role\"] == \"user\":\n",
    "            formatted_text += ROLE_TOS[TULU][message[\"role\"]] + message[\"content\"] + EOL\n",
    "        elif message[\"role\"] == \"assistant\":\n",
    "            formatted_text += ROLE_TOS[TULU][message[\"role\"]] + \\\n",
    "                message[\"content\"].strip() + eos + EOL\n",
    "        else:\n",
    "            raise ValueError(\n",
    "                \"Tulu chat template only supports 'system', 'user' and 'assistant' roles. Invalid role: {}.\".format(\n",
    "                    message[\"role\"]\n",
    "                )\n",
    "            )\n",
    "\n",
    "    formatted_text = bos + formatted_text if add_bos else formatted_text\n",
    "    return formatted_text\n",
    "\n",
    "# 2. Chat format\n",
    "\n",
    "def create_prompt_with_chat_format(messages):\n",
    "    formatted_text = \"\"\n",
    "    for message in messages:\n",
    "        if message[\"role\"] in { \"system\", \"user\", \"assistant\"}:\n",
    "            formatted_text += ROLE_TOS[CHAT][message[\"role\"]] + message[\"content\"] + EOL\n",
    "        else:\n",
    "            raise ValueError(\n",
    "                \"Alpaca template only supports 'system', 'user' and 'assistant' roles. Invalid role: {}.\".format(\n",
    "                    message[\"role\"]\n",
    "                )\n",
    "            )\n",
    "    return formatted_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_funcs = [\n",
    "    (CHAT, lambda ds_row: {\n",
    "        \"text\": create_prompt_with_chat_format(messages=ds_row[\"messages\"])\n",
    "    }), \n",
    "    (TULU, lambda ds_row: {\n",
    "        \"text\": create_prompt_with_tulu_chat_format(messages=ds_row[\"messages\"])\n",
    "    })\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# push to hub\n",
    "    \n",
    "for (f_name, func) in map_funcs:\n",
    "    ds = ds = load_dataset(DATASET)\n",
    "    ds[\"train\"] = ds[\"train\"].map(func)\n",
    "    ds.push_to_hub(f\"shwubham/samvaad-hi-v1-{f_name}-format\")\n",
    "\n",
    "    for sample in ds[\"train\"][\"text\"][-2:]:\n",
    "        print(sample)\n",
    "        print(\"=\" * 100, \"\\n\")\n",
    "    \n",
    "    # save dataset\n",
    "    ds.save_to_disk(DATA_PATH / f_name)\n",
    "\n",
    "    del ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>challenge</th>\n",
       "      <th>model</th>\n",
       "      <th>shot</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mmlu</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-c...</td>\n",
       "      <td>0shot</td>\n",
       "      <td>0.267626</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mmlu</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-t...</td>\n",
       "      <td>0shot</td>\n",
       "      <td>0.312277</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mmlu</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-t...</td>\n",
       "      <td>5shot</td>\n",
       "      <td>0.309358</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mmlu</td>\n",
       "      <td>openhathi-4bit-quantized-samvaad-hi-v1-chat-fo...</td>\n",
       "      <td>0shot</td>\n",
       "      <td>0.268979</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mmlu</td>\n",
       "      <td>openhathi-4bit-quantized-samvaad-hi-v1-tulu-fo...</td>\n",
       "      <td>0shot</td>\n",
       "      <td>0.284575</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>arc-challenge</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-ch...</td>\n",
       "      <td>5shot</td>\n",
       "      <td>0.302901</td>\n",
       "      <td>0.202335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>arc-challenge</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base</td>\n",
       "      <td>0shot</td>\n",
       "      <td>0.402730</td>\n",
       "      <td>0.359230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>arc-challenge</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-c...</td>\n",
       "      <td>5shot</td>\n",
       "      <td>0.261092</td>\n",
       "      <td>0.090591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>arc-challenge</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base</td>\n",
       "      <td>5shot</td>\n",
       "      <td>0.459044</td>\n",
       "      <td>0.438521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>arc-challenge</td>\n",
       "      <td>OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-ch...</td>\n",
       "      <td>0shot</td>\n",
       "      <td>0.374573</td>\n",
       "      <td>0.248736</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>192 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         challenge                                              model   shot  \\\n",
       "0             mmlu  OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-c...  0shot   \n",
       "1             mmlu  OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-t...  0shot   \n",
       "2             mmlu  OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-t...  5shot   \n",
       "3             mmlu  openhathi-4bit-quantized-samvaad-hi-v1-chat-fo...  0shot   \n",
       "4             mmlu  openhathi-4bit-quantized-samvaad-hi-v1-tulu-fo...  0shot   \n",
       "..             ...                                                ...    ...   \n",
       "187  arc-challenge  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-ch...  5shot   \n",
       "188  arc-challenge                          OpenHathi-7B-Hi-v0.1-Base  0shot   \n",
       "189  arc-challenge  OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-c...  5shot   \n",
       "190  arc-challenge                          OpenHathi-7B-Hi-v0.1-Base  5shot   \n",
       "191  arc-challenge  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-ch...  0shot   \n",
       "\n",
       "     accuracy        f1  \n",
       "0    0.267626 -1.000000  \n",
       "1    0.312277 -1.000000  \n",
       "2    0.309358 -1.000000  \n",
       "3    0.268979 -1.000000  \n",
       "4    0.284575 -1.000000  \n",
       "..        ...       ...  \n",
       "187  0.302901  0.202335  \n",
       "188  0.402730  0.359230  \n",
       "189  0.261092  0.090591  \n",
       "190  0.459044  0.438521  \n",
       "191  0.374573  0.248736  \n",
       "\n",
       "[192 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "# RESULTS = Path(\"results\")\n",
    "\n",
    "from pathlib import Path\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "class Metrics:\n",
    "    def __init__(self, accuracy, precision, recall, f1):\n",
    "        self.accuracy = accuracy\n",
    "        self.precision = precision\n",
    "        self.recall = recall\n",
    "        self.f1 = f1\n",
    "    \n",
    "    @classmethod\n",
    "    def from_json(cls, json_file):\n",
    "        with open(json_file, \"r\") as f:\n",
    "            data = json.load(f)\n",
    "            return cls(\n",
    "                accuracy=data[\"accuracy\"] if data.get(\"accuracy\", None) is not None else data.get(\"average_acc\"),\n",
    "                precision=data[\"precision\"] if data.get(\"precision\", None) is not None else -1,\n",
    "                recall=data[\"recall\"] if data.get(\"recall\", None) is not None else -1,\n",
    "                f1=data[\"f1\"] if data.get(\"f1\", None) is not None else -1\n",
    "            )\n",
    "\n",
    "# Define the results directory\n",
    "results_dir = Path(\"/home/shwu/eval_results\")\n",
    "\n",
    "# Create a dictionary to store the results\n",
    "results_data = {\n",
    "    \"challenge\": [],\n",
    "    \"model\": [],\n",
    "    \"shot\": [],\n",
    "    \"accuracy\": [],\n",
    "    # \"precision\": [],\n",
    "    # \"recall\": [],\n",
    "    \"f1\": [],\n",
    "}\n",
    "\n",
    "# Loop through each challenge directory\n",
    "for challenge_dir in results_dir.iterdir():\n",
    "    if challenge_dir.is_dir():\n",
    "        challenge_name = challenge_dir.name\n",
    "        # Loop through each model directory within the challenge directory\n",
    "        for model_dir in challenge_dir.iterdir():\n",
    "            if model_dir.is_dir():\n",
    "                model_name = model_dir.name\n",
    "                # Read metrics.json file\n",
    "                metrics_file = model_dir / \"metrics.json\"\n",
    "                if metrics_file.exists():\n",
    "                    metrics = Metrics.from_json(metrics_file)\n",
    "                    # Append data to results dictionary\n",
    "                    results_data[\"challenge\"].append(challenge_name)    \n",
    "                    results_data[\"model\"].append(model_name.replace(\"-awq-4\", \"-4bit-quantized\")[:-6]) # when displaying shots a unique column\n",
    "                    # results_data[\"model\"].append(model_name.replace(\"-awq-4\", \"-4bit-quantized\")) # when not displaying shots\n",
    "                    results_data[\"shot\"].append(model_name[-5:])\n",
    "                    results_data[\"accuracy\"].append(metrics.accuracy)\n",
    "                    # results_data[\"precision\"].append(metrics.precision)\n",
    "                    # results_data[\"recall\"].append(metrics.recall)\n",
    "                    results_data[\"f1\"].append(metrics.f1)\n",
    "\n",
    "# Create DataFrame from results dictionary\n",
    "results_df = pd.DataFrame(results_data)\n",
    "\n",
    "# Print the DataFrame\n",
    "# print(results_df)\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop all rows that have LoRA_openhathi_samvaad_chat_v1 in model name\n",
    "results_df = results_df[~results_df[\"model\"].str.contains(\"LoRA_openhathi_samvaad_chat_v1\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "Challenge: ('arc-challenge', '0shot')\n",
      "\n",
      "    challenge                                                    model  shot  accuracy       f1\n",
      "arc-challenge                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.402730 0.359230\n",
      "arc-challenge OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.283276 0.156770\n",
      "arc-challenge OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.317406 0.202435\n",
      "arc-challenge  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.374573 0.248736\n",
      "arc-challenge       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.373720 0.329699\n",
      "arc-challenge       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.375427 0.333539\n",
      "arc-challenge                        openhathi-4bit-quantized-wikitext 0shot  0.367747 0.315462\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('arc-challenge', '5shot')\n",
      "\n",
      "    challenge                                                    model  shot  accuracy       f1\n",
      "arc-challenge                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.459044 0.438521\n",
      "arc-challenge OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.261092 0.090591\n",
      "arc-challenge OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.260239 0.116684\n",
      "arc-challenge  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.302901 0.202335\n",
      "arc-challenge       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.414676 0.384616\n",
      "arc-challenge       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.429181 0.402727\n",
      "arc-challenge                        openhathi-4bit-quantized-wikitext 5shot  0.451365 0.434281\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('arc-challenge-hi', '0shot')\n",
      "\n",
      "       challenge                                                    model  shot  accuracy       f1\n",
      "arc-challenge-hi                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.299488 0.248502\n",
      "arc-challenge-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.269625 0.115558\n",
      "arc-challenge-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.290956 0.180217\n",
      "arc-challenge-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.288396 0.236710\n",
      "arc-challenge-hi       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.261092 0.180623\n",
      "arc-challenge-hi       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.262799 0.191640\n",
      "arc-challenge-hi                        openhathi-4bit-quantized-wikitext 0shot  0.290102 0.232515\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('arc-challenge-hi', '5shot')\n",
      "\n",
      "       challenge                                                    model  shot  accuracy       f1\n",
      "arc-challenge-hi                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.320819 0.295926\n",
      "arc-challenge-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.261092 0.084548\n",
      "arc-challenge-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.266212 0.127336\n",
      "arc-challenge-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.271331 0.173403\n",
      "arc-challenge-hi       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.285836 0.245928\n",
      "arc-challenge-hi       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.301195 0.273684\n",
      "arc-challenge-hi                        openhathi-4bit-quantized-wikitext 5shot  0.322526 0.307961\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('arc-easy', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      " arc-easy                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.570286 0.427362\n",
      " arc-easy OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.313973 0.180325\n",
      " arc-easy OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.361111 0.218348\n",
      " arc-easy  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.440657 0.296996\n",
      " arc-easy       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.521044 0.384793\n",
      " arc-easy       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.533249 0.392761\n",
      " arc-easy                        openhathi-4bit-quantized-wikitext 0shot  0.497054 0.370104\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('arc-easy', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      " arc-easy                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.619108 0.481674\n",
      " arc-easy OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.250421 0.221121\n",
      " arc-easy OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.255471 0.097614\n",
      " arc-easy  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.302609 0.314066\n",
      " arc-easy       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.579966 0.450155\n",
      " arc-easy       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.587542 0.457736\n",
      " arc-easy                        openhathi-4bit-quantized-wikitext 5shot  0.587542 0.460387\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('arc-easy-hi', '0shot')\n",
      "\n",
      "  challenge                                                    model  shot  accuracy       f1\n",
      "arc-easy-hi                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.355640 0.241157\n",
      "arc-easy-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.264310 0.255433\n",
      "arc-easy-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.291246 0.152862\n",
      "arc-easy-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.306397 0.406492\n",
      "arc-easy-hi       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.289983 0.153570\n",
      "arc-easy-hi       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.312710 0.184983\n",
      "arc-easy-hi                        openhathi-4bit-quantized-wikitext 0shot  0.327441 0.211753\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('arc-easy-hi', '5shot')\n",
      "\n",
      "  challenge                                                    model  shot  accuracy       f1\n",
      "arc-easy-hi                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.399832 0.301462\n",
      "arc-easy-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.253788 0.227074\n",
      "arc-easy-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.275253 0.128456\n",
      "arc-easy-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.261785 0.139151\n",
      "arc-easy-hi       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.359848 0.255034\n",
      "arc-easy-hi       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.366582 0.268888\n",
      "arc-easy-hi                        openhathi-4bit-quantized-wikitext 5shot  0.390152 0.298013\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('boolq', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "    boolq                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.542508 0.526282\n",
      "    boolq OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.420795 0.539173\n",
      "    boolq OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.378287 0.548924\n",
      "    boolq  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.621713 0.000000\n",
      "    boolq       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.603670 0.118367\n",
      "    boolq       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.607339 0.121751\n",
      "    boolq                        openhathi-4bit-quantized-wikitext 0shot  0.617431 0.074019\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('boolq', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "    boolq                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.634862 0.088550\n",
      "    boolq OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.378899 0.548767\n",
      "    boolq OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.378287 0.548724\n",
      "    boolq  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.621713 0.000000\n",
      "    boolq       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.619878 0.081301\n",
      "    boolq       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.621713 0.026751\n",
      "    boolq                        openhathi-4bit-quantized-wikitext 5shot  0.621407 0.185526\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('boolq-hi', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      " boolq-hi                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.606116 0.408632\n",
      " boolq-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.442202 0.526234\n",
      " boolq-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.378899 0.549168\n",
      " boolq-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.621713 0.000000\n",
      " boolq-hi       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.616514 0.144611\n",
      " boolq-hi                        openhathi-4bit-quantized-wikitext 0shot  0.623853 0.070997\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('boolq-hi', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      " boolq-hi                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.620489 0.483132\n",
      " boolq-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.377982 0.548401\n",
      " boolq-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.378287 0.548724\n",
      " boolq-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.621713 0.000000\n",
      " boolq-hi       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.617737 0.042879\n",
      " boolq-hi                        openhathi-4bit-quantized-wikitext 5shot  0.567890 0.269767\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('hellaswag', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "hellaswag OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.247660 0.099783\n",
      "hellaswag OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.265385 0.145365\n",
      "hellaswag  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.265087 0.162946\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('hellaswag', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "hellaswag OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.247660 0.099783\n",
      "hellaswag OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.265385 0.145365\n",
      "hellaswag  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.265087 0.162946\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('hellaswag-hi', '0shot')\n",
      "\n",
      "   challenge                                                    model  shot  accuracy       f1\n",
      "hellaswag-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.247959 0.101346\n",
      "hellaswag-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.267576 0.161672\n",
      "hellaswag-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.254531 0.137961\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('hellaswag-hi', '5shot')\n",
      "\n",
      "   challenge                                                    model  shot  accuracy       f1\n",
      "hellaswag-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.247959 0.101346\n",
      "hellaswag-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.267576 0.161672\n",
      "hellaswag-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.254531 0.137961\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indiccopa', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "indiccopa                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.556793 0.675367\n",
      "indiccopa OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.512249 0.661515\n",
      "indiccopa OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.492205 0.659701\n",
      "indiccopa  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.505568 0.000000\n",
      "indiccopa       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.538976 0.342857\n",
      "indiccopa       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.541203 0.449198\n",
      "indiccopa                        openhathi-4bit-quantized-wikitext 0shot  0.554566 0.476440\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indiccopa', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "indiccopa                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.594655 0.427673\n",
      "indiccopa OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.496659 0.662687\n",
      "indiccopa OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.494432 0.661699\n",
      "indiccopa  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.505568 0.000000\n",
      "indiccopa       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.532294 0.270833\n",
      "indiccopa       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.543430 0.220532\n",
      "indiccopa                        openhathi-4bit-quantized-wikitext 5shot  0.596882 0.539440\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indicsentiment', '0shot')\n",
      "\n",
      "     challenge                                                    model  shot  accuracy       f1\n",
      "indicsentiment                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.590180 0.706389\n",
      "indicsentiment OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.572144 0.236136\n",
      "indicsentiment OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.812625 0.769420\n",
      "indicsentiment  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.507014 0.000000\n",
      "indicsentiment       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.562124 0.691602\n",
      "indicsentiment       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.573146 0.697013\n",
      "indicsentiment                        openhathi-4bit-quantized-wikitext 0shot  0.544088 0.682484\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indicsentiment', '5shot')\n",
      "\n",
      "     challenge                                                    model  shot  accuracy       f1\n",
      "indicsentiment                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.966934 0.966223\n",
      "indicsentiment OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.511022 0.016129\n",
      "indicsentiment OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.553106 0.688112\n",
      "indicsentiment  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.507014 0.000000\n",
      "indicsentiment       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.918838 0.918593\n",
      "indicsentiment       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.924850 0.926108\n",
      "indicsentiment                        openhathi-4bit-quantized-wikitext 5shot  0.907816 0.912046\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indicxnli', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "indicxnli                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.333333 0.166667\n",
      "indicxnli OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.333533 0.167115\n",
      "indicxnli OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.333333 0.166667\n",
      "indicxnli  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.335329 0.198628\n",
      "indicxnli       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.335529 0.177577\n",
      "indicxnli       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.334531 0.178811\n",
      "indicxnli                        openhathi-4bit-quantized-wikitext 0shot  0.337924 0.197139\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indicxnli', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy       f1\n",
      "indicxnli                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.425349 0.367979\n",
      "indicxnli OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.331337 0.224210\n",
      "indicxnli OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.333134 0.167039\n",
      "indicxnli  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.329142 0.173236\n",
      "indicxnli       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.379840 0.299829\n",
      "indicxnli       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.423353 0.391117\n",
      "indicxnli                        openhathi-4bit-quantized-wikitext 5shot  0.354691 0.259220\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indicxparaphrase', '0shot')\n",
      "\n",
      "       challenge                                                    model  shot  accuracy       f1\n",
      "indicxparaphrase                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.593407 0.704859\n",
      "indicxparaphrase OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.602897 0.714747\n",
      "indicxparaphrase OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.721778 0.764283\n",
      "indicxparaphrase  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.503497 0.036822\n",
      "indicxparaphrase       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.539461 0.545813\n",
      "indicxparaphrase       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.601898 0.574025\n",
      "indicxparaphrase                        openhathi-4bit-quantized-wikitext 0shot  0.536963 0.465089\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('indicxparaphrase', '5shot')\n",
      "\n",
      "       challenge                                                    model  shot  accuracy       f1\n",
      "indicxparaphrase                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.500000 0.666667\n",
      "indicxparaphrase OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.497003 0.661057\n",
      "indicxparaphrase OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.501998 0.661919\n",
      "indicxparaphrase  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.483017 0.647839\n",
      "indicxparaphrase       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 5shot  0.481019 0.567264\n",
      "indicxparaphrase       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 5shot  0.486014 0.625954\n",
      "indicxparaphrase                        openhathi-4bit-quantized-wikitext 5shot  0.479021 0.467041\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('mmlu', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy   f1\n",
      "     mmlu                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.360561 -1.0\n",
      "     mmlu OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.267626 -1.0\n",
      "     mmlu OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.312277 -1.0\n",
      "     mmlu  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.259578 -1.0\n",
      "     mmlu       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.268979 -1.0\n",
      "     mmlu       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.284575 -1.0\n",
      "     mmlu                        openhathi-4bit-quantized-wikitext 0shot  0.283720 -1.0\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('mmlu', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy   f1\n",
      "     mmlu                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.392537 -1.0\n",
      "     mmlu OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.256374 -1.0\n",
      "     mmlu OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.309358 -1.0\n",
      "     mmlu  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.260219 -1.0\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('mmlu-hi', '0shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy   f1\n",
      "  mmlu-hi                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.300071 -1.0\n",
      "  mmlu-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.248052 -1.0\n",
      "  mmlu-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.267179 -1.0\n",
      "  mmlu-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.256654 -1.0\n",
      "  mmlu-hi       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.250987 -1.0\n",
      "  mmlu-hi       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.252404 -1.0\n",
      "  mmlu-hi                        openhathi-4bit-quantized-wikitext 0shot  0.258172 -1.0\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('mmlu-hi', '5shot')\n",
      "\n",
      "challenge                                                    model  shot  accuracy   f1\n",
      "  mmlu-hi                                OpenHathi-7B-Hi-v0.1-Base 5shot  0.321729 -1.0\n",
      "  mmlu-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 5shot  0.248558 -1.0\n",
      "  mmlu-hi OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 5shot  0.282562 -1.0\n",
      "  mmlu-hi  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 5shot  0.259792 -1.0\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n",
      "====================================================================================================\n",
      "Challenge: ('winogrande', '0shot')\n",
      "\n",
      " challenge                                                    model  shot  accuracy       f1\n",
      "winogrande                                OpenHathi-7B-Hi-v0.1-Base 0shot  0.494870 0.360368\n",
      "winogrande OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-chat-format 0shot  0.496448 0.347721\n",
      "winogrande OpenHathi-7B-Hi-v0.1-Base-LoRA-samvaad-hi-v1-tulu-format 0shot  0.498027 0.494421\n",
      "winogrande  OpenHathi-7B-Hi-v0.1-Base-SSF-samvaad-hi-v1-chat-format 0shot  0.501973 0.367619\n",
      "winogrande       openhathi-4bit-quantized-samvaad-hi-v1-chat-format 0shot  0.490923 0.378943\n",
      "winogrande       openhathi-4bit-quantized-samvaad-hi-v1-tulu-format 0shot  0.495659 0.438577\n",
      "winogrande                        openhathi-4bit-quantized-wikitext 0shot  0.494870 0.349468\n",
      "==================================================================================================== \n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Grouping the DataFrame by challenge\n",
    "\n",
    "grouped_df = results_df.groupby(by=[\"challenge\", \"shot\"])\n",
    "# Printing a nice representation of the grouped DataFrame\n",
    "for key, group in grouped_df:\n",
    "    group = group.sort_values(by=[\"model\"])\n",
    "    print(\"=\" * 100)\n",
    "    print(f\"Challenge: {key}\\n\")\n",
    "    print(group.to_string(index=False))\n",
    "    print(\"=\" * 100, \"\\n\" * 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}