txtproj/txt_dataset_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ee95a38117f298b2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T01:54:29.113121Z",
     "start_time": "2025-02-04T01:54:29.109287Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, HTML\n",
    "\n",
    "# Set the width to a specific percentage of the screen width, e.g., 100%\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T01:54:29.134956Z",
     "start_time": "2025-02-04T01:54:29.131861Z"
    }
   },
   "outputs": [],
   "source": [
    "from os import path\n",
    "\n",
    "PROJECT_ROOT = path.abspath(path.join(globals()['_dh'][0], '..'))\n",
    "DATALAKE_PATH = path.abspath(path.join(PROJECT_ROOT, '..', '..', 'datalake', 'txtproj'))\n",
    "MODEL_DIR = path.abspath(path.join(PROJECT_ROOT, 'txtproj', 'classifier'))\n",
    "\n",
    "FQFN_PROCESSED_DF = path.join(DATALAKE_PATH, 'processed', 'processed_20250203.jsonl')\n",
    "FQFN_VOCABULARY = path.join(DATALAKE_PATH, 'processed', 'vocabulary_20250203.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fc6dd4051e3faca8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T01:54:48.386877Z",
     "start_time": "2025-02-04T01:54:29.140350Z"
    }
   },
   "outputs": [],
   "source": [
    "from txtproj.trainer import txt_folder_loader\n",
    "\n",
    "wtl = txt_folder_loader.TextFolderLoader()\n",
    "wtl.read(DATALAKE_PATH, labels=['0', '1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ae34cdb5cc6b9285",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T01:56:14.476392Z",
     "start_time": "2025-02-04T01:56:14.440839Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text_tfidf</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>chaikin analytics opens new window opens exter...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.114...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>stansberry research opens new window opens ext...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>&lt;empty&gt;</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>forbidden &lt;num&gt; forbidden</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>sauravtest &lt;num&gt;</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  \\\n",
       "60  chaikin analytics opens new window opens exter...   \n",
       "61  stansberry research opens new window opens ext...   \n",
       "62                                            <empty>   \n",
       "63                          forbidden <num> forbidden   \n",
       "64                                   sauravtest <num>   \n",
       "\n",
       "                                           text_tfidf  label  \n",
       "60  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.114...      0  \n",
       "61  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...      0  \n",
       "62  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...      0  \n",
       "63  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...      0  \n",
       "64  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...      0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = wtl.df\n",
    "df[60:65][['text', 'text_tfidf', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7414ecfae34ebc92",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T01:56:33.067113Z",
     "start_time": "2025-02-04T01:56:33.017515Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAJslJREFUeJzt3QtwVOX9//FvNgkLUQKEGJPUcLVclJuCRIQqtxDBAalpvaAWKAUvwIzJVCkKmBBbKKXIlEYZWy52JGLpACoiEEBASkDBZhTU1CCKFAJFC5Hkx5LL+c/z/GcDmwuQuOfkOcn7NXNc9pyzJ89+d5Pz8TnPsxtiWZYlAAAABvE0dAMAAACqIqAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIwTJi5UUVEhx48fl5YtW0pISEhDNwcAAFwF9dmw33//vcTHx4vH42l8AUWFk4SEhIZuBgAAqIdvvvlGbrjhhsYXUFTPif8JRkZGBvXYpaWlsmXLFhkxYoSEh4cH9di4iDo7gzo7gzo7gzq7v9ZFRUW6g8F/Hm90AcV/WUeFEzsCSkREhD4uvwD2oc7OoM7OoM7OoM6Np9ZXMzyDQbIAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxglr6AaYqkf6ZvGVX/nroE3x1fx7GroJAAAEDT0oAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAALg7oMybN09uu+02admypcTExMjYsWMlPz8/YJ/z58/L1KlTpW3btnLttddKSkqKnDx5MmCfo0ePyj333CMRERH6OE8//bSUlZUF5xkBAICmFVB27typw8fevXslJydHSktLZcSIEVJcXFy5T2pqqrz99tuyZs0avf/x48flvvvuq9xeXl6uw8mFCxdkz5498uqrr8rKlStlzpw5wX1mAADAtcLqsvOmTZsC7qtgoXpADhw4IHfeeaecPXtWli1bJtnZ2TJ06FC9z4oVK6R79+461Nx+++2yZcsW+fTTT2Xr1q1y/fXXS58+fSQzM1NmzJgh6enp0qxZs+A+QwAA0LgDSlUqkChRUVH6VgUV1asyfPjwyn26desm7dq1k9zcXB1Q1G3Pnj11OPFLTk6WJ554Qg4dOiS33HJLtZ/j8/n04ldUVKRv1c9SSzD5j+f1WOImwa6DU+11W7vdhjo7gzo7gzq7v9Z1OV69A0pFRYU89dRTMnDgQOnRo4deV1hYqHtAWrduHbCvCiNqm3+fS8OJf7t/W21jXzIyMqqtV70xahyLHTL7VYibbNy4UdxIXSqE/aizM6izM6ize2tdUlJif0BRY1EOHjwou3fvFrvNnDlT0tLSAnpQEhIS9PiXyMjIoKc79YLM3u8RX0WIuMXB9GRxE3+dk5KSJDw8vKGb02hRZ2dQZ2e4tc490jeL23g9lv4f9WDX2n8FxLaAMm3aNNmwYYPs2rVLbrjhhsr1sbGxevDrmTNnAnpR1Cwetc2/zwcffBBwPP8sH/8+VXm9Xr1UpYpm15tUhRNfuXsCipt+WZ16DXERdXYGdXaG2+rspnOJ3bWuy7HqNIvHsiwdTtatWyfbt2+Xjh07Bmzv27ev/uHbtm2rXKemIatpxQMGDND31e0nn3wip06dqtxHJWLVE3LTTTfVpTkAAKCRCqvrZR01Q+fNN9/Un4XiHzPSqlUradGihb6dNGmSvhyjBs6q0DF9+nQdStQAWUVdllFB5NFHH5UFCxboY8yaNUsfu6ZeEgAA0PTUKaC8/PLL+nbw4MEB69VU4gkTJuh/v/jii+LxePQHtKmZN2qGzksvvVS5b2hoqL48pGbtqOByzTXXyPjx42Xu3LnBeUYAAKBpBRR1iedKmjdvLllZWXqpTfv27V076wQAANiP7+IBAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAALg/oOzatUtGjx4t8fHxEhISIuvXrw/YrtbVtPzhD3+o3KdDhw7Vts+fPz84zwgAADS9gFJcXCy9e/eWrKysGrefOHEiYFm+fLkOICkpKQH7zZ07N2C/6dOn1/9ZAACARiWsrg8YOXKkXmoTGxsbcP/NN9+UIUOGSKdOnQLWt2zZstq+AAAA9QoodXHy5El555135NVXX622TV3SyczMlHbt2sm4ceMkNTVVwsJqbo7P59OLX1FRkb4tLS3VSzD5j+f1WOImwa6DU+11W7vdhjo7gzo7w6119oa663xy6TnQrnPs1QixLKvelVOXbtatWydjx46tcfuCBQt0EDl+/Lg0b968cv2iRYvk1ltvlaioKNmzZ4/MnDlTJk6cqNfXJD09XTIyMqqtz87OloiIiPo2HwAAOKikpER3Spw9e1YiIyMbLqB069ZNkpKSZMmSJZc9jhqn8thjj8m5c+fE6/VeVQ9KQkKCnD59+opPsD7pLicnR2bv94ivIkTc4mB6sriJv87q/REeHt7QzWm0qLMzqLMz3FrnHumbxY09KJn9KoJea3X+jo6OvqqAYtslnvfff1/y8/PljTfeuOK+iYmJUlZWJl999ZV07dq12nYVWmoKLqpodr1JVTjxlbsnoLjpl9Wp1xAXUWdnUGdnuK3ObjqX2F3ruhzLts9BWbZsmfTt21fP+LmSvLw88Xg8EhMTY1dzAACAi9S5B0VdhikoKKi8f+TIER0w1HgSNeDV34WzZs0a+eMf/1jt8bm5ubJv3z49s0fN5FH31QDZRx55RNq0afNDnw8AAGiKAWX//v06XPilpaXp2/Hjx8vKlSv1v1evXi1qaMtDDz1U7fHqUo3arga+qnElHTt21AHFfxwAAIA6B5TBgwfr8HE5U6ZM0UtN1OydvXv31vXHAgCAJoTv4gEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAuD+g7Nq1S0aPHi3x8fESEhIi69evD9g+YcIEvf7S5e677w7Y57vvvpOHH35YIiMjpXXr1jJp0iQ5d+7cD382AACgaQaU4uJi6d27t2RlZdW6jwokJ06cqFxef/31gO0qnBw6dEhycnJkw4YNOvRMmTKlfs8AAAA0OmF1fcDIkSP1cjler1diY2Nr3PbZZ5/Jpk2b5MMPP5R+/frpdUuWLJFRo0bJwoULdc8MAABo2uocUK7Gjh07JCYmRtq0aSNDhw6VF154Qdq2bau35ebm6ss6/nCiDB8+XDwej+zbt09++tOfVjuez+fTi19RUZG+LS0t1Usw+Y/n9VjiJsGug1PtdVu73YY6O4M6O8OtdfaGuut8cuk50K5zbIMEFHV557777pOOHTvK4cOH5dlnn9U9LiqYhIaGSmFhoQ4vAY0IC5OoqCi9rSbz5s2TjIyMauu3bNkiERERYofMfhXiJhs3bhQ3Upf5YD/q7Azq7Ay31XlBf3GtnCDXuqSkpOECyoMPPlj57549e0qvXr2kc+fOuldl2LBh9TrmzJkzJS0tLaAHJSEhQUaMGKEH2gY73akXZPZ+j/gqQsQtDqYni5v465yUlCTh4eEN3ZxGizo7gzo7w6117pG+WdzYg5LZryLotfZfAWmwSzyX6tSpk0RHR0tBQYEOKGpsyqlTpwL2KSsr0zN7ahu3osa0qKUqVTS73qQqnPjK3RNQ3PTL6tRriIuoszOoszPcVmc3nUvsrnVdjmX756AcO3ZMvv32W4mLi9P3BwwYIGfOnJEDBw5U7rN9+3apqKiQxMREu5sDAABcoM49KOrzSlRviN+RI0ckLy9PjyFRixorkpKSontD1BiUZ555Rm688UZJTv7/lyC6d++ux6lMnjxZli5dqrvspk2bpi8NMYMHAADUqwdl//79csstt+hFUWND1L/nzJmjB8F+/PHHMmbMGOnSpYv+ALa+ffvK+++/H3CJZtWqVdKtWzd9yUdNLx40aJC88sorvCIAAKB+PSiDBw8Wy6p9ytTmzVceDKR6WrKzs+v6owEAQBPBd/EAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAANwfUHbt2iWjR4+W+Ph4CQkJkfXr11duKy0tlRkzZkjPnj3lmmuu0fv84he/kOPHjwcco0OHDvqxly7z588PzjMCAABNL6AUFxdL7969JSsrq9q2kpIS+eijj2T27Nn6du3atZKfny9jxoyptu/cuXPlxIkTlcv06dPr/ywAAECjElbXB4wcOVIvNWnVqpXk5OQErPvzn/8s/fv3l6NHj0q7du0q17ds2VJiY2Pr02YAANDI1Tmg1NXZs2f1JZzWrVsHrFeXdDIzM3VoGTdunKSmpkpYWM3N8fl8evErKiqqvKSklmDyH8/rscRNgl0Hp9rrtna7DXV2BnV2hlvr7A111/nk0nOgXefYqxFiWVa9K6eCx7p162Ts2LE1bj9//rwMHDhQunXrJqtWrapcv2jRIrn11lslKipK9uzZIzNnzpSJEyfq9TVJT0+XjIyMauuzs7MlIiKivs0HAAAOUkNBVKeE6ryIjIxsmICiUlJKSoocO3ZMduzYcdmGLF++XB577DE5d+6ceL3eq+pBSUhIkNOnT1/xCdaVare6TDV7v0d8FSHiFgfTk8VN/HVOSkqS8PDwhm5Oo0WdnUGdneHWOvdI3yxu7EHJ7FcR9Fqr83d0dPRVBZQwu95E999/v3z99deyffv2KzYiMTFRysrK5KuvvpKuXbtW265CS03BRRXNrjepCie+cvcEFDf9sjr1GuIi6uwM6uwMt9XZTecSu2tdl2OF2RVOvvjiC3nvvfekbdu2V3xMXl6eeDweiYmJCXZzAACAC9U5oKjLMAUFBZX3jxw5ogOGGk8SFxcnP/vZz/QU4w0bNkh5ebkUFhbq/dT2Zs2aSW5uruzbt0+GDBmiZ/Ko+2qA7COPPCJt2rQJ7rMDAABNI6Ds379fhwu/tLQ0fTt+/Hg9mPWtt97S9/v06RPwONWbMnjwYH2pZvXq1XpfNa6kY8eOOqD4jwMAAFDngKJCxuXG1V5pzK2avbN37966/lgAANCE8F08AADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADuDyi7du2S0aNHS3x8vISEhMj69esDtluWJXPmzJG4uDhp0aKFDB8+XL744ouAfb777jt5+OGHJTIyUlq3bi2TJk2Sc+fO/fBnAwAAmmZAKS4ult69e0tWVlaN2xcsWCB/+tOfZOnSpbJv3z655pprJDk5Wc6fP1+5jwonhw4dkpycHNmwYYMOPVOmTPlhzwQAADQaYXV9wMiRI/VSE9V7snjxYpk1a5bce++9et3f/vY3uf7663VPy4MPPiifffaZbNq0ST788EPp16+f3mfJkiUyatQoWbhwoe6ZAQAATVtQx6AcOXJECgsL9WUdv1atWkliYqLk5ubq++pWXdbxhxNF7e/xeHSPCwAAQJ17UC5HhRNF9ZhcSt33b1O3MTExgY0IC5OoqKjKfary+Xx68SsqKtK3paWlegkm//G8HkvcJNh1cKq9bmu321BnZ1BnZ7i1zt5Qd51PLj0H2nWOdTyg2GXevHmSkZFRbf2WLVskIiLClp+Z2a9C3GTjxo3iRmocEuxHnZ1BnZ3htjov6C+ulRPkWpeUlDRMQImNjdW3J0+e1LN4/NT9Pn36VO5z6tSpgMeVlZXpmT3+x1c1c+ZMSUtLC+hBSUhIkBEjRuiZQMFOd+oFmb3fI76KEHGLg+nJ4ib+OiclJUl4eHhDN6fRos7OoM7OcGude6RvFjf2oGT2qwh6rf1XQBwPKB07dtQhY9u2bZWBRDVGjS154okn9P0BAwbImTNn5MCBA9K3b1+9bvv27VJRUaHHqtTE6/XqpSpVNLvepCqc+MrdE1Dc9Mvq1GuIi6izM6izM9xWZzedS+yudV2OVeeAoj6vpKCgIGBgbF5enh5D0q5dO3nqqafkhRdekB//+Mc6sMyePVvPzBk7dqzev3v37nL33XfL5MmT9VRklYinTZumZ/gwgwcAANQroOzfv1+GDBlSed9/6WX8+PGycuVKeeaZZ/RnpajPNVE9JYMGDdLTips3b175mFWrVulQMmzYMD17JyUlRX92CgAAQL0CyuDBg/XnndRGfbrs3Llz9VIb1duSnZ3NKwAAAGrEd/EAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAABp/QOnQoYOEhIRUW6ZOnaq3Dx48uNq2xx9/PNjNAAAALhYW7AN++OGHUl5eXnn/4MGDkpSUJD//+c8r102ePFnmzp1beT8iIiLYzQAAAC4W9IBy3XXXBdyfP3++dO7cWe66666AQBIbGxvsHw0AABoJW8egXLhwQV577TX55S9/qS/l+K1atUqio6OlR48eMnPmTCkpKbGzGQAAoKn3oFxq/fr1cubMGZkwYULlunHjxkn79u0lPj5ePv74Y5kxY4bk5+fL2rVraz2Oz+fTi19RUZG+LS0t1Usw+Y/n9VjiJsGug1PtdVu73YY6O4M6O8OtdfaGuut8cuk50K5z7NUIsSzLtsolJydLs2bN5O233651n+3bt8uwYcOkoKBAXwqqSXp6umRkZFRbn52dzfgVAABcQl0xUR0VZ8+elcjIyIYJKF9//bV06tRJ94zce++9te5XXFws1157rWzatEkHmqvtQUlISJDTp09f8QnWJ93l5OTI7P0e8VVcvCxluoPpNdfOVP46qwHU4eHhDd2cRos6O4M6O8Otde6Rvlnc2IOS2a8i6LVW5281xONqAoptl3hWrFghMTExcs8991x2v7y8PH0bFxdX6z5er1cvVami2fUmVeHEV+6egOKmX1anXkNcRJ2dQZ2d4bY6u+lcYnet63IsWwJKRUWFDijjx4+XsLCLP+Lw4cP6ssyoUaOkbdu2egxKamqq3HnnndKrVy87mgIAAFzIloCydetWOXr0qJ69cyk1HkVtW7x4sb60oy7TpKSkyKxZs+xoBgAAcClbAsqIESOkpqEtKpDs3LnTjh8JAAAaEb6LBwAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAADQ+ANKenq6hISEBCzdunWr3H7+/HmZOnWqtG3bVq699lpJSUmRkydPBrsZAADAxWzpQbn55pvlxIkTlcvu3bsrt6Wmpsrbb78ta9askZ07d8rx48flvvvus6MZAADApcJsOWhYmMTGxlZbf/bsWVm2bJlkZ2fL0KFD9boVK1ZI9+7dZe/evXL77bfb0RwAAOAytgSUL774QuLj46V58+YyYMAAmTdvnrRr104OHDggpaWlMnz48Mp91eUftS03N7fWgOLz+fTiV1RUpG/VsdQSTP7jeT2WuEmw6+BUe93Wbrehzs6gzs5wa529oe46n1x6DrTrHHs1QizLCmrl3n33XTl37px07dpVX97JyMiQ//znP3Lw4EF9aWfixIkBYUPp37+/DBkyRH7/+9/XOq5FHacq1RMTERERzOYDAACblJSUyLhx4/QVlcjISGcDSlVnzpyR9u3by6JFi6RFixb1Cig19aAkJCTI6dOnr/gE65PucnJyZPZ+j/gqQsQtDqYni5v465yUlCTh4eEN3ZxGizo7gzo7w6117pG+WdzYg5LZryLotVbn7+jo6KsKKLZc4rlU69atpUuXLlJQUKCf6IULF3RoUev91Cyemsas+Hm9Xr1UpYpm15tUhRNfuXsCipt+WZ16DXERdXYGdXaG2+rspnOJ3bWuy7Fs/xwUdbnn8OHDEhcXJ3379tWN27ZtW+X2/Px8OXr0qB6rAgAAYEsPyq9//WsZPXq0vqyjphA///zzEhoaKg899JC0atVKJk2aJGlpaRIVFaW7d6ZPn67DCTN4AACAbQHl2LFjOox8++23ct1118mgQYP0FGL1b+XFF18Uj8ejP6BNjStJTk6Wl156KdjNAAAALhb0gLJ69erLbldTj7OysvQCAABQE76LBwAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAADQ+APKvHnz5LbbbpOWLVtKTEyMjB07VvLz8wP2GTx4sISEhAQsjz/+eLCbAgAAXCroAWXnzp0ydepU2bt3r+Tk5EhpaamMGDFCiouLA/abPHmynDhxonJZsGBBsJsCAABcKizYB9y0aVPA/ZUrV+qelAMHDsidd95ZuT4iIkJiY2OD/eMBAEAjEPSAUtXZs2f1bVRUVMD6VatWyWuvvaZDyujRo2X27Nk6tNTE5/Ppxa+oqEjfqt4ZtQST/3hejyVuEuw6ONVet7XbbaizM6izM9xaZ2+ou84nl54D7TrHXo0Qy7Jsq1xFRYWMGTNGzpw5I7t3765c/8orr0j79u0lPj5ePv74Y5kxY4b0799f1q5dW+Nx0tPTJSMjo9r67OzsWkMNAAAwS0lJiYwbN053XkRGRjZcQHniiSfk3Xff1eHkhhtuqHW/7du3y7Bhw6SgoEA6d+58VT0oCQkJcvr06Ss+wfqkOzV2ZvZ+j/gqQsQtDqYni5v465yUlCTh4eEN3ZxGizo7gzo7w6117pG+WdzYg5LZryLotVbn7+jo6KsKKLZd4pk2bZps2LBBdu3addlwoiQmJurb2gKK1+vVS1WqaHa9SVU48ZW7J6C46ZfVqdcQF1FnZ1BnZ7itzm46l9hd67ocK+gBRXXITJ8+XdatWyc7duyQjh07XvExeXl5+jYuLi7YzQEAAC4U9ICiphirsSFvvvmm/iyUwsJCvb5Vq1bSokULOXz4sN4+atQoadu2rR6Dkpqaqmf49OrVK9jNAQAALhT0gPLyyy9XfhjbpVasWCETJkyQZs2aydatW2Xx4sX6s1HUWJKUlBSZNWtWsJsCAABcypZLPJejAon6MDcAAIDa8F08AADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjEFAAAIBxCCgAAMA4BBQAAGAcAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAjENAAQAAxiGgAAAA4xBQAACAcQgoAADAOAQUAABgHAIKAAAwDgEFAAAYh4ACAACMQ0ABAADGIaAAAADjNGhAycrKkg4dOkjz5s0lMTFRPvjgg4ZsDgAAaOoB5Y033pC0tDR5/vnn5aOPPpLevXtLcnKynDp1qqGaBAAAmnpAWbRokUyePFkmTpwoN910kyxdulQiIiJk+fLlDdUkAABgiLCG+KEXLlyQAwcOyMyZMyvXeTweGT58uOTm5lbb3+fz6cXv7Nmz+va7776T0tLSoLZNHa+kpETCSj1SXhEibvHtt9+Km/jrrNodHh7e0M1ptKizM6izM9xa57CyYnGbsApLSkoqgl7r77//Xt9alnXlNkgDOH36tJSXl8v1118fsF7d//zzz6vtP2/ePMnIyKi2vmPHjra2002i/9jQLQAANCbjbDy2CiqtWrUyL6DUleppUeNV/CoqKnTvSdu2bSUkJLi9HEVFRZKQkCDffPONREZGBvXYuIg6O4M6O4M6O4M6u7/WqudEhZP4+Pgr7tsgASU6OlpCQ0Pl5MmTAevV/djY2Gr7e71evVyqdevWtrZRvSD8AtiPOjuDOjuDOjuDOru71lfqOWnQQbLNmjWTvn37yrZt2wJ6RdT9AQMGNESTAACAQRrsEo+6ZDN+/Hjp16+f9O/fXxYvXizFxcV6Vg8AAGjaGiygPPDAA/Lf//5X5syZI4WFhdKnTx/ZtGlTtYGzTlOXktRns1S9pITgos7OoM7OoM7OoM5Nq9Yh1tXM9QEAAHAQ38UDAACMQ0ABAADGIaAAAADjEFAAAIBxmmRAycrKkg4dOkjz5s0lMTFRPvjgg8vuv2bNGunWrZvev2fPnrJx40bH2tpU6vyXv/xFfvKTn0ibNm30or6X6UqvC+r3fvZbvXq1/iTmsWPH2t7GpljnM2fOyNSpUyUuLk7PhOjSpQt/O2yos/qIiq5du0qLFi30J5+mpqbK+fPnHWuvG+3atUtGjx6tP81V/Q1Yv379FR+zY8cOufXWW/V7+cYbb5SVK1fa31CriVm9erXVrFkza/ny5dahQ4esyZMnW61bt7ZOnjxZ4/7//Oc/rdDQUGvBggXWp59+as2aNcsKDw+3PvnkE8fb3pjrPG7cOCsrK8v617/+ZX322WfWhAkTrFatWlnHjh1zvO2Nuc5+R44csX70ox9ZP/nJT6x7773XsfY2lTr7fD6rX79+1qhRo6zdu3freu/YscPKy8tzvO2Nuc6rVq2yvF6vvlU13rx5sxUXF2elpqY63nY32bhxo/Xcc89Za9euVbN4rXXr1l12/y+//NKKiIiw0tLS9HlwyZIl+ry4adMmW9vZ5AJK//79ralTp1beLy8vt+Lj46158+bVuP/9999v3XPPPQHrEhMTrccee8z2tjalOldVVlZmtWzZ0nr11VdtbGXTrLOq7R133GH99a9/tcaPH09AsaHOL7/8stWpUyfrwoULDray6dVZ7Tt06NCAdeokOnDgQNvb2ljIVQSUZ555xrr55psD1j3wwANWcnKyrW1rUpd4Lly4IAcOHNCXD/w8Ho++n5ubW+Nj1PpL91eSk5Nr3R/1q3NV6ivV1VerR0VF2djSplnnuXPnSkxMjEyaNMmhlja9Or/11lv6azvUJR714ZM9evSQ3/3ud/pb3BG8Ot9xxx36Mf7LQF9++aW+jDZq1CjH2t0U5DbQedAV32YcLKdPn9Z/IKp+Wq26//nnn9f4GPUptzXtr9YjeHWuasaMGfr6aNVfCvywOu/evVuWLVsmeXl5DrWyadZZnSi3b98uDz/8sD5hFhQUyJNPPqlDt/p0TgSnzuPGjdOPGzRokP6W3LKyMnn88cfl2WefdajVTUNhLedB9Y3H//d//6fH/9ihSfWgwB3mz5+vB3CuW7dOD5RDcKivOH/00Uf1gGT1jeKwj/ryU9VL9corr+gvRlVf7fHcc8/J0qVLG7ppjYoauKl6pl566SX56KOPZO3atfLOO+9IZmZmQzcNQdCkelDUH+XQ0FA5efJkwHp1PzY2tsbHqPV12R/1q7PfwoULdUDZunWr9OrVy+aWNq06Hz58WL766is9ev/SE6kSFhYm+fn50rlzZwda3vjfz2rmTnh4uH6cX/fu3fX/iapLGeob3fHD6zx79mwdun/1q1/p+2qWpfrS2SlTpuhAqC4R4Yer7TwYGRlpW++J0qRePfVHQf3fzLZt2wL+QKv76npxTdT6S/dXcnJyat0f9auzsmDBAv1/PupLI9W3XCO4dVZT5T/55BN9ece/jBkzRoYMGaL/raZoIjjv54EDB+rLOv4AqPz73//WwYVwErw6q7FqVUOIPxTyNXPB02DnQasJTmNT09JWrlypp0tNmTJFT2MrLCzU2x999FHrN7/5TcA047CwMGvhwoV6+uvzzz/PNGMb6jx//nw9vfAf//iHdeLEicrl+++/b8Bn0fjqXBWzeOyp89GjR/UstGnTpln5+fnWhg0brJiYGOuFF15owGfR+Oqs/h6rOr/++ut6KuyWLVuszp0769mXqJ36u6o+0kEtKgYsWrRI//vrr7/W21WNVa2rTjN++umn9XlQfSQE04xtouZwt2vXTp8Q1bS2vXv3Vm6766679B/tS/3973+3unTpovdXU63eeeedBmh1465z+/bt9S9K1UX9AUJw38+XIqDYV+c9e/bojyRQJ1w15fi3v/2tnuKN4NW5tLTUSk9P16GkefPmVkJCgvXkk09a//vf/xqo9e7w3nvv1fj31l9bdatqXfUxffr00a+Lej+vWLHC9naGqP/Y20cDAABQN01qDAoAAHAHAgoAADAOAQUAABiHgAIAAIxDQAEAAMYhoAAAAOMQUAAAgHEIKAAAwDgEFAAAYBwCCgAAMA4BBQAAGIeAAgAAxDT/D+3AOdSKdfl2AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['label'].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "319905f9545ff990",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T03:08:11.804430Z",
     "start_time": "2025-02-04T03:08:11.802724Z"
    }
   },
   "outputs": [],
   "source": [
    "# for label in df['label'].unique():\n",
    "#     sampled_df = df[df['label'] == label]['text'].sample(n=10)\n",
    "#     print(f'Sampled Series for text with label {label}:')\n",
    "#     display(sampled_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2c214ba4cf09217c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T01:58:59.475399Z",
     "start_time": "2025-02-04T01:58:59.462780Z"
    }
   },
   "outputs": [],
   "source": [
    "df.to_json(FQFN_PROCESSED_DF, orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ec0beeeb4ea373eb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T01:59:02.158023Z",
     "start_time": "2025-02-04T01:59:02.153702Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive case vocabulary size=256\n",
      "Positive case vocabulary size=256\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "for fqfp in [FQFN_VOCABULARY, './classifier/vocabulary.json']:\n",
    "    print(f'Positive case vocabulary size={len(wtl.vocabulary)}')\n",
    "    with open(fqfp, encoding='utf-8', mode='w+') as f:\n",
    "        json.dump(wtl.vocabulary, fp=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf0a199ddc53bd4c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}