txtproj/txt_smoke_tester.ipynb

{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T02:09:28.934593Z",
     "start_time": "2025-02-04T02:09:28.931466Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from os import path\n",
    "\n",
    "PROJECT_ROOT = path.abspath(path.join(globals()['_dh'][0], '..'))\n",
    "DATALAKE_PATH = path.abspath(path.join(PROJECT_ROOT, '..', '..', 'datalake', 'txtproj'))\n"
   ],
   "id": "fed39de34efc97cb",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T02:09:29.056462Z",
     "start_time": "2025-02-04T02:09:28.938473Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import base64\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def encode_text_body(text_body: str | bytes) -> str:\n",
    "    \"\"\"\n",
    "    Returns Base64-encoded string version of a UTF-8 string or bytes representing a UTF-8 string\n",
    "    \"\"\"\n",
    "    if isinstance(text_body, bytes):\n",
    "        return base64.b64encode(text_body).decode('utf-8')\n",
    "    else:\n",
    "        return base64.b64encode(text_body.encode('utf-8')).decode('utf-8')\n",
    "\n",
    "\n",
    "def text_as_base64(fqfn: str) -> tuple[str, str]:\n",
    "    with open(fqfn, 'r') as f:\n",
    "        text_bytes = f.read()\n",
    "        b64str = encode_text_body(text_bytes)\n",
    "        return text_bytes, b64str\n",
    "\n",
    "\n",
    "def read(folder_path: str, labels: list[str] = ['0', '1'], k: int = 3) -> pd.DataFrame:\n",
    "    txt_labels: list[int] = []\n",
    "    txt_fqfns: list[str] = []\n",
    "    txt_file_names: list[str] = []\n",
    "    txt_lengths: list[int] = []\n",
    "    txt_b64s: list[str] = []\n",
    "    txt_contents: list[str] = []\n",
    "\n",
    "    for label in labels:\n",
    "        label_counter = 0\n",
    "        label_folder = os.path.join(folder_path, label)\n",
    "        if not os.path.isdir(label_folder):\n",
    "            continue  # Skip if the folder does not exist\n",
    "\n",
    "        for file_name in os.listdir(label_folder):\n",
    "            if file_name.endswith('.txt'):\n",
    "                label_counter += 1\n",
    "                if label_counter > k:\n",
    "                    break\n",
    "\n",
    "                txt_file_names.append(file_name)\n",
    "\n",
    "                text_fqfn = os.path.join(label_folder, file_name)\n",
    "                txt_fqfns.append(text_fqfn)\n",
    "\n",
    "                text_bytes, b64str = text_as_base64(text_fqfn)\n",
    "                txt_b64s.append(b64str)\n",
    "                txt_contents.append(text_bytes)\n",
    "\n",
    "                txt_lengths.append(len(text_bytes))\n",
    "                txt_labels.append(int(label))\n",
    "\n",
    "\n",
    "    return pd.DataFrame(data={\n",
    "        'file_name': txt_file_names,\n",
    "        'fqfn': txt_fqfns,\n",
    "        'text': txt_contents,\n",
    "        'text_size': txt_lengths,\n",
    "        'text_body_b64': txt_b64s,\n",
    "        'label': txt_labels,\n",
    "    })\n"
   ],
   "id": "1988519f6cc68600",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T02:09:29.100357Z",
     "start_time": "2025-02-04T02:09:29.095203Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df = read(DATALAKE_PATH, labels=['0', '1'], k=3)\n",
    "df[['text', 'text_size', 'text_body_b64', 'label']]"
   ],
   "id": "1d54a5b4adb41de2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                                text  text_size  \\\n",
       "0  <html lang=\"en\"><head>\\n    <meta charset=\"UTF...      16401   \n",
       "1  <html class=\"desktop portrait\"><head><meta htt...      22303   \n",
       "2  <html><head>\\n  <title>Index of /demonServer/n...       1206   \n",
       "3  <html><head>\\n<title>404 Not Found</title>\\n</...        295   \n",
       "4  <html><head><meta name=\"color-scheme\" content=...        171   \n",
       "5  <html><head>\\n<title>404 Not Found</title>\\n</...        295   \n",
       "\n",
       "                                       text_body_b64  label  \n",
       "0  PGh0bWwgbGFuZz0iZW4iPjxoZWFkPgogICAgPG1ldGEgY2...      0  \n",
       "1  PGh0bWwgY2xhc3M9ImRlc2t0b3AgcG9ydHJhaXQiPjxoZW...      0  \n",
       "2  PGh0bWw+PGhlYWQ+CiAgPHRpdGxlPkluZGV4IG9mIC9kZW...      0  \n",
       "3  PGh0bWw+PGhlYWQ+Cjx0aXRsZT40MDQgTm90IEZvdW5kPC...      1  \n",
       "4  PGh0bWw+PGhlYWQ+PG1ldGEgbmFtZT0iY29sb3Itc2NoZW...      1  \n",
       "5  PGh0bWw+PGhlYWQ+Cjx0aXRsZT40MDQgTm90IEZvdW5kPC...      1  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text_size</th>\n",
       "      <th>text_body_b64</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;html lang=\"en\"&gt;&lt;head&gt;\\n    &lt;meta charset=\"UTF...</td>\n",
       "      <td>16401</td>\n",
       "      <td>PGh0bWwgbGFuZz0iZW4iPjxoZWFkPgogICAgPG1ldGEgY2...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;html class=\"desktop portrait\"&gt;&lt;head&gt;&lt;meta htt...</td>\n",
       "      <td>22303</td>\n",
       "      <td>PGh0bWwgY2xhc3M9ImRlc2t0b3AgcG9ydHJhaXQiPjxoZW...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;html&gt;&lt;head&gt;\\n  &lt;title&gt;Index of /demonServer/n...</td>\n",
       "      <td>1206</td>\n",
       "      <td>PGh0bWw+PGhlYWQ+CiAgPHRpdGxlPkluZGV4IG9mIC9kZW...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;html&gt;&lt;head&gt;\\n&lt;title&gt;404 Not Found&lt;/title&gt;\\n&lt;/...</td>\n",
       "      <td>295</td>\n",
       "      <td>PGh0bWw+PGhlYWQ+Cjx0aXRsZT40MDQgTm90IEZvdW5kPC...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;html&gt;&lt;head&gt;&lt;meta name=\"color-scheme\" content=...</td>\n",
       "      <td>171</td>\n",
       "      <td>PGh0bWw+PGhlYWQ+PG1ldGEgbmFtZT0iY29sb3Itc2NoZW...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;html&gt;&lt;head&gt;\\n&lt;title&gt;404 Not Found&lt;/title&gt;\\n&lt;/...</td>\n",
       "      <td>295</td>\n",
       "      <td>PGh0bWw+PGhlYWQ+Cjx0aXRsZT40MDQgTm90IEZvdW5kPC...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-02-04T02:09:54.061940Z",
     "start_time": "2025-02-04T02:09:53.355662Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from txtproj.classifier.inf_server_driver import InferenceServerDriver\n",
    "\n",
    "model = InferenceServerDriver()\n",
    "df['outcome'] = model.predict(inputs=df)\n",
    "print(df[['text', 'text_size', 'text_body_b64', 'label', 'outcome']])\n"
   ],
   "id": "initial_id",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label=0\n",
      "Label=0\n",
      "Label=0\n",
      "Label=1\n",
      "Label=1\n",
      "Label=1\n",
      "                                                text  text_size  \\\n",
      "0  <html lang=\"en\"><head>\\n    <meta charset=\"UTF...      16401   \n",
      "1  <html class=\"desktop portrait\"><head><meta htt...      22303   \n",
      "2  <html><head>\\n  <title>Index of /demonServer/n...       1206   \n",
      "3  <html><head>\\n<title>404 Not Found</title>\\n</...        295   \n",
      "4  <html><head><meta name=\"color-scheme\" content=...        171   \n",
      "5  <html><head>\\n<title>404 Not Found</title>\\n</...        295   \n",
      "\n",
      "                                       text_body_b64  label  outcome  \n",
      "0  PGh0bWwgbGFuZz0iZW4iPjxoZWFkPgogICAgPG1ldGEgY2...      0        0  \n",
      "1  PGh0bWwgY2xhc3M9ImRlc2t0b3AgcG9ydHJhaXQiPjxoZW...      0        0  \n",
      "2  PGh0bWw+PGhlYWQ+CiAgPHRpdGxlPkluZGV4IG9mIC9kZW...      0        0  \n",
      "3  PGh0bWw+PGhlYWQ+Cjx0aXRsZT40MDQgTm90IEZvdW5kPC...      1        1  \n",
      "4  PGh0bWw+PGhlYWQ+PG1ldGEgbmFtZT0iY29sb3Itc2NoZW...      1        1  \n",
      "5  PGh0bWw+PGhlYWQ+Cjx0aXRsZT40MDQgTm90IEZvdW5kPC...      1        1  \n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-04T02:09:30.974904Z",
     "start_time": "2025-02-04T02:09:30.973543Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "53cad88c1e27a5bf",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}