05_classification_in_ml.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 利用Embedding，训练机器学习模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://github.com/aceimnorstuvwxz/toutiao-text-classfication-dataset/raw/master/toutiao_cat_data.txt.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!unzip ./toutiao_cat_data.txt.zip\n",
    "!mv ./toutiao_cat_data.txt data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/lk/1jcy9d_j6258__3gv0rpj8h40000gn/T/ipykernel_3501/2567486315.py:16: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  df = pd.read_csv(os.environ.get(\"JUPYTER_HOME\") + '/data/toutiao_cat_data.txt', sep='_!_', names=['id', 'code', 'category', 'title', 'keywords'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lines of text before filtering:  382688\n",
      "Lines of text after filtering:  382688\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tiktoken\n",
    "import openai\n",
    "import os\n",
    "import backoff\n",
    "\n",
    "openai.api_key = os.environ.get(\"OPENAI_API_KEY\")\n",
    "\n",
    "# embedding model parameters\n",
    "embedding_model = \"text-embedding-ada-002\"\n",
    "embedding_encoding = \"cl100k_base\"  # this the encoding for text-embedding-ada-002\n",
    "max_tokens = 8000  # the maximum for text-embedding-ada-002 is 8191\n",
    "\n",
    "\n",
    "# import data/toutiao_cat_data.txt as a pandas dataframe\n",
    "df = pd.read_csv(os.environ.get(\"JUPYTER_HOME\") + '/data/toutiao_cat_data.txt', sep='_!_', names=['id', 'code', 'category', 'title', 'keywords'])\n",
    "df = df.fillna(\"\")\n",
    "df[\"combined\"] = (\n",
    "    \"标题: \" + df.title.str.strip() + \"; 关键字: \" + df.keywords.str.strip()\n",
    ")\n",
    "\n",
    "print(\"Lines of text before filtering: \", len(df))\n",
    "\n",
    "encoding = tiktoken.get_encoding(embedding_encoding)\n",
    "# omit reviews that are too long to embed\n",
    "df[\"n_tokens\"] = df.combined.apply(lambda x: len(encoding.encode(x)))\n",
    "df = df[df.n_tokens <= max_tokens]\n",
    "\n",
    "print(\"Lines of text after filtering: \", len(df))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 以下代码生成的文件可以在百度网盘提取，信息如下：\n",
    "# 链接: https://pan.baidu.com/s/1Cl0eFNLOkQqquf9ls0trEw 提取码: jvr4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "import os\n",
    "\n",
    "client = OpenAI(api_key=os.environ['OPENAI_API_KEY'])\n",
    "\n",
    "EMBEDDING_MODEL = \"text-embedding-ada-002\"\n",
    "\n",
    "def get_embedding(text, model=EMBEDDING_MODEL):\n",
    "   text = text.replace(\"\\n\", \" \")\n",
    "   return client.embeddings.create(input = [text], model=model).data[0].embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 请不要执行如下代码\n",
    "\n",
    "# df_1k = df.sample(1000, random_state=42)\n",
    "\n",
    "# df_1k[\"embedding\"] = df_1k.combined.apply(lambda x : get_embedding(x, model=embedding_model))\n",
    "# df_1k.to_csv(\"data/toutiao_cat_data_10k_with_embeddings.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 请不需要执行如下代码\n",
    "# @backoff.on_exception(backoff.expo, openai.RateLimitError)\n",
    "# def get_embedding_with_backoff(**kwargs):\n",
    "#    return get_embedding(**kwargs)\n",
    "#\n",
    "# df_10k = df.sample(10000, random_state=42)\n",
    "#\n",
    "# df_10k[\"embedding\"] = df_10k.combined.apply(lambda x : get_embedding_with_backoff(text=x, model=embedding_model))\n",
    "# df_10k.to_csv(\"data/toutiao_cat_data_10k_with_embeddings.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 0 Number of embeddings: 2000\n",
      "Batch 0 Number of embeddings: 2000\n",
      "Batch 0 Number of embeddings: 2000\n",
      "Batch 0 Number of embeddings: 2000\n",
      "Batch 0 Number of embeddings: 2000\n"
     ]
    }
   ],
   "source": [
    "# 如果你不想重新计算一遍embedding，请不要运行如下代码\n",
    "\n",
    "batch_size = 2000\n",
    "\n",
    "def get_embeddings(list_of_text, model):\n",
    "    response = client.embeddings.create(input=list_of_text, model=model)\n",
    "    return [item.embedding for item in response.data]\n",
    "\n",
    "@backoff.on_exception(backoff.expo, openai.RateLimitError)\n",
    "def get_embeddings_with_backoff(prompts, model):\n",
    "    embeddings = []\n",
    "    for i in range(0, len(prompts), batch_size):\n",
    "        batch = prompts[i:i+batch_size]\n",
    "        batch_embeddings = get_embeddings(list_of_text=batch, model=model)\n",
    "        embeddings += batch_embeddings\n",
    "        print(f\"Batch {i} Number of embeddings: {len(embeddings)}\")\n",
    "    return embeddings\n",
    "\n",
    "# randomly sample 10k rows\n",
    "# df_all = df.sample(10000, random_state=42)\n",
    "df_all = df\n",
    "# group prompts into batches of 100\n",
    "prompts = df_all.combined.tolist()\n",
    "prompt_batches = [prompts[i:i+batch_size] for i in range(0, len(prompts), batch_size)]\n",
    "\n",
    "embeddings = []\n",
    "for batch in prompt_batches:\n",
    "    batch_embeddings = get_embeddings_with_backoff(prompts=batch, model=embedding_model)\n",
    "    embeddings += batch_embeddings\n",
    "\n",
    "df_all[\"embedding\"] = embeddings\n",
    "df_all.to_parquet(os.environ.get(\"JUPYTER_HOME\") + \"/data/toutiao_cat_data_all_with_embeddings.parquet\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练模型，看看效果怎么样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                    precision    recall  f1-score   support\n",
      "\n",
      "  news_agriculture       0.82      0.86      0.84       495\n",
      "          news_car       0.89      0.94      0.91       895\n",
      "      news_culture       0.87      0.76      0.81       741\n",
      "          news_edu       0.86      0.89      0.88       708\n",
      "news_entertainment       0.71      0.92      0.80      1051\n",
      "      news_finance       0.81      0.76      0.78       735\n",
      "         news_game       0.91      0.83      0.87       742\n",
      "        news_house       0.91      0.86      0.89       450\n",
      "     news_military       0.89      0.83      0.86       688\n",
      "       news_sports       0.90      0.92      0.91       968\n",
      "        news_story       0.94      0.46      0.61       197\n",
      "         news_tech       0.81      0.86      0.83      1052\n",
      "       news_travel       0.80      0.75      0.77       599\n",
      "        news_world       0.82      0.72      0.77       671\n",
      "             stock       0.00      0.00      0.00         8\n",
      "\n",
      "          accuracy                           0.84     10000\n",
      "         macro avg       0.80      0.76      0.77     10000\n",
      "      weighted avg       0.84      0.84      0.84     10000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/Caskroom/miniconda/base/envs/geektime/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/homebrew/Caskroom/miniconda/base/envs/geektime/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/homebrew/Caskroom/miniconda/base/envs/geektime/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "\n",
    "training_data = pd.read_parquet(os.environ.get(\"JUPYTER_HOME\") + \"/data/toutiao_cat_data_all_with_embeddings.parquet\")\n",
    "\n",
    "df =  training_data.sample(50000, random_state=42)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    list(df.embedding.values), df.category, test_size=0.2, random_state=42\n",
    ")\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=300)\n",
    "clf.fit(X_train, y_train)\n",
    "preds = clf.predict(X_test)\n",
    "probas = clf.predict_proba(X_test)\n",
    "\n",
    "report = classification_report(y_test, preds)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/Caskroom/miniconda/base/envs/geektime/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/opt/homebrew/Caskroom/miniconda/base/envs/geektime/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                    precision    recall  f1-score   support\n",
      "\n",
      "  news_agriculture       0.85      0.88      0.87      3908\n",
      "          news_car       0.92      0.92      0.92      7101\n",
      "      news_culture       0.82      0.85      0.83      5719\n",
      "          news_edu       0.88      0.89      0.89      5376\n",
      "news_entertainment       0.85      0.88      0.86      7908\n",
      "      news_finance       0.82      0.78      0.80      5409\n",
      "         news_game       0.91      0.87      0.89      5899\n",
      "        news_house       0.90      0.91      0.91      3463\n",
      "     news_military       0.86      0.82      0.84      4976\n",
      "       news_sports       0.93      0.93      0.93      7611\n",
      "        news_story       0.83      0.81      0.82      1308\n",
      "         news_tech       0.84      0.86      0.85      8168\n",
      "       news_travel       0.80      0.79      0.79      4252\n",
      "        news_world       0.79      0.80      0.80      5370\n",
      "             stock       0.00      0.00      0.00        70\n",
      "\n",
      "          accuracy                           0.86     76538\n",
      "         macro avg       0.80      0.80      0.80     76538\n",
      "      weighted avg       0.86      0.86      0.86     76538\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/Caskroom/miniconda/base/envs/geektime/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/opt/homebrew/Caskroom/miniconda/base/envs/geektime/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "df =  training_data\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    list(df.embedding.values), df.category, test_size=0.2, random_state=42\n",
    ")\n",
    "\n",
    "clf = LogisticRegression()\n",
    "clf.fit(X_train, y_train)\n",
    "preds = clf.predict(X_test)\n",
    "probas = clf.predict_proba(X_test)\n",
    "\n",
    "report = classification_report(y_test, preds)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "8114e84f04cf14e493992e1b725447accf84073d5ec18e7063d492738bf032cb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}