skill_classifier/02-pre-processing-clean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "375b9dc1",
   "metadata": {},
   "source": [
    "# Data Pre-Processing ( removing punct-stopwords)\n",
    "\n",
    "## Table of contents\n",
    "\n",
    "1. [Data Normalization](#normdata)\n",
    "2. [Deletion of Duplicates](#deldups)\n",
    "3. [Category Processing](#catproc)\n",
    "4. [Split the dataset into Train, Val and Test](#split)\n",
    "4. [Save the model](#save)\n",
    "\n",
    "Now that we got a feel for the data, let's do some data pre-processing and cleaning before moving towards modeling. \n",
    "\n",
    "In this section we will: \n",
    "\n",
    "- Process the documents in the dataset with spaCy using the de_core_news_lg pipeline models, so that we can build our baselines and models later on. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a7cf59ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import json\n",
    "import spacy\n",
    "from spacy.tokens import Doc\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ef79123d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make it deterministic \n",
    "import random \n",
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc257002",
   "metadata": {},
   "source": [
    "Load both pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "22f13edc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model(name): \n",
    "    nlp = spacy.load(name)\n",
    "    de_stop_words = nlp.Defaults.stop_words\n",
    "    return nlp, de_stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2e207914",
   "metadata": {},
   "outputs": [],
   "source": [
    "def len_doc(doc): \n",
    "    doc_length = len(doc)\n",
    "    return doc_length\n",
    "\n",
    "def len_doc_only_alpha(doc): \n",
    "    doc_length = sum([1 if token.is_alpha else 0 for token in doc])\n",
    "    return doc_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "42f15f72",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_dataset(data_path):\n",
    "    \"\"\"\n",
    "    Read a json file and return it as dataframe. \n",
    "    Each record should be on a list of json objects inside `data`\n",
    "    \n",
    "    args: \n",
    "        - data_path, str: path to the dataset to be read\n",
    "    returns: \n",
    "        - Pandas DataFrame with text and label as columns\n",
    "    \"\"\"\n",
    "    with open(data_path, 'r') as f: \n",
    "        data = json.loads(f.read())\n",
    "    \n",
    "    return pd.json_normalize(data, record_path = [\"data\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0d01172a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"tech_soft_none.json\"\n",
    "data_path = os.path.join(f\"./data/raw/{dataset_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2497af52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Standort Trovarit AG München, Deutschland</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Wir freuen uns auf Ihre Bewerbung unter Angabe...</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Qualifikation zur Heimleitung gemäß Heimperson...</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gute organisatorische und konzeptionelle Fähig...</td>\n",
       "      <td>soft</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Teamfähigkeit, hohe Flexibilität und Einsatzbe...</td>\n",
       "      <td>soft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label\n",
       "0          Standort Trovarit AG München, Deutschland  none\n",
       "1  Wir freuen uns auf Ihre Bewerbung unter Angabe...  none\n",
       "2  Qualifikation zur Heimleitung gemäß Heimperson...  tech\n",
       "3  Gute organisatorische und konzeptionelle Fähig...  soft\n",
       "4  Teamfähigkeit, hohe Flexibilität und Einsatzbe...  soft"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skills_df = read_dataset(data_path=data_path)\n",
    "skills_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5811634b",
   "metadata": {},
   "outputs": [],
   "source": [
    "large_model = \"de_core_news_lg\"\n",
    "\n",
    "nlp_lg, stop_words_lg = load_model(large_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8bbacc6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the extension on the doc so we can plot it later\n",
    "Doc.set_extension(\"doc_len\", getter=len_doc, force=True)\n",
    "Doc.set_extension(\"doc_len_alpha\", getter=len_doc_only_alpha, force=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53d55021",
   "metadata": {},
   "source": [
    "## 1. Data Normalization <a name=\"normdata\"></a>\n",
    "\n",
    "Dataframe will contain: \n",
    "- text: raw text\n",
    "- label: corresponding to the class\n",
    "- text_proc: normalized text without punctuation and stop words\n",
    "- tokens_norm: normalized tokens\n",
    "- tokens: raw tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "45dbbcfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(doc): \n",
    "    tokens_cleaned = []\n",
    "    for token in doc: \n",
    "        if not token.is_punct and not token.is_stop and not token.is_space:\n",
    "            tokens_cleaned.append(token.lemma_.lower())\n",
    "    return tokens_cleaned, \" \".join(tokens_cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "92452f6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.3 s, sys: 331 ms, total: 16.6 s\n",
      "Wall time: 16.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "docs = []\n",
    "tokens = []\n",
    "tokens_norm = []\n",
    "text_procs = []\n",
    "for doc in nlp_lg.pipe(skills_df['text'].to_list(), batch_size=100):\n",
    "    docs.append(doc)\n",
    "    tokens.append([token for token in doc])\n",
    "    tokens_normalized, text_proc = normalize(doc)\n",
    "    tokens_norm.append(tokens_normalized)\n",
    "    text_procs.append(text_proc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a9e45a69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>doc</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_norm</th>\n",
       "      <th>text_proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Standort Trovarit AG München, Deutschland</td>\n",
       "      <td>none</td>\n",
       "      <td>(Standort, Trovarit, AG, München, ,, Deutschland)</td>\n",
       "      <td>[Standort, Trovarit, AG, München, ,, Deutschland]</td>\n",
       "      <td>[standort, trovarit, münchen, deutschland]</td>\n",
       "      <td>standort trovarit münchen deutschland</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Wir freuen uns auf Ihre Bewerbung unter Angabe...</td>\n",
       "      <td>none</td>\n",
       "      <td>(Wir, freuen, uns, auf, Ihre, Bewerbung, unter...</td>\n",
       "      <td>[Wir, freuen, uns, auf, Ihre, Bewerbung, unter...</td>\n",
       "      <td>[freuen, bewerbung, angabe, gehaltsvorstellung...</td>\n",
       "      <td>freuen bewerbung angabe gehaltsvorstellung mög...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Qualifikation zur Heimleitung gemäß Heimperson...</td>\n",
       "      <td>tech</td>\n",
       "      <td>(Qualifikation, zur, Heimleitung, gemäß, Heimp...</td>\n",
       "      <td>[Qualifikation, zur, Heimleitung, gemäß, Heimp...</td>\n",
       "      <td>[qualifikation, heimleitung, gemäß, heimperson...</td>\n",
       "      <td>qualifikation heimleitung gemäß heimpersonalve...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label  \\\n",
       "0          Standort Trovarit AG München, Deutschland  none   \n",
       "1  Wir freuen uns auf Ihre Bewerbung unter Angabe...  none   \n",
       "2  Qualifikation zur Heimleitung gemäß Heimperson...  tech   \n",
       "\n",
       "                                                 doc  \\\n",
       "0  (Standort, Trovarit, AG, München, ,, Deutschland)   \n",
       "1  (Wir, freuen, uns, auf, Ihre, Bewerbung, unter...   \n",
       "2  (Qualifikation, zur, Heimleitung, gemäß, Heimp...   \n",
       "\n",
       "                                              tokens  \\\n",
       "0  [Standort, Trovarit, AG, München, ,, Deutschland]   \n",
       "1  [Wir, freuen, uns, auf, Ihre, Bewerbung, unter...   \n",
       "2  [Qualifikation, zur, Heimleitung, gemäß, Heimp...   \n",
       "\n",
       "                                         tokens_norm  \\\n",
       "0         [standort, trovarit, münchen, deutschland]   \n",
       "1  [freuen, bewerbung, angabe, gehaltsvorstellung...   \n",
       "2  [qualifikation, heimleitung, gemäß, heimperson...   \n",
       "\n",
       "                                           text_proc  \n",
       "0              standort trovarit münchen deutschland  \n",
       "1  freuen bewerbung angabe gehaltsvorstellung mög...  \n",
       "2  qualifikation heimleitung gemäß heimpersonalve...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skills_df['doc'] = pd.Series(docs)    \n",
    "skills_df['tokens'] = pd.Series(tokens)\n",
    "skills_df['tokens_norm'] = pd.Series(tokens_norm)\n",
    "skills_df['text_proc'] = pd.Series(text_procs)\n",
    "skills_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6fe7488",
   "metadata": {},
   "source": [
    "## 2. Delete duplicates for Text_Proc column only <a name=\"deldups\"></a>\n",
    "\n",
    "Delete duplicates from the dataframe, as it might cause issues during training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c42e05f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(910,)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dups_idx = skills_df[skills_df['text_proc'].duplicated() == True].index\n",
    "dups_idx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b07c47b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "soft    395\n",
       "none    374\n",
       "tech    141\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Approximately 10% duplicates after cleaning for soft and none classes\n",
    "skills_df.iloc[dups_idx]['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "61ce419b",
   "metadata": {},
   "outputs": [],
   "source": [
    "skills_df_processed = skills_df.drop(index=dups_idx)\n",
    "skills_df_processed.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c5dad89a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9339"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skills_df_processed['text_proc'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "09480d85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>doc</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_norm</th>\n",
       "      <th>text_proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Standort Trovarit AG München, Deutschland</td>\n",
       "      <td>none</td>\n",
       "      <td>(Standort, Trovarit, AG, München, ,, Deutschland)</td>\n",
       "      <td>[Standort, Trovarit, AG, München, ,, Deutschland]</td>\n",
       "      <td>[standort, trovarit, münchen, deutschland]</td>\n",
       "      <td>standort trovarit münchen deutschland</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Wir freuen uns auf Ihre Bewerbung unter Angabe...</td>\n",
       "      <td>none</td>\n",
       "      <td>(Wir, freuen, uns, auf, Ihre, Bewerbung, unter...</td>\n",
       "      <td>[Wir, freuen, uns, auf, Ihre, Bewerbung, unter...</td>\n",
       "      <td>[freuen, bewerbung, angabe, gehaltsvorstellung...</td>\n",
       "      <td>freuen bewerbung angabe gehaltsvorstellung mög...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Qualifikation zur Heimleitung gemäß Heimperson...</td>\n",
       "      <td>tech</td>\n",
       "      <td>(Qualifikation, zur, Heimleitung, gemäß, Heimp...</td>\n",
       "      <td>[Qualifikation, zur, Heimleitung, gemäß, Heimp...</td>\n",
       "      <td>[qualifikation, heimleitung, gemäß, heimperson...</td>\n",
       "      <td>qualifikation heimleitung gemäß heimpersonalve...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label  \\\n",
       "0          Standort Trovarit AG München, Deutschland  none   \n",
       "1  Wir freuen uns auf Ihre Bewerbung unter Angabe...  none   \n",
       "2  Qualifikation zur Heimleitung gemäß Heimperson...  tech   \n",
       "\n",
       "                                                 doc  \\\n",
       "0  (Standort, Trovarit, AG, München, ,, Deutschland)   \n",
       "1  (Wir, freuen, uns, auf, Ihre, Bewerbung, unter...   \n",
       "2  (Qualifikation, zur, Heimleitung, gemäß, Heimp...   \n",
       "\n",
       "                                              tokens  \\\n",
       "0  [Standort, Trovarit, AG, München, ,, Deutschland]   \n",
       "1  [Wir, freuen, uns, auf, Ihre, Bewerbung, unter...   \n",
       "2  [Qualifikation, zur, Heimleitung, gemäß, Heimp...   \n",
       "\n",
       "                                         tokens_norm  \\\n",
       "0         [standort, trovarit, münchen, deutschland]   \n",
       "1  [freuen, bewerbung, angabe, gehaltsvorstellung...   \n",
       "2  [qualifikation, heimleitung, gemäß, heimperson...   \n",
       "\n",
       "                                           text_proc  \n",
       "0              standort trovarit münchen deutschland  \n",
       "1  freuen bewerbung angabe gehaltsvorstellung mög...  \n",
       "2  qualifikation heimleitung gemäß heimpersonalve...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skills_df_processed.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4ad5d333",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n",
      "RangeIndex: 9339 entries, 0 to 9338\n",
      "Series name: text_proc\n",
      "Non-Null Count  Dtype \n",
      "--------------  ----- \n",
      "9339 non-null   object\n",
      "dtypes: object(1)\n",
      "memory usage: 73.1+ KB\n"
     ]
    }
   ],
   "source": [
    "skills_df_processed['text_proc'].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d3edda9",
   "metadata": {},
   "source": [
    "### Remove text_proc columns that are empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "480b2a15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    9338\n",
       "True        1\n",
       "Name: text_proc, dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_lt_one = skills_df_processed['text_proc'].apply(lambda d: len(d)<1).to_frame()\n",
    "df_lt_one['text_proc'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "01d6970e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([17], dtype='int64')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "empty_idx = df_lt_one[df_lt_one['text_proc'] == True].index\n",
    "empty_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "170f2986",
   "metadata": {},
   "outputs": [],
   "source": [
    "skills_df_processed.drop(index=empty_idx, inplace=True)\n",
    "skills_df_processed.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a3a92f12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>doc</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_norm</th>\n",
       "      <th>text_proc</th>\n",
       "      <th>cats</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Idealerweise Kenntnisse im Bereich der digital...</td>\n",
       "      <td>tech</td>\n",
       "      <td>(Idealerweise, Kenntnisse, im, Bereich, der, d...</td>\n",
       "      <td>[Idealerweise, Kenntnisse, im, Bereich, der, d...</td>\n",
       "      <td>[idealerweise, kenntnis, bereich, digital, sig...</td>\n",
       "      <td>idealerweise kenntnis bereich digital signalve...</td>\n",
       "      <td>{'tech': True, 'soft': False, 'none': False}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Freude am Kundenkontakt, eine hohe Kunden- und...</td>\n",
       "      <td>soft</td>\n",
       "      <td>(Freude, am, Kundenkontakt, ,, eine, hohe, Kun...</td>\n",
       "      <td>[Freude, am, Kundenkontakt, ,, eine, hohe, Kun...</td>\n",
       "      <td>[freude, kundenkontakt, hoch, kunden, serviceo...</td>\n",
       "      <td>freude kundenkontakt hoch kunden serviceorient...</td>\n",
       "      <td>{'tech': False, 'soft': True, 'none': False}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Jobs &amp; Projekte Kontakt Impressum Datenschutz</td>\n",
       "      <td>none</td>\n",
       "      <td>(Jobs, &amp;, Projekte, Kontakt, Impressum, Datens...</td>\n",
       "      <td>[Jobs, &amp;, Projekte, Kontakt, Impressum, Datens...</td>\n",
       "      <td>[job, projekt, kontakt, impressum, datenschutz]</td>\n",
       "      <td>job projekt kontakt impressum datenschutz</td>\n",
       "      <td>{'tech': False, 'soft': False, 'none': True}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text label  \\\n",
       "15  Idealerweise Kenntnisse im Bereich der digital...  tech   \n",
       "16  Freude am Kundenkontakt, eine hohe Kunden- und...  soft   \n",
       "17      Jobs & Projekte Kontakt Impressum Datenschutz  none   \n",
       "\n",
       "                                                  doc  \\\n",
       "15  (Idealerweise, Kenntnisse, im, Bereich, der, d...   \n",
       "16  (Freude, am, Kundenkontakt, ,, eine, hohe, Kun...   \n",
       "17  (Jobs, &, Projekte, Kontakt, Impressum, Datens...   \n",
       "\n",
       "                                               tokens  \\\n",
       "15  [Idealerweise, Kenntnisse, im, Bereich, der, d...   \n",
       "16  [Freude, am, Kundenkontakt, ,, eine, hohe, Kun...   \n",
       "17  [Jobs, &, Projekte, Kontakt, Impressum, Datens...   \n",
       "\n",
       "                                          tokens_norm  \\\n",
       "15  [idealerweise, kenntnis, bereich, digital, sig...   \n",
       "16  [freude, kundenkontakt, hoch, kunden, serviceo...   \n",
       "17    [job, projekt, kontakt, impressum, datenschutz]   \n",
       "\n",
       "                                            text_proc  \\\n",
       "15  idealerweise kenntnis bereich digital signalve...   \n",
       "16  freude kundenkontakt hoch kunden serviceorient...   \n",
       "17          job projekt kontakt impressum datenschutz   \n",
       "\n",
       "                                            cats  \n",
       "15  {'tech': True, 'soft': False, 'none': False}  \n",
       "16  {'tech': False, 'soft': True, 'none': False}  \n",
       "17  {'tech': False, 'soft': False, 'none': True}  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skills_df_processed[15:18]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60ebadf3",
   "metadata": {},
   "source": [
    "## 3. Category processing <a name=\"catproc\"></a>\n",
    "\n",
    "We need to process the categories for the model, so that the spaCy models can processed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a20ed823",
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_categories(label): \n",
    "    \"\"\"\n",
    "    set_categories will create a dictionary for each label to be passed by assigning True \n",
    "    to the label that match and false otherwise.\n",
    "    \n",
    "    Example: \n",
    "    input: 'tech'\n",
    "    \n",
    "    output: {'tech': True, 'soft': False, 'none': False}\n",
    "    \n",
    "    args: \n",
    "    label: str, representing the category \n",
    "    \n",
    "    return: \n",
    "    dict, representing the categories. \n",
    "    \"\"\"\n",
    "    return dict({'tech': label == 'tech', \n",
    "            'soft': label == 'soft',\n",
    "            'none': label == 'none'\n",
    "           })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e58a335d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>doc</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_norm</th>\n",
       "      <th>text_proc</th>\n",
       "      <th>cats</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Standort Trovarit AG München, Deutschland</td>\n",
       "      <td>none</td>\n",
       "      <td>(Standort, Trovarit, AG, München, ,, Deutschland)</td>\n",
       "      <td>[Standort, Trovarit, AG, München, ,, Deutschland]</td>\n",
       "      <td>[standort, trovarit, münchen, deutschland]</td>\n",
       "      <td>standort trovarit münchen deutschland</td>\n",
       "      <td>{'tech': False, 'soft': False, 'none': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Wir freuen uns auf Ihre Bewerbung unter Angabe...</td>\n",
       "      <td>none</td>\n",
       "      <td>(Wir, freuen, uns, auf, Ihre, Bewerbung, unter...</td>\n",
       "      <td>[Wir, freuen, uns, auf, Ihre, Bewerbung, unter...</td>\n",
       "      <td>[freuen, bewerbung, angabe, gehaltsvorstellung...</td>\n",
       "      <td>freuen bewerbung angabe gehaltsvorstellung mög...</td>\n",
       "      <td>{'tech': False, 'soft': False, 'none': True}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Qualifikation zur Heimleitung gemäß Heimperson...</td>\n",
       "      <td>tech</td>\n",
       "      <td>(Qualifikation, zur, Heimleitung, gemäß, Heimp...</td>\n",
       "      <td>[Qualifikation, zur, Heimleitung, gemäß, Heimp...</td>\n",
       "      <td>[qualifikation, heimleitung, gemäß, heimperson...</td>\n",
       "      <td>qualifikation heimleitung gemäß heimpersonalve...</td>\n",
       "      <td>{'tech': True, 'soft': False, 'none': False}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gute organisatorische und konzeptionelle Fähig...</td>\n",
       "      <td>soft</td>\n",
       "      <td>(Gute, organisatorische, und, konzeptionelle, ...</td>\n",
       "      <td>[Gute, organisatorische, und, konzeptionelle, ...</td>\n",
       "      <td>[organisatorisch, konzeptionell, fähigkeit]</td>\n",
       "      <td>organisatorisch konzeptionell fähigkeit</td>\n",
       "      <td>{'tech': False, 'soft': True, 'none': False}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Teamfähigkeit, hohe Flexibilität und Einsatzbe...</td>\n",
       "      <td>soft</td>\n",
       "      <td>(Teamfähigkeit, ,, hohe, Flexibilität, und, Ei...</td>\n",
       "      <td>[Teamfähigkeit, ,, hohe, Flexibilität, und, Ei...</td>\n",
       "      <td>[teamfähigkeit, hoch, flexibilität, einsatzber...</td>\n",
       "      <td>teamfähigkeit hoch flexibilität einsatzbereits...</td>\n",
       "      <td>{'tech': False, 'soft': True, 'none': False}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label  \\\n",
       "0          Standort Trovarit AG München, Deutschland  none   \n",
       "1  Wir freuen uns auf Ihre Bewerbung unter Angabe...  none   \n",
       "2  Qualifikation zur Heimleitung gemäß Heimperson...  tech   \n",
       "3  Gute organisatorische und konzeptionelle Fähig...  soft   \n",
       "4  Teamfähigkeit, hohe Flexibilität und Einsatzbe...  soft   \n",
       "\n",
       "                                                 doc  \\\n",
       "0  (Standort, Trovarit, AG, München, ,, Deutschland)   \n",
       "1  (Wir, freuen, uns, auf, Ihre, Bewerbung, unter...   \n",
       "2  (Qualifikation, zur, Heimleitung, gemäß, Heimp...   \n",
       "3  (Gute, organisatorische, und, konzeptionelle, ...   \n",
       "4  (Teamfähigkeit, ,, hohe, Flexibilität, und, Ei...   \n",
       "\n",
       "                                              tokens  \\\n",
       "0  [Standort, Trovarit, AG, München, ,, Deutschland]   \n",
       "1  [Wir, freuen, uns, auf, Ihre, Bewerbung, unter...   \n",
       "2  [Qualifikation, zur, Heimleitung, gemäß, Heimp...   \n",
       "3  [Gute, organisatorische, und, konzeptionelle, ...   \n",
       "4  [Teamfähigkeit, ,, hohe, Flexibilität, und, Ei...   \n",
       "\n",
       "                                         tokens_norm  \\\n",
       "0         [standort, trovarit, münchen, deutschland]   \n",
       "1  [freuen, bewerbung, angabe, gehaltsvorstellung...   \n",
       "2  [qualifikation, heimleitung, gemäß, heimperson...   \n",
       "3        [organisatorisch, konzeptionell, fähigkeit]   \n",
       "4  [teamfähigkeit, hoch, flexibilität, einsatzber...   \n",
       "\n",
       "                                           text_proc  \\\n",
       "0              standort trovarit münchen deutschland   \n",
       "1  freuen bewerbung angabe gehaltsvorstellung mög...   \n",
       "2  qualifikation heimleitung gemäß heimpersonalve...   \n",
       "3            organisatorisch konzeptionell fähigkeit   \n",
       "4  teamfähigkeit hoch flexibilität einsatzbereits...   \n",
       "\n",
       "                                           cats  \n",
       "0  {'tech': False, 'soft': False, 'none': True}  \n",
       "1  {'tech': False, 'soft': False, 'none': True}  \n",
       "2  {'tech': True, 'soft': False, 'none': False}  \n",
       "3  {'tech': False, 'soft': True, 'none': False}  \n",
       "4  {'tech': False, 'soft': True, 'none': False}  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skills_df_processed['cats'] = skills_df_processed['label'].apply(set_categories)\n",
    "skills_df_processed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1418df7e",
   "metadata": {},
   "source": [
    "## 4. Split dataset into Training, Validation and Testing <a name=\"split\"></a>\n",
    "\n",
    "Perform data splitting for the dataset with 3 slices with the following proportions: \n",
    "\n",
    "- Training: 69% (6508)\n",
    "- Validation: 15% (1429)\n",
    "- Testing: 15% (1401)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "af5bcc13",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_raw, X_test, y_train_raw, y_test = train_test_split(skills_df_processed.drop(['label','doc'], axis=1),\n",
    "                                                    skills_df_processed[['label']], \n",
    "                                                    random_state=42, \n",
    "                                                    test_size=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "27d17cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(X_train_raw,\n",
    "                                                  y_train_raw, \n",
    "                                                  random_state=42, \n",
    "                                                  test_size=0.18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "f476c1db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6508, 1429, 1401)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape[0], X_val.shape[0], X_test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "72c6c2b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Splits \n",
      "Training:69.69 \n",
      "Dev:15.30  \n",
      "Test:15.00\n"
     ]
    }
   ],
   "source": [
    "print(f\"Splits \\nTraining:{X_train.shape[0]/skills_df_processed.shape[0]*100:.2f} \\nDev:{X_val.shape[0]/skills_df_processed.shape[0]*100:.2f}  \\nTest:{X_test.shape[0]/skills_df_processed.shape[0]*100:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e912ac6b",
   "metadata": {},
   "source": [
    "## 5. Save the data to build the models <a name=\"save\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "8d555659",
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_csv(path_to_save, filename, features, label):\n",
    "    full_path = os.path.join(path_to_save, filename)\n",
    "    pd.concat([features,label], axis=1).to_csv(full_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "d494611c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p ./data/processed/clean/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "d0797076",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_csv(\"./data/processed/clean\", \"training.csv\", X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "a4f2c851",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_csv(\"./data/processed/clean\", \"dev.csv\", X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "55c706e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_csv(\"./data/processed/clean\", \"test.csv\", X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "662668f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text,tokens,tokens_norm,text_proc,cats,label\r\n",
      "Blogs Foto Video Leserartikel Print-Archiv Schlagworte Bildrechte AGB Cookies Hilfe/ Kontakt Newsletter RSS,\"[Blogs, Foto, Video, Leserartikel, Print-Archiv, Schlagworte, Bildrechte, AGB, Cookies, Hilfe, /, Kontakt, Newsletter, RSS]\",\"['blog', 'foto', 'video', 'leserartikel', 'print-archiv', 'schlagwort', 'bildrechte', 'agb', 'cookies', 'hilfe', 'kontakt', 'newsletter', 'rss']\",blog foto video leserartikel print-archiv schlagwort bildrechte agb cookies hilfe kontakt newsletter rss,\"{'tech': False, 'soft': False, 'none': True}\",none\r\n",
      "Braunschweig,[Braunschweig],['braunschweig'],braunschweig,\"{'tech': False, 'soft': False, 'none': True}\",none\r\n"
     ]
    }
   ],
   "source": [
    "!head -n 3 ./data/processed/clean/training.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d1b62f7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baf94125",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}