main_submission.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_58979/3326930799.py:3: DeprecationWarning: \n",
      "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n",
      "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n",
      "but was not found to be installed on your system.\n",
      "If this would cause problems for you,\n",
      "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n",
      "        \n",
      "  import pandas as pd\n"
     ]
    }
   ],
   "source": [
    "# Importing necessary libraries/modules for the task\n",
    "import requests\n",
    "import pprint\n",
    "import pandas as pd\n",
    "import imaplib\n",
    "import email"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping Jobs and Job Descriptions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "from scrapers.mycareersfuture import MyCareersFuture # Importing the MyCareersFuture class from the scrapers.mycareersfuture module\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MyCareersFuture() # Creating an instance of the MyCareersFuture scraper class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total pages 63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 62/62 [02:32<00:00,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted 6225 jobs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data = client.collect(sleep_time=2)  # Calling the 'collect' method of the MyCareersFuture scraper instance to initiate the scraping process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6225/6225 [41:09<00:00,  2.52it/s]  \n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm  # Importing tqdm for progress tracking\n",
    "import time  # Importing the time module for time-related operations\n",
    "\n",
    "count = 0  # Initializing a counter variable\n",
    "\n",
    "# Iterating over each item in the 'data' variable with tqdm for progress tracking\n",
    "for i in tqdm(data):\n",
    "    id = i['uuid']  # Extracting the UUID of the job listing\n",
    "    response = client.get_job_data(id)  # Getting job data for the specific UUID\n",
    "    i['job_data'] = response  # Storing the job data in the 'job_data' field of the item\n",
    "    count += 1  # Incrementing the counter variable\n",
    "\n",
    "    # Checking if 30 iterations have been completed to implement a delay\n",
    "    if count % 30 == 0:\n",
    "        time.sleep(10)  # Waiting for 10 seconds to avoid overwhelming the server with requests\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.get_job_data('aec44687225e1c1f74b918d617490ae3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using pandas to normalize the 'job_data' field of the DataFrame and converting it to JSON format\n",
    "pd.json_normalize(pd.DataFrame(data)['job_data'],max_level=0).to_json(\"mycareersfuture_with_descriptions.json\",orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_json(\"mycareersfuture_with_descriptions.json\",orient=\"records\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning Scraped Data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def load_and_clean_data(file_path):\n",
    "    \"\"\"\n",
    "    Load and clean the scraped data from a JSON file.\n",
    "\n",
    "    Parameters:\n",
    "    file_path (str): The file path to the JSON file containing the scraped data.\n",
    "\n",
    "    Returns:\n",
    "    pandas.DataFrame: A cleaned DataFrame containing the scraped data.\n",
    "    \"\"\"\n",
    "    # Read the JSON file into a DataFrame\n",
    "    df = pd.read_json(file_path, orient='records')\n",
    "\n",
    "    # Extract salary information and create separate columns for minimum, maximum, and type\n",
    "    if 'salary' in df.columns:\n",
    "        df['salary_min'] = None\n",
    "        df['salary_max'] = None\n",
    "        df['salary_type'] = None\n",
    "        for index, row in df.iterrows():\n",
    "            salary_info = row['salary']\n",
    "            if salary_info is not None:\n",
    "                df.at[index, 'salary_min'] = salary_info.get('minimum')\n",
    "                df.at[index, 'salary_max'] = salary_info.get('maximum')\n",
    "                if 'type' in salary_info and salary_info['type'] is not None:\n",
    "                    df.at[index, 'salary_type'] = salary_info['type'].get('salaryType')\n",
    "        df.drop(columns=['salary'], inplace=True)\n",
    "\n",
    "    # Normalize and expand company information into separate columns\n",
    "    if 'postedCompany' in df.columns:\n",
    "        company_info = pd.json_normalize(df['postedCompany'].dropna())\n",
    "        company_info.columns = ['company_' + col for col in company_info.columns]\n",
    "        df = pd.concat([df.drop(columns=['postedCompany']), company_info], axis=1)\n",
    "\n",
    "    # Normalize lists of skills, categories, employment types, and position levels\n",
    "    list_columns = ['skills', 'categories', 'employmentTypes', 'positionLevels']\n",
    "    for column in list_columns:\n",
    "        df[column] = df[column].apply(lambda x: [item.get(column[:-1], '') for item in x] if isinstance(x, list) else [])\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning Job Descriptions in Cleaned Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def clean_job_descriptions(df_uncleaned, df_clean):\n",
    "    \"\"\"\n",
    "    Clean job descriptions and requirements from uncleaned DataFrame and update the cleaned DataFrame.\n",
    "\n",
    "    Parameters:\n",
    "    df_uncleaned (pandas.DataFrame): DataFrame containing uncleaned job data.\n",
    "    df_clean (pandas.DataFrame): DataFrame containing cleaned job data.\n",
    "\n",
    "    Returns:\n",
    "    pandas.DataFrame: DataFrame with cleaned job descriptions and requirements.\n",
    "    \"\"\"\n",
    "    # Define keywords for job descriptions and job requirements\n",
    "    jd_keywords = ['things you need to do', 'job description', 'job duties', 'responsibilities', 'job description', 'duties', 'key responsibilities', 'job details']\n",
    "    jr_keywords = ['things you need to have', 'qualification', 'requirements', 'job details', 'qualifications']\n",
    "\n",
    "    jobpostid_list = {}\n",
    "\n",
    "    # Iterate over each row in the uncleaned DataFrame\n",
    "    for index, row in df_uncleaned.iterrows():\n",
    "        soup = BeautifulSoup(row['description'], 'html.parser')\n",
    "        job_description_items, job_requirements_items = [], []\n",
    "\n",
    "        # Adjusted lambda function to check for None before calling .lower()\n",
    "        def keyword_in_text(tag, keywords):\n",
    "            if tag and tag.string:\n",
    "                text = tag.string\n",
    "                return any(keyword in text.lower() for keyword in keywords)\n",
    "            return False\n",
    "\n",
    "        # Process Job Descriptions\n",
    "        job_description_strong = soup.find('strong', string=lambda t: keyword_in_text(t, jd_keywords))\n",
    "        if job_description_strong:\n",
    "            ul_tag = job_description_strong.find_next('ul')\n",
    "            if ul_tag:\n",
    "                job_description_items = [li.text.strip() for li in ul_tag.find_all('li')]\n",
    "\n",
    "        # Process Job Requirements\n",
    "        job_requirements_strong = soup.find('strong', string=lambda t: keyword_in_text(t, jr_keywords))\n",
    "        if job_requirements_strong:\n",
    "            ul_tag = job_requirements_strong.find_next('ul')\n",
    "            if ul_tag:\n",
    "                job_requirements_items = [li.text.strip() for li in ul_tag.find_all('li')]\n",
    "\n",
    "        if job_description_items or job_requirements_items:\n",
    "            jobpostid_list[row['metadata']['jobPostId']] = {\n",
    "                'job_descriptions': job_description_items,\n",
    "                'job_requirements': job_requirements_items\n",
    "            }\n",
    "\n",
    "    # Update df_clean with the structured job descriptions and requirements\n",
    "    clean_job_descriptions = []\n",
    "    for index, row in df_clean.iterrows():\n",
    "        job_info = jobpostid_list.get(row['metadata']['jobPostId'])\n",
    "        if job_info:\n",
    "            clean_description = \"Job Description:\\n\" + '\\n'.join(job_info['job_descriptions']) + \"\\n\\nJob Requirements:\\n\" + '\\n'.join(job_info['job_requirements'])\n",
    "            clean_job_descriptions.append(clean_description)\n",
    "        else:\n",
    "            clean_job_descriptions.append(None)\n",
    "\n",
    "    df_clean['Job Description'] = clean_job_descriptions\n",
    "    return df_clean\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned data saved to cleaned_job_data.csv\n"
     ]
    }
   ],
   "source": [
    "def main():\n",
    "    file_path = 'mycareersfuture_with_descriptions.json'  \n",
    "    df_clean = load_and_clean_data(file_path)  # Load and initially clean your data\n",
    "    df_uncleaned = pd.read_json(file_path, orient='records') \n",
    "    df_final = clean_job_descriptions(df_uncleaned, df_clean)  # Apply job descriptions cleaning\n",
    "    subset_df_clean = df_final[df_final['Job Description'].notna()]\n",
    "    subset_df_clean\n",
    "    # Save the cleaned DataFrame to a new CSV file\n",
    "    subset_df_clean.to_csv('cleaned_job_data1.csv', index=False)\n",
    "    print(\"Cleaned data saved to cleaned_job_data.csv\")\n",
    "    \n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>sourceCode</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>minimumYearsExperience</th>\n",
       "      <th>shiftPattern</th>\n",
       "      <th>schemes</th>\n",
       "      <th>skills</th>\n",
       "      <th>otherRequirements</th>\n",
       "      <th>ssocCode</th>\n",
       "      <th>...</th>\n",
       "      <th>company_lastSyncDate</th>\n",
       "      <th>company_ssicCode2020</th>\n",
       "      <th>company_badges</th>\n",
       "      <th>company_logoFileName</th>\n",
       "      <th>company_logoUploadPath</th>\n",
       "      <th>company__links.self.href</th>\n",
       "      <th>company__links.jobs.href</th>\n",
       "      <th>company__links.addresses.href</th>\n",
       "      <th>company__links.schemes.href</th>\n",
       "      <th>Job Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8cebe95d75634f77e04130d74bbeee98</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Enrolled Nurse</td>\n",
       "      <td>&lt;h2&gt;&lt;strong&gt;Duties and Responsibilities&lt;/stron...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Nursing Homes', 'BCLS', 'Work Well Independe...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32200</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T01:21:20.000Z</td>\n",
       "      <td>87010</td>\n",
       "      <td>[]</td>\n",
       "      <td>107b61cad03e8efab8b6c56f4c3242cc/lentor-reside...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nProvide quality nursing care...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>589ec3cb56c1979d9d1c158490e66870</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Sales Development Representative</td>\n",
       "      <td>&lt;p&gt;&lt;u&gt;&lt;strong&gt;Job Responsibilities&lt;/strong&gt;&lt;/u...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['CRM', 'Lead Generation', 'Microsoft Excel', ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24333</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T01:28:12.000Z</td>\n",
       "      <td>70201</td>\n",
       "      <td>[]</td>\n",
       "      <td>60b9f613063dd0513340e97dcc9cc9f8/stone-forest-...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nIdentify new business opport...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000f0be7e4c9d857f4fcec17b3fc6b0</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Assistant Coach [ Special Education / Siglap /...</td>\n",
       "      <td>&lt;p&gt;&lt;strong&gt;Assistant Coach (Special Education)...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Leadership', 'Classroom', 'Teaching', 'Class...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23622</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T00:13:15.000Z</td>\n",
       "      <td>78104</td>\n",
       "      <td>[]</td>\n",
       "      <td>d162667fbe1365aa541016a407de7ae4/supreme-hr-ad...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nAssist and support Coaches i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>874e52b4f633cb30687a5985b272b1ed</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Production Operator #65770</td>\n",
       "      <td>&lt;p&gt;&lt;strong&gt;Job Description&lt;/strong&gt;&lt;/p&gt;\\n&lt;ul&gt;\\...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Factory', 'Soldering', 'Housekeeping', 'Work...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31164</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T03:05:25.000Z</td>\n",
       "      <td>78104</td>\n",
       "      <td>[]</td>\n",
       "      <td>737b2bbd5f7b6aa4a3e393db45fe9479/ANRADUS PTE. ...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nIndustry/ Organization Type:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b3260c8354d8cfa26675db2eea497cc5</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Retail Assistant #65750</td>\n",
       "      <td>&lt;p&gt;&lt;strong&gt;Job Description&lt;/strong&gt;&lt;/p&gt;\\n&lt;ul&gt;\\...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Store Operations', 'Customer Service Oriente...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52202</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T03:05:25.000Z</td>\n",
       "      <td>78104</td>\n",
       "      <td>[]</td>\n",
       "      <td>737b2bbd5f7b6aa4a3e393db45fe9479/ANRADUS PTE. ...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nIndustry/ Organization Type:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2843</th>\n",
       "      <td>2eef1f9ecf2286b8d0a25143054584c3</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Retail Sales Assistant (Fashion Accessories Wh...</td>\n",
       "      <td>&lt;p&gt;We are a fashion jewellery brand but also r...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Sales', 'Visual Merchandising', 'Housekeepin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52202</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T07:46:53.000Z</td>\n",
       "      <td>47109</td>\n",
       "      <td>[]</td>\n",
       "      <td>bb87069f2d459ae2633c3d84141d66a1/yesseny-tradi...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nProactively provide professi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2844</th>\n",
       "      <td>2c25fc6c2986af6f9cd2408a72884b6e</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>English Language Teacher [L1/L2]</td>\n",
       "      <td>&lt;ul&gt;\\n  &lt;li&gt;&lt;strong&gt;Aljunied&lt;/strong&gt;&lt;/li&gt;\\n  ...</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Coaching', 'Childcare', 'Academic English', ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36100</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T01:05:53.000Z</td>\n",
       "      <td>78104</td>\n",
       "      <td>[]</td>\n",
       "      <td>b2bf56fc25ba3a759e198a68aa0d4ce3/allied-search...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nMaintain and upkeep a conduc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2845</th>\n",
       "      <td>d2d7dd45575fe206f767f79e4b1591e1</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Superstar Front Desk (Location: Ang Mo Kio)</td>\n",
       "      <td>&lt;p&gt;We at De Pacific Dental Group are in search...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Dental Sales', 'Search', 'Microsoft Works', ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>42243</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-02-26T02:41:29.000Z</td>\n",
       "      <td>86204</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\n\\n\\nJob Requirements:\\nFull ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2846</th>\n",
       "      <td>3973d6ab277c9cc3b9257c63c3720173</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Individual Support Specialist (Special Needs) ...</td>\n",
       "      <td>&lt;p&gt;&lt;strong&gt;Job Description&lt;/strong&gt;&lt;/p&gt;\\n&lt;ul&gt;\\...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Psychology', 'Discipline', 'Classroom', 'Cla...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23622</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-06T03:05:25.000Z</td>\n",
       "      <td>78104</td>\n",
       "      <td>[]</td>\n",
       "      <td>737b2bbd5f7b6aa4a3e393db45fe9479/ANRADUS PTE. ...</td>\n",
       "      <td>https://static.mycareersfuture.gov.sg/images/c...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nIndustry/ Organization Type:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2847</th>\n",
       "      <td>3253ffa2692b137322b4b720316927f7</td>\n",
       "      <td>Employer Portal</td>\n",
       "      <td>Business Development Manager</td>\n",
       "      <td>&lt;p&gt;&lt;strong&gt;Job Description:&lt;/strong&gt;&lt;/p&gt;\\n&lt;p&gt;A...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Negotiation', 'CRM', 'Account Management', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33221</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-03T05:20:37.000Z</td>\n",
       "      <td>62090</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>https://api.mycareersfuture.gov.sg/v2/companie...</td>\n",
       "      <td>Job Description:\\nIdentify and pursue new busi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2848 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  uuid       sourceCode  \\\n",
       "0     8cebe95d75634f77e04130d74bbeee98  Employer Portal   \n",
       "1     589ec3cb56c1979d9d1c158490e66870  Employer Portal   \n",
       "2     0000f0be7e4c9d857f4fcec17b3fc6b0  Employer Portal   \n",
       "3     874e52b4f633cb30687a5985b272b1ed  Employer Portal   \n",
       "4     b3260c8354d8cfa26675db2eea497cc5  Employer Portal   \n",
       "...                                ...              ...   \n",
       "2843  2eef1f9ecf2286b8d0a25143054584c3  Employer Portal   \n",
       "2844  2c25fc6c2986af6f9cd2408a72884b6e  Employer Portal   \n",
       "2845  d2d7dd45575fe206f767f79e4b1591e1  Employer Portal   \n",
       "2846  3973d6ab277c9cc3b9257c63c3720173  Employer Portal   \n",
       "2847  3253ffa2692b137322b4b720316927f7  Employer Portal   \n",
       "\n",
       "                                                  title  \\\n",
       "0                                        Enrolled Nurse   \n",
       "1                      Sales Development Representative   \n",
       "2     Assistant Coach [ Special Education / Siglap /...   \n",
       "3                            Production Operator #65770   \n",
       "4                               Retail Assistant #65750   \n",
       "...                                                 ...   \n",
       "2843  Retail Sales Assistant (Fashion Accessories Wh...   \n",
       "2844                   English Language Teacher [L1/L2]   \n",
       "2845        Superstar Front Desk (Location: Ang Mo Kio)   \n",
       "2846  Individual Support Specialist (Special Needs) ...   \n",
       "2847                       Business Development Manager   \n",
       "\n",
       "                                            description  \\\n",
       "0     <h2><strong>Duties and Responsibilities</stron...   \n",
       "1     <p><u><strong>Job Responsibilities</strong></u...   \n",
       "2     <p><strong>Assistant Coach (Special Education)...   \n",
       "3     <p><strong>Job Description</strong></p>\\n<ul>\\...   \n",
       "4     <p><strong>Job Description</strong></p>\\n<ul>\\...   \n",
       "...                                                 ...   \n",
       "2843  <p>We are a fashion jewellery brand but also r...   \n",
       "2844  <ul>\\n  <li><strong>Aljunied</strong></li>\\n  ...   \n",
       "2845  <p>We at De Pacific Dental Group are in search...   \n",
       "2846  <p><strong>Job Description</strong></p>\\n<ul>\\...   \n",
       "2847  <p><strong>Job Description:</strong></p>\\n<p>A...   \n",
       "\n",
       "      minimumYearsExperience  shiftPattern schemes  \\\n",
       "0                          1           NaN      []   \n",
       "1                          0           NaN      []   \n",
       "2                          0           NaN      []   \n",
       "3                          0           NaN      []   \n",
       "4                          0           NaN      []   \n",
       "...                      ...           ...     ...   \n",
       "2843                       1           NaN      []   \n",
       "2844                       2           NaN      []   \n",
       "2845                       1           NaN      []   \n",
       "2846                       0           NaN      []   \n",
       "2847                       0           NaN      []   \n",
       "\n",
       "                                                 skills  otherRequirements  \\\n",
       "0     ['Nursing Homes', 'BCLS', 'Work Well Independe...                NaN   \n",
       "1     ['CRM', 'Lead Generation', 'Microsoft Excel', ...                NaN   \n",
       "2     ['Leadership', 'Classroom', 'Teaching', 'Class...                NaN   \n",
       "3     ['Factory', 'Soldering', 'Housekeeping', 'Work...                NaN   \n",
       "4     ['Store Operations', 'Customer Service Oriente...                NaN   \n",
       "...                                                 ...                ...   \n",
       "2843  ['Sales', 'Visual Merchandising', 'Housekeepin...                NaN   \n",
       "2844  ['Coaching', 'Childcare', 'Academic English', ...                NaN   \n",
       "2845  ['Dental Sales', 'Search', 'Microsoft Works', ...                NaN   \n",
       "2846  ['Psychology', 'Discipline', 'Classroom', 'Cla...                NaN   \n",
       "2847  ['Negotiation', 'CRM', 'Account Management', '...                NaN   \n",
       "\n",
       "      ssocCode  ...      company_lastSyncDate company_ssicCode2020  \\\n",
       "0        32200  ...  2024-03-06T01:21:20.000Z                87010   \n",
       "1        24333  ...  2024-03-06T01:28:12.000Z                70201   \n",
       "2        23622  ...  2024-03-06T00:13:15.000Z                78104   \n",
       "3        31164  ...  2024-03-06T03:05:25.000Z                78104   \n",
       "4        52202  ...  2024-03-06T03:05:25.000Z                78104   \n",
       "...        ...  ...                       ...                  ...   \n",
       "2843     52202  ...  2024-03-06T07:46:53.000Z                47109   \n",
       "2844     36100  ...  2024-03-06T01:05:53.000Z                78104   \n",
       "2845     42243  ...  2024-02-26T02:41:29.000Z                86204   \n",
       "2846     23622  ...  2024-03-06T03:05:25.000Z                78104   \n",
       "2847     33221  ...  2024-03-03T05:20:37.000Z                62090   \n",
       "\n",
       "      company_badges                               company_logoFileName  \\\n",
       "0                 []  107b61cad03e8efab8b6c56f4c3242cc/lentor-reside...   \n",
       "1                 []  60b9f613063dd0513340e97dcc9cc9f8/stone-forest-...   \n",
       "2                 []  d162667fbe1365aa541016a407de7ae4/supreme-hr-ad...   \n",
       "3                 []  737b2bbd5f7b6aa4a3e393db45fe9479/ANRADUS PTE. ...   \n",
       "4                 []  737b2bbd5f7b6aa4a3e393db45fe9479/ANRADUS PTE. ...   \n",
       "...              ...                                                ...   \n",
       "2843              []  bb87069f2d459ae2633c3d84141d66a1/yesseny-tradi...   \n",
       "2844              []  b2bf56fc25ba3a759e198a68aa0d4ce3/allied-search...   \n",
       "2845              []                                                NaN   \n",
       "2846              []  737b2bbd5f7b6aa4a3e393db45fe9479/ANRADUS PTE. ...   \n",
       "2847              []                                                NaN   \n",
       "\n",
       "                                 company_logoUploadPath  \\\n",
       "0     https://static.mycareersfuture.gov.sg/images/c...   \n",
       "1     https://static.mycareersfuture.gov.sg/images/c...   \n",
       "2     https://static.mycareersfuture.gov.sg/images/c...   \n",
       "3     https://static.mycareersfuture.gov.sg/images/c...   \n",
       "4     https://static.mycareersfuture.gov.sg/images/c...   \n",
       "...                                                 ...   \n",
       "2843  https://static.mycareersfuture.gov.sg/images/c...   \n",
       "2844  https://static.mycareersfuture.gov.sg/images/c...   \n",
       "2845                                                NaN   \n",
       "2846  https://static.mycareersfuture.gov.sg/images/c...   \n",
       "2847                                                NaN   \n",
       "\n",
       "                               company__links.self.href  \\\n",
       "0     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "1     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "3     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "4     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "...                                                 ...   \n",
       "2843  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2844  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2845  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2846  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2847  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "\n",
       "                               company__links.jobs.href  \\\n",
       "0     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "1     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "3     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "4     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "...                                                 ...   \n",
       "2843  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2844  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2845  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2846  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2847  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "\n",
       "                          company__links.addresses.href  \\\n",
       "0     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "1     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "3     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "4     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "...                                                 ...   \n",
       "2843  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2844  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2845  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2846  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2847  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "\n",
       "                            company__links.schemes.href  \\\n",
       "0     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "1     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "3     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "4     https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "...                                                 ...   \n",
       "2843  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2844  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2845  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2846  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "2847  https://api.mycareersfuture.gov.sg/v2/companie...   \n",
       "\n",
       "                                        Job Description  \n",
       "0     Job Description:\\nProvide quality nursing care...  \n",
       "1     Job Description:\\nIdentify new business opport...  \n",
       "2     Job Description:\\nAssist and support Coaches i...  \n",
       "3     Job Description:\\nIndustry/ Organization Type:...  \n",
       "4     Job Description:\\nIndustry/ Organization Type:...  \n",
       "...                                                 ...  \n",
       "2843  Job Description:\\nProactively provide professi...  \n",
       "2844  Job Description:\\nMaintain and upkeep a conduc...  \n",
       "2845  Job Description:\\n\\n\\nJob Requirements:\\nFull ...  \n",
       "2846  Job Description:\\nIndustry/ Organization Type:...  \n",
       "2847  Job Description:\\nIdentify and pursue new busi...  \n",
       "\n",
       "[2848 rows x 45 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_data = pd.read_csv('cleaned_job_data1.csv')  # Reading the cleaned job data from a CSV file into a DataFrame\n",
    "job_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Description:\n",
      "Identify new business opportunities through outbound lead generation, research and prospecting\n",
      "Manage and respond to inbound requests from prospective clients via cold calling and email\n",
      "Secure meetings for the GWI sales organization\n",
      "Analyse customer needs and identify potential solutions\n",
      "Present and explain GWI product offerings\n",
      "Prospect and make outbound calls and emails to potential clients\n",
      "\n",
      "Job Requirements:\n",
      "Proven experience of relevant work or internship\n",
      "Excellent communication and presentation skills\n",
      "Effective time management and organization skills\n",
      "Good MS Office Skills\n"
     ]
    }
   ],
   "source": [
    "print(job_data.iloc[1]['Job Description'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embedding Job Descriptions into ChromaDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import chromadb\n",
    "from chromadb.utils import embedding_functions\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "def main():\n",
    "    \"\"\"\n",
    "    Main function to embed job descriptions into ChromaDB.\n",
    "    \"\"\"\n",
    "    documents = []  # List to store job descriptions\n",
    "    metadatas = []  # List to store metadata associated with each job description\n",
    "    ids = []  # List to store unique IDs for each job description\n",
    "\n",
    "    # Read cleaned job descriptions and metadata from My Career Future\n",
    "    df = pd.read_json(\"https://talenttrove.s3.ap-southeast-1.amazonaws.com/mycareersfuture_with_descriptions_cleaned.json\")\n",
    "\n",
    "    # Read additional job descriptions from Glassdoor\n",
    "    df2 = pd.read_csv(\"https://talenttrove.s3.ap-southeast-1.amazonaws.com/ALL_Glass.csv\")\n",
    "\n",
    "    # Process job descriptions and metadata from My Career Future\n",
    "    for index, row in df.iterrows():\n",
    "        company_name = row[\"company_name\"] if row[\"company_name\"] else \"\"\n",
    "        company_logo = row[\"company_logo\"] if row[\"company_logo\"] else \"\"\n",
    "        title = row[\"title\"] if row[\"title\"] else \"\"\n",
    "        job_post_id = row[\"jobPostId\"] if row[\"jobPostId\"] else \"\"\n",
    "        apply_url = row[\"apply_url\"] if row[\"apply_url\"] else \"\"\n",
    "        date = row[\"updatedAt\"] if row[\"updatedAt\"] else \"\"\n",
    "        jobtype = str(row[\"positionLevels\"]) if row[\"positionLevels\"] else \"\"\n",
    "\n",
    "        documents.append(row[\"description\"])  # Append job description to documents list\n",
    "        metadatas.append({  # Append metadata to metadatas list\n",
    "            \"company_name\": company_name,\n",
    "            \"company_logo\": company_logo,\n",
    "            \"job_post_id\": job_post_id,\n",
    "            \"title\": title,\n",
    "            \"apply_url\": apply_url,\n",
    "            \"date\": date,\n",
    "            \"jobtype\": jobtype,\n",
    "        })\n",
    "        ids.append(str(index))  # Append unique ID to ids list\n",
    "\n",
    "    # Calculate max_id for later use in generating unique IDs\n",
    "    max_id = max(ids)\n",
    "\n",
    "    # Process additional job descriptions from Glassdoor\n",
    "    for index, row in df2.iterrows():\n",
    "        company_name = row[\"employerNameFromSearch\"] if row[\"employerNameFromSearch\"] else \"\"\n",
    "        company_logo = row[\"employer.squareLogoUrl\"] if row[\"employer.squareLogoUrl\"] else \"\"\n",
    "        title = row[\"jobTitleText\"] if row[\"jobTitleText\"] else \"\"\n",
    "        job_post_id = row[\"jobReqId\"] if row[\"jobReqId\"] else \"\"\n",
    "        apply_url = row[\"jobLink\"] if row[\"jobLink\"] else \"\"\n",
    "        date = row[\"discoverDate\"] if row[\"discoverDate\"] else \"\"\n",
    "        jobtype = str(row[\"jobType\"]) if row[\"jobType\"] else \"\"\n",
    "\n",
    "        documents.append(row[\"cleaned_description\"])  # Append job description to documents list\n",
    "        metadatas.append({  # Append metadata to metadatas list\n",
    "            \"company_name\": company_name,\n",
    "            \"company_logo\": company_logo,\n",
    "            \"job_post_id\": job_post_id,\n",
    "            \"title\": title,\n",
    "            \"apply_url\": apply_url,\n",
    "            \"date\": date,\n",
    "            \"jobtype\": jobtype,\n",
    "        })\n",
    "        ids.append(str(max_id) + str(index))  # Append unique ID to ids list\n",
    "\n",
    "    # Create directory for storing data if it doesn't exist\n",
    "    path = os.path.join(os.getcwd(), \"data\")\n",
    "    if not os.path.exists(path):\n",
    "        os.mkdir(path)\n",
    "\n",
    "    # Initialize ChromaDB client and collection\n",
    "    chroma_client = chromadb.PersistentClient(path=os.path.join(os.getcwd(), \"data/jd_vectordb\"))\n",
    "    collection = chroma_client.get_or_create_collection(\n",
    "        name=\"mycareersfuture_jd\",\n",
    "        embedding_function=embedding_functions.DefaultEmbeddingFunction(),\n",
    "    )\n",
    "\n",
    "    batch_size = 64  # Adjust batch size based on the maximum allowed\n",
    "    # Iterate over batches of documents and add them to the collection\n",
    "    for i in tqdm(range(0, len(documents), batch_size)):\n",
    "        batch_documents = documents[i: i + batch_size]\n",
    "        batch_metadatas = metadatas[i: i + batch_size]\n",
    "        batch_ids = ids[i: i + batch_size]\n",
    "\n",
    "        # Ensure order of documents, metadatas, and ids within each batch\n",
    "        assert len(batch_documents) == len(batch_metadatas) == len(batch_ids)\n",
    "\n",
    "        # Add batch to collection\n",
    "        collection.add(\n",
    "            documents=batch_documents,\n",
    "            metadatas=batch_metadatas,\n",
    "            ids=batch_ids\n",
    "        )\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Job Title and Company Name Extractors using Gemini\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HRT Application Status - Pratham Agarwala\n",
      "Hi Pratham, We want to thank you very much for your interest in Hudson River Trading\n",
      "Calling Gemini API\n",
      "Algorithm Developer\n",
      "Hudson River Trading\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
    "import torch\n",
    "from dotenv import load_dotenv\n",
    "import google.generativeai as genai\n",
    "\n",
    "load_dotenv()\n",
    "class JobTitleCompanyNameExtractor:\n",
    "    \"\"\"\n",
    "    Extract job titles and company names from email text using a pre-trained T5 model.\n",
    "    \"\"\"\n",
    "    def __init__(self, model=\"google/flan-t5-base\"):\n",
    "        self.model = AutoModelForSeq2SeqLM.from_pretrained(\"google/flan-t5-base\")\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(\"google/flan-t5-base\")\n",
    "\n",
    "    def get_jobtitle(self, email_text: str):\n",
    "        \"\"\"\n",
    "        Extracts the job title from the given email text.\n",
    "        \"\"\"\n",
    "        question = (\"What is the job title? If not available, output None\",)\n",
    "        input_text = f\"question: {question} context: {email_text}\"\n",
    "        inputs = self.tokenizer(input_text, return_tensors=\"pt\")\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = self.model.generate(**inputs)\n",
    "        answer = self.tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "        return answer\n",
    "\n",
    "    def get_company(self, email_text: str):\n",
    "        \"\"\"\n",
    "        Extracts the company name from the given email text.\n",
    "        \"\"\"\n",
    "        question = (\"What is the name of the company? If not available, output None\",)\n",
    "        input_text = f\"question: {question} context: {email_text}\"\n",
    "        inputs = self.tokenizer(input_text, return_tensors=\"pt\")\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = self.model.generate(**inputs)\n",
    "        answer = self.tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "        return answer\n",
    "\n",
    "class GeminiJobTitleCompanyNameExtractor:\n",
    "    \"\"\"\n",
    "    Extract job titles and company names from email text using the Gemini API.\n",
    "    \"\"\"\n",
    "    def __init__(self, model=\"google/flan-t5-base\"):\n",
    "        self.apikey = os.getenv(\"GOOGLE_API_KEY\")\n",
    "        genai.configure(api_key=self.apikey)\n",
    "        self.model = genai.GenerativeModel(\"gemini-pro\")\n",
    "        print(\"Calling Gemini API\")\n",
    "\n",
    "    def get_jobtitle(self, email_text: str):\n",
    "        \"\"\"\n",
    "        Extracts the job title from the given email text using the Gemini API.\n",
    "        \"\"\"\n",
    "        question = (\"What is the job title? If not available, output None\",)\n",
    "        input_text = f\"question: {question} context: {email_text}\"\n",
    "        response = self.model.generate_content(input_text)\n",
    "        return response.text\n",
    "\n",
    "    def get_company(self, email_text: str):\n",
    "        \"\"\"\n",
    "        Extracts the company name from the given email text using the Gemini API.\n",
    "        \"\"\"\n",
    "        question = (\"What is the name of the company? If not available, output None\",)\n",
    "        input_text = f\"question: {question} context: {email_text}\"\n",
    "        response = self.model.generate_content(input_text)\n",
    "        return response.text\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # Example email text\n",
    "    email_text = \"Subject: HRT Application Status - Pratham Agarwala. Body: Hi Pratham, We want to thank you very much for your interest in Hudson River Trading and the Algorithm Developer role. We have reviewed your candidacy for this position, along with other available opportunities, and have decided not to move forward with your application at this time. We ask that you refrain from applying again during this campus recruiting cycle. The new recruiting season will start in July/August 2024. We would encourage you to stay in touch with us, because as HRT grows, it's possible our hiring needs will change. We're always eager to network with smart candidates with an interest in our industry. Thanks again for your interest and time, and good luck with your job search. Regards, Hudson River Trading\"\n",
    "    \n",
    "    # Instantiate and utilize the JobTitleCompanyNameExtractor\n",
    "    job_title_extractor = JobTitleCompanyNameExtractor()\n",
    "    print(job_title_extractor.get_jobtitle(email_text))\n",
    "    print(job_title_extractor.get_company(email_text))\n",
    "    \n",
    "    # Instantiate and utilize the GeminiJobTitleCompanyNameExtractor\n",
    "    gemini_job_title_extractor = GeminiJobTitleCompanyNameExtractor()\n",
    "    print(gemini_job_title_extractor.get_jobtitle(email_text))\n",
    "    print(gemini_job_title_extractor.get_company(email_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gmail Email Retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import imaplib\n",
    "import email\n",
    "from datetime import datetime, timedelta\n",
    "from email.header import decode_header\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "\n",
    "class Gmail:\n",
    "    \"\"\"\n",
    "    A class to interact with Gmail via IMAP and retrieve email messages.\n",
    "    \"\"\"\n",
    "    def __init__(self, username, password):\n",
    "        \"\"\"\n",
    "        Initialize the Gmail object with the provided username and password.\n",
    "        \"\"\"\n",
    "        self.username = username\n",
    "        self.password = password\n",
    "\n",
    "    def authenticate(self):\n",
    "        \"\"\"\n",
    "        Authenticate with the Gmail server.\n",
    "        \"\"\"\n",
    "        # Authenticate with Gmail\n",
    "        gmail_host = \"imap.gmail.com\"\n",
    "        mail = imaplib.IMAP4_SSL(gmail_host)\n",
    "        mail.login(self.username, self.password)\n",
    "        self.mail = mail\n",
    "\n",
    "    def search_mail(self, category=\"INBOX\", search_criteria=\"ALL\"):\n",
    "        \"\"\"\n",
    "        Search for emails in the specified category using the given search criteria.\n",
    "        \"\"\"\n",
    "        self.mail.select(category)\n",
    "        _, email_ids = self.mail.search(None, search_criteria)\n",
    "        email_ids = email_ids[0].split()\n",
    "        return email_ids\n",
    "\n",
    "    def get_email_by_date(\n",
    "        self,\n",
    "        from_date,\n",
    "        category=\"INBOX\",\n",
    "        primary=True,\n",
    "        to_date=(datetime.now() + timedelta(days=1)).strftime(\"%d-%b-%Y\"),\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Get emails from a specific date range in the specified category.\n",
    "        \"\"\"\n",
    "        primary = ' X-GM-RAW \"Category:Primary\"' if primary else \"\"\n",
    "        search_criteria = f'(SINCE \"{from_date}\" BEFORE \"{to_date}\"){primary}'\n",
    "        email_ids = self.search_mail(category=category, search_criteria=search_criteria)\n",
    "        return email_ids\n",
    "\n",
    "    @staticmethod\n",
    "    def get_body(msg):\n",
    "        \"\"\"\n",
    "        Extract the body of an email message.\n",
    "        \"\"\"\n",
    "        body = \"\"\n",
    "        if msg.is_multipart():\n",
    "            for part in msg.walk():\n",
    "                content_disposition = str(part.get(\"Content-Disposition\"))\n",
    "                if (\n",
    "                    part.get_content_type() == \"text/plain\"\n",
    "                    and \"attachment\" not in content_disposition\n",
    "                ):\n",
    "                    body = part.get_payload(decode=True).decode(\n",
    "                        part.get_content_charset() or \"utf-8\"\n",
    "                    )\n",
    "                    if body is None:\n",
    "                        continue\n",
    "        else:\n",
    "            try:\n",
    "                body = msg.get_payload(decode=True).decode(\n",
    "                    msg.get_content_charset() or \"utf-8\"\n",
    "                )\n",
    "            except UnicodeDecodeError:\n",
    "                print(\"UnicodeDecodeError\")\n",
    "                return body\n",
    "        return body\n",
    "\n",
    "    def parse_emails(self, ids: list):\n",
    "        \"\"\"\n",
    "        Parse email messages and extract relevant information.\n",
    "        \"\"\"\n",
    "        data = []\n",
    "        for email_id in tqdm(ids):\n",
    "            _, email_data = self.mail.fetch(email_id, \"(RFC822)\")\n",
    "            raw_email = email_data[0][1]\n",
    "\n",
    "            # Parse the email\n",
    "            msg = email.message_from_bytes(raw_email)\n",
    "            # Extract email information (e.g., subject and sender)\n",
    "            subject, _ = decode_header(msg[\"Subject\"])[0]\n",
    "            sender, _ = decode_header(msg[\"From\"])[0]\n",
    "            data.append(\n",
    "                {\n",
    "                    \"id\": email_id.decode(\"utf-8\"),\n",
    "                    \"subject\": subject,\n",
    "                    \"sender\": sender,\n",
    "                    \"date\": msg[\"Date\"],\n",
    "                    \"body\": self.get_body(msg),\n",
    "                }\n",
    "            )\n",
    "\n",
    "        return data\n",
    "\n",
    "    def close(self):\n",
    "        \"\"\"\n",
    "        Close the connection to the Gmail server.\n",
    "        \"\"\"\n",
    "        self.mail.close()\n",
    "        self.mail.logout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Email Classification and Prediction Script\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/codespace/.local/lib/python3.10/site-packages/sklearn/base.py:376: InconsistentVersionWarning: Trying to unpickle estimator LogisticRegression from version 1.3.0 when using version 1.4.0. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calling Gemini API\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:05<00:00,  3.60it/s]\n"
     ]
    }
   ],
   "source": [
    "import importlib\n",
    "import sys\n",
    "sys.path.append(\"/workspaces/IcharusAI\")\n",
    "from jobTracker.gmail import Gmail\n",
    "from datetime import datetime\n",
    "from setfit import SetFitModel\n",
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "import pandas as pd\n",
    "from jobTracker.flan import JobTitleCompanyNameExtractor, GeminiJobTitleCompanyNameExtractor\n",
    "import gdown\n",
    "\n",
    "# Define the URL for downloading the email classifier model\n",
    "email_classifier_model = \"https://drive.google.com/drive/folders/1Jn_cjP1OjO5Ttj9-xs63o9cTPNhKwHOv?usp=drive_link\"\n",
    "# Get the current directory\n",
    "file_path = \"jobTracker\"\n",
    "# Create a directory for storing the model if it doesn't exist\n",
    "os.makedirs(file_path + \"/model\", exist_ok=True)\n",
    "# Define the path for the model\n",
    "model_path = file_path + \"/model/email_classifer\"\n",
    "\n",
    "\n",
    "def download_model():\n",
    "    \"\"\"\n",
    "    Download the email classifier model if it doesn't exist.\n",
    "    \"\"\"\n",
    "    if os.path.exists(model_path):\n",
    "        print(\"Model already exists\")\n",
    "        return\n",
    "    print(\"Downloading model\")\n",
    "\n",
    "    gdown.download_folder(\n",
    "        email_classifier_model,\n",
    "        quiet=True,\n",
    "        use_cookies=False,\n",
    "        output=model_path,\n",
    "    )\n",
    "    return\n",
    "\n",
    "\n",
    "class JobClassifier:\n",
    "    def __init__(self) -> None:\n",
    "        \"\"\"\n",
    "        Initialize the JobClassifier class.\n",
    "        \"\"\"\n",
    "        download_model()\n",
    "        # Load the email classifier model\n",
    "        self.model = SetFitModel.from_pretrained(model_path)\n",
    "        # Initialize the JobTitleCompanyNameExtractor\n",
    "        self.extractor = GeminiJobTitleCompanyNameExtractor()\n",
    "\n",
    "    def infer(self, sentence):\n",
    "        \"\"\"\n",
    "        Perform inference on a sentence using the email classifier model.\n",
    "        \"\"\"\n",
    "        predtext = [sentence]\n",
    "        predicted_class = self.model(predtext)\n",
    "        return str(predicted_class.numpy()[0])\n",
    "\n",
    "    def classify(self, email, preprocess=True):\n",
    "        \"\"\"\n",
    "        Classify an email using the email classifier model.\n",
    "        \"\"\"\n",
    "        if preprocess:\n",
    "            _, email = self.preprocess_email(email)\n",
    "\n",
    "        predicted_class = self.infer(email)\n",
    "        return _, predicted_class\n",
    "\n",
    "    def preprocess_email(self, email):\n",
    "        \"\"\"\n",
    "        Preprocess an email before classification.\n",
    "        \"\"\"\n",
    "        try:\n",
    "            subject = (email[\"subject\"]).decode(\"utf-8\")\n",
    "        except:\n",
    "            subject = email[\"subject\"]\n",
    "        html = str(BeautifulSoup(email[\"body\"], \"html.parser\").text)\n",
    "        string_list = [s.strip() for s in str(html).split()]\n",
    "        final_string = \" \".join(string_list)\n",
    "        final_string_without_subject = \" \".join(string_list)\n",
    "        final_string = \"Subject: \" + str(subject) + \". Body: \" + final_string\n",
    "        return final_string, final_string_without_subject\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # Initialize JobClassifier\n",
    "    jc = JobClassifier()\n",
    "    # Initialize Gmail client and please input your own gmail and password\n",
    "    gmail = Gmail(\n",
    "        username=\"agarwalpratham2001@gmail.com\", password=\"write your own password\"\n",
    "    )\n",
    "    # Authenticate with Gmail\n",
    "    gmail.authenticate()\n",
    "    # Specify the date to retrieve emails from\n",
    "    specified_date = datetime(2024, 2, 1)\n",
    "    formatted_date = specified_date.strftime(\"%d-%b-%Y\")\n",
    "    # Get email IDs from the specified date\n",
    "    ids = gmail.get_email_by_date(from_date=formatted_date)\n",
    "    # Parse email content\n",
    "    email_dict = gmail.parse_emails(ids[:20])\n",
    "    preds = []\n",
    "    # Classify each email\n",
    "    for i in email_dict:\n",
    "        out = jc.classify(i)\n",
    "        preds.append(out)\n",
    "    # Save predictions to CSV\n",
    "    pd.Series(preds).to_csv(\"preds.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Job Stage Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from jobTracker.gmail import Gmail\n",
    "from datetime import datetime\n",
    "from setfit import SetFitModel\n",
    "import os\n",
    "import pandas as pd\n",
    "import gdown\n",
    "\n",
    "# Define the URL for the stage classifier model\n",
    "stage_classifier_model = \"https://drive.google.com/drive/folders/1gq-9kA_MIa6KsULSjWC9lHyQXXgWFekT?usp=drive_link\"\n",
    "\n",
    "# Get the directory path of the current file\n",
    "file_path = \"jobTracker\"\n",
    "\n",
    "# Create a directory to store the model if it doesn't exist\n",
    "os.makedirs(file_path + \"/model\", exist_ok=True)\n",
    "\n",
    "# Define the path for storing the model\n",
    "model_path = file_path + \"/model/stage_classifer\"\n",
    "\n",
    "def download_model():\n",
    "    \"\"\"\n",
    "    Download the stage classifier model if it doesn't exist.\n",
    "    \"\"\"\n",
    "    if os.path.exists(model_path):\n",
    "        print(\"Model already exists\")\n",
    "        return\n",
    "    print(\"Downloading model\")\n",
    "\n",
    "    gdown.download_folder(\n",
    "        stage_classifier_model,\n",
    "        quiet=True,\n",
    "        use_cookies=False,\n",
    "        output=model_path,\n",
    "    )\n",
    "    return\n",
    "\n",
    "class JobStageClassifier:\n",
    "    def __init__(self) -> None:\n",
    "        \"\"\"\n",
    "        Initialize the JobStageClassifier object.\n",
    "        \"\"\"\n",
    "        download_model()\n",
    "        self.model = SetFitModel.from_pretrained(model_path)\n",
    "\n",
    "    def infer(self, sentence):\n",
    "        \"\"\"\n",
    "        Infer the class label of the input sentence.\n",
    "        \"\"\"\n",
    "        predtext = [sentence]\n",
    "        predicted_class = self.model(predtext)\n",
    "        return str(predicted_class.numpy()[0])\n",
    "\n",
    "    def classify(self, email):\n",
    "        \"\"\"\n",
    "        Classify the email using the trained model.\n",
    "        \"\"\"\n",
    "        predicted_class = self.infer(email)\n",
    "        return predicted_class\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting python-docx\n",
      "  Downloading python_docx-1.1.0-py3-none-any.whl.metadata (2.0 kB)\n",
      "Requirement already satisfied: lxml>=3.1.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from python-docx) (5.1.0)\n",
      "Requirement already satisfied: typing-extensions in /home/codespace/.local/lib/python3.10/site-packages (from python-docx) (4.9.0)\n",
      "Downloading python_docx-1.1.0-py3-none-any.whl (239 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m239.6/239.6 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: python-docx\n",
      "Successfully installed python-docx-1.1.0\n"
     ]
    }
   ],
   "source": [
    "!pip install python-docx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Job Recommendation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from docx import Document\n",
    "from openai import OpenAI\n",
    "import chromadb\n",
    "import logging\n",
    "from chromadb.utils import embedding_functions\n",
    "\n",
    "class Recommendation:\n",
    "    \"\"\"\n",
    "    Job recommendation engine and UI to display them.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, resume, openai_key=None, jobtitle=None):\n",
    "        \"\"\"\n",
    "        Initializes the Recommendation object.\n",
    "\n",
    "        Parameters:\n",
    "            resume (str): Path or I/O object of the resume.\n",
    "            openai_key (str): API key for OpenAI API.\n",
    "            jobtitle (str): Title of the job.\n",
    "        \"\"\"\n",
    "        self.resume = resume\n",
    "        self.jobtitle = jobtitle\n",
    "        self.openai_key = openai_key\n",
    "        self.client = OpenAI(api_key=self.openai_key)\n",
    "        self.file_path = os.path.join(os.getcwd(), \"data/jd_vectordb\")\n",
    "        self.chroma_client = chromadb.PersistentClient(path=self.file_path)\n",
    "        self.collection = self.chroma_client.get_or_create_collection(\n",
    "            name=\"mycareersfuture_jd\",\n",
    "            embedding_function=embedding_functions.DefaultEmbeddingFunction(),\n",
    "        )\n",
    "\n",
    "    def read_word_document(self):\n",
    "        \"\"\"\n",
    "        Reads the content of the resume Word document.\n",
    "\n",
    "        Returns:\n",
    "            str: Text content of the resume.\n",
    "        \"\"\"\n",
    "        doc = Document(self.resume)\n",
    "        text = \"\"\n",
    "        for paragraph in doc.paragraphs:\n",
    "            text += paragraph.text + \"\\n\"\n",
    "        return text\n",
    "\n",
    "    def get_generated_jd(self):\n",
    "        \"\"\"\n",
    "        Generates a job description based on the provided resume.\n",
    "\n",
    "        Returns:\n",
    "            str: Generated job description.\n",
    "        \"\"\"\n",
    "        text = self.read_word_document()\n",
    "        logging.info(\"Document Read\")\n",
    "        try:\n",
    "            response = self.client.chat.completions.create(\n",
    "                model=\"gpt-3.5-turbo\",\n",
    "                messages=[\n",
    "                    {\n",
    "                        \"role\": \"system\",\n",
    "                        \"content\": \"Based on the given resume above, create a suitable job posting for this resume. The job posting must include the job description, job responsibilities, and requirements such as qualifications and skills. Do not include the company name and location in this job posting.\",\n",
    "                    },\n",
    "                    {\n",
    "                        \"role\": \"user\",\n",
    "                        \"content\": f\"Resume: {text}\\n\\n---\\n\\nJob Description:\",\n",
    "                    },\n",
    "                ],\n",
    "                temperature=0.3,\n",
    "                top_p=1,\n",
    "                frequency_penalty=0,\n",
    "                presence_penalty=0,\n",
    "            )\n",
    "            logging.info(\"CV Generated\")\n",
    "            return response.choices[0].message.content.strip()\n",
    "        except Exception as e:\n",
    "            logging.warning(e)\n",
    "            print(e)\n",
    "            return \"\"\n",
    "\n",
    "    def search_jd(self, jd, k=20):\n",
    "        \"\"\"\n",
    "        Searches for job descriptions similar to the provided text.\n",
    "\n",
    "        Parameters:\n",
    "            jd (str): Job description text to search for.\n",
    "            k (int): Number of results to return.\n",
    "\n",
    "        Returns:\n",
    "            dict: Query results containing documents, distances, and metadatas.\n",
    "        \"\"\"\n",
    "        results = self.collection.query(\n",
    "            query_texts=[jd],\n",
    "            n_results=k,\n",
    "            include=[\"documents\", \"distances\", \"metadatas\"],\n",
    "        )\n",
    "        return results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Job Tracker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import os\n",
    "from .gmail import Gmail\n",
    "from .job_classifier import JobClassifier\n",
    "from .job_stage import JobStageClassifier\n",
    "\n",
    "class TrackFile:\n",
    "    def __init__(self):\n",
    "        # Initialize the TrackFile class with the file path\n",
    "        self.file_path = os.path.join(\n",
    "            os.getcwd(),\n",
    "            \"track.csv\",\n",
    "        )\n",
    "        self.initiate_file()\n",
    "\n",
    "    def initiate_file(self):\n",
    "        # Check if the track file exists, if not, create it with required columns\n",
    "        if os.path.exists(self.file_path):\n",
    "            return\n",
    "        else:\n",
    "            df = pd.DataFrame(columns=[\"date\"])\n",
    "            df.to_csv(self.file_path, index=False)\n",
    "\n",
    "    def read_file(self):\n",
    "        # Read the track file if it exists, otherwise return an empty DataFrame\n",
    "        if os.path.exists(self.file_path):\n",
    "            return pd.read_csv(self.file_path)\n",
    "        else:\n",
    "            return pd.DataFrame()\n",
    "\n",
    "    def update_file(self, df: pd.DataFrame):\n",
    "        # Update the track file with new data\n",
    "        track_data = self.read_file()\n",
    "        track_data = pd.concat([df, track_data], axis=0)\n",
    "        track_data.sort_values(by=\"date\", inplace=True, ascending=False)\n",
    "        track_data.to_csv(self.file_path, index=False)\n",
    "        return track_data\n",
    "\n",
    "    def delete_file(self, index):\n",
    "        # Delete a record from the track file based on the index\n",
    "        track_data = self.read_file()\n",
    "        track_data.drop(index, inplace=True)\n",
    "        track_data.to_csv(self.file_path, index=False)\n",
    "        return track_data\n",
    "\n",
    "    def add_record(\n",
    "        self,\n",
    "        title,\n",
    "        company,\n",
    "        stage,\n",
    "        date=str(pd.to_datetime(\"today\").date()),\n",
    "        rejected=0,\n",
    "        logo=\"https://storage.googleapis.com/simplify-imgs/company/default/logo.png\",\n",
    "        location=\"SG\",\n",
    "        text=\" \",\n",
    "    ):\n",
    "        # Add a new record to the track file\n",
    "        track_data = self.read_file()\n",
    "        new_record = pd.DataFrame(\n",
    "            {\n",
    "                \"text\": [text],\n",
    "                \"job\": 1,\n",
    "                \"title\": [title],\n",
    "                \"company\": [company],\n",
    "                \"stage\": [stage],\n",
    "                \"date\": [date],\n",
    "                \"rejected\": [rejected],\n",
    "                \"logo\": [logo],\n",
    "                \"location\": [location],\n",
    "            }\n",
    "        )\n",
    "        track_data = pd.concat([new_record, track_data], axis=0)\n",
    "        track_data.to_csv(self.file_path, index=False)\n",
    "        return track_data\n",
    "\n",
    "    def get_last_update_date(self):\n",
    "        # Get the last update date from the track file\n",
    "        if os.path.exists(self.file_path):\n",
    "            if not pd.isna(pd.read_csv(self.file_path)[\"date\"].max()):\n",
    "                date = (\n",
    "                    pd.to_datetime(pd.read_csv(self.file_path)[\"date\"].max())\n",
    "                    + pd.DateOffset(days=1)\n",
    "                ).date()\n",
    "                formatted_date = date.strftime(\"%d-%b-%Y\")\n",
    "                return formatted_date\n",
    "            else:\n",
    "                date = (pd.to_datetime(\"today\") - pd.DateOffset(days=1)).date()\n",
    "                formatted_date = date.strftime(\"%d-%b-%Y\")\n",
    "                return formatted_date\n",
    "        else:\n",
    "            # If the file doesn't exist, return a default date\n",
    "            date = (pd.to_datetime(\"today\") - pd.DateOffset(days=1)).date()\n",
    "            formatted_date = date.strftime(\"%d-%b-%Y\")\n",
    "            return formatted_date\n",
    "\n",
    "def get_logo_trustpilot(company_name):\n",
    "    # Function to get the logo of a company from Trustpilot\n",
    "    url = \"https://www.trustpilot.com/api/consumersitesearch-api/businessunits/search\"\n",
    "    params = {\n",
    "        \"country\": \"US\",\n",
    "        \"page\": 1,\n",
    "        \"pageSize\": 1,\n",
    "        \"query\": company_name,\n",
    "    }\n",
    "    try:\n",
    "        response = requests.get(\n",
    "            url, params=params, headers={\"user-agent\": \"Mozilla/5.0\"}\n",
    "        )\n",
    "        if pd.DataFrame(response.json().get(\"businessUnits\", [])).shape[0] == 1:\n",
    "            temp = pd.DataFrame(response.json().get(\"businessUnits\", []))\n",
    "            if pd.isna(temp[\"logoUrl\"].iloc[0]):\n",
    "                print(\"No Logo Found\")\n",
    "                return \"https://storage.googleapis.com/simplify-imgs/company/default/logo.png\"\n",
    "            else:\n",
    "                return f'https://consumersiteimages.trustpilot.net/business-units/{temp[\"businessUnitId\"].iloc[0]}-198x149-1x.jpg'\n",
    "    except Exception as e:\n",
    "        pass\n",
    "    return \"https://storage.googleapis.com/simplify-imgs/company/default/logo.png\"\n",
    "\n",
    "def update_track_data(gmail_username, gmail_api_key):\n",
    "    # Function to update track data based on new emails\n",
    "    track = TrackFile()\n",
    "    latest_date = track.get_last_update_date()\n",
    "    gmail = Gmail(username=gmail_username, password=gmail_api_key)\n",
    "    gmail.authenticate()\n",
    "    ids = gmail.get_email_by_date(from_date=latest_date)\n",
    "    print(\"Fetching emails\")\n",
    "    email_dict = gmail.parse_emails(ids)\n",
    "    print(\"Identifying Job Emails\")\n",
    "    classifier = JobClassifier()\n",
    "    preds = []\n",
    "    for email in email_dict:\n",
    "        out = classifier.classify(email)\n",
    "        preds.append(out)\n",
    "    dates = [pd.to_datetime(i[\"date\"]).strftime(\"%Y-%m-%d\") for i in email_dict]\n",
    "    jobs = pd.DataFrame(preds, columns=[\"text\", \"job\"])\n",
    "    jobs[\"date\"] = dates\n",
    "    jobs = jobs[jobs[\"job\"] != \"0\"]\n",
    "    jobs.reset_index(inplace=True, drop=True)\n",
    "    jobs[\"title\"] = None\n",
    "    jobs[\"company\"] = None\n",
    "    jobs[\"rejected\"] = 0\n",
    "    jobs[\"logo\"] = (\n",
    "        \"https://storage.googleapis.com/simplify-imgs/company/default/logo.png\"  # deafult placeholder logo\n",
    "    )\n",
    "    jobs[\"location\"] = \"Singapore\"\n",
    "    print(\"Identifying Company and Job title\")\n",
    "    for index, i in enumerate(jobs[\"text\"]):\n",
    "        jobs.loc[index, \"company\"] = classifier.extractor.get_company(i)\n",
    "        jobs.loc[index, \"title\"] = classifier.extractor.get_jobtitle(i)\n",
    "        jobs.loc[index, \"logo\"] = get_logo_trustpilot(jobs.loc[index, \"company\"])\n",
    "    if jobs.shape[0] == 0:\n",
    "        track_data = track.read_file()\n",
    "        track_data[\"title\"] = track_data[\"title\"].fillna(\"\")\n",
    "        track_data[\"company\"] = track_data[\"company\"].fillna(\"\")\n",
    "        return track_data.to_dict(orient=\"records\")\n",
    "    print(\"Stage Classfiying\")\n",
    "    stage_classifier = JobStageClassifier()\n",
    "    stages = []\n",
    "    for index, i in enumerate(jobs[\"text\"]):\n",
    "        out = stage_classifier.classify(i)\n",
    "        if int(out) == 4:  # rejected\n",
    "            jobs.loc[index, \"rejected\"] = 1\n",
    "        stages.append(int(out))\n",
    "    jobs[\"stage\"] = stages\n",
    "    print(\"Updating Track Data\")\n",
    "    track_data = track.update_file(jobs)\n",
    "    track_data[\"title\"] = track_data[\"title\"].fillna(\"\")\n",
    "    track_data[\"company\"] = track_data[\"company\"].fillna(\"\")\n",
    "    return track_data.to_dict(orient=\"records\")\n",
    "\n",
    "def get_track_data():\n",
    "    # Function to get track data\n",
    "    track = TrackFile().read_file()\n",
    "    track[\"title\"] = track[\"title\"].fillna(\"\")\n",
    "    track[\"company\"] = track[\"company\"].fillna(\"\")\n",
    "    return track.to_dict(orient=\"records\")\n",
    "\n",
    "def delete_track_data(index):\n",
    "    # Function to delete track data based on index\n",
    "    track = TrackFile()\n",
    "    track_data = track.delete_file(index)\n",
    "    track_data[\"title\"] = track_data[\"title\"].fillna(\"\")\n",
    "    track_data[\"company\"] = track_data[\"company\"].fillna(\"\")\n",
    "    return track_data.to_dict(orient=\"records\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python Function for Generating Explanations using LLMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "import boto3\n",
    "import logging\n",
    "import os\n",
    "import json\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "# Load environment variables from .env file\n",
    "load_dotenv()\n",
    "\n",
    "def explain_openai_gpt(jd_text, res, rec=True, ai_service='llama', api_key=None):\n",
    "    \"\"\"\n",
    "    Generate explanations using OpenAI's GPT-3.5 or LLAMA model.\n",
    "\n",
    "    Args:\n",
    "        jd_text (str): The job description text.\n",
    "        res (str): The resume text.\n",
    "        rec (bool, optional): Indicates whether the job was recommended or not. Defaults to True.\n",
    "        ai_service (str, optional): Specifies the AI service to use ('openai' or 'llama'). Defaults to 'llama'.\n",
    "        api_key (str, optional): OpenAI API key. Required if ai_service is 'openai'. Defaults to None.\n",
    "\n",
    "    Returns:\n",
    "        str: The generated explanation.\n",
    "\n",
    "    Raises:\n",
    "        Exception: If an error occurs during model invocation.\n",
    "\n",
    "    Note:\n",
    "        - If ai_service is 'openai', the function uses OpenAI's GPT-3.5 to generate the explanation.\n",
    "        - If ai_service is 'llama', the function uses a custom LLAMA model deployed on AWS Bedrock Runtime.\n",
    "    \"\"\"\n",
    "    # Define the content based on whether the job description was recommended or not\n",
    "    if rec:\n",
    "        content =  \"You are an Explainable Job Recommendation system. The resume submitted by the user is:\\n {res}\\n Please give reasons why the user was recommended the role with the following job description and job requirements:{jd}\\n Do not use more than 3 lines. Use passive voice for the response\".format(res=res,jd=jd_text)\n",
    "    else:\n",
    "        content = \"The resume submitted by the user is:\\n{res}\\nAs a job recommendation system, give reasons why the user should not apply for the role with the following job description and job requirements:{jd}\".format(res=res, jd=jd_text)\n",
    "\n",
    "    try:\n",
    "        if ai_service == 'openai':\n",
    "            # Use OpenAI's GPT-3.5 to generate explanation\n",
    "            client = OpenAI(api_key=api_key)\n",
    "            response = client.chat.completions.create(\n",
    "                model=\"gpt-3.5-turbo\",\n",
    "                messages=[\n",
    "                    {\n",
    "                        \"role\": \"user\",\n",
    "                        \"content\": content,\n",
    "                    }\n",
    "                ],\n",
    "                temperature=0.3,\n",
    "                top_p=1,\n",
    "                frequency_penalty=0,\n",
    "                presence_penalty=0,\n",
    "            )\n",
    "            result = response.choices[0].message.content.strip()\n",
    "        elif ai_service == 'llama':\n",
    "            # Use LLAMA model deployed on AWS Bedrock Runtime to generate explanation\n",
    "            prompt = \"You are an Explainable Job Recommendation system. The resume submitted by the user is:\\n {res}\\n Please give reasons why the user was recommended the role with the following job description and job requirements:{jd}\\n Do not use more than 3 lines. Use passive voice for the response\".format(res=res,jd=jd_text)\n",
    "\n",
    "            body = {\n",
    "                \"prompt\": prompt,\n",
    "                \"temperature\": 0.3,\n",
    "                \"top_p\": 1,\n",
    "            }\n",
    "            # Invoke LLAMA model\n",
    "            bedrock_runtime_client = boto3.client('bedrock-runtime',region_name='us-east-1', aws_access_key_id=os.getenv(\"AWS_ACCESS_KEY_ID\"), aws_secret_access_key=os.getenv(\"AWS_SECRET_ACCESS_KEY\"))\n",
    "            response = bedrock_runtime_client.invoke_model(\n",
    "                modelId=\"meta.llama2-13b-chat-v1\", body=json.dumps(body)\n",
    "            )\n",
    "            response_body = json.loads(response[\"body\"].read())\n",
    "            completion = response_body[\"generation\"]\n",
    "            logging.info(\"Explanation generated with LLAMA\")\n",
    "            return completion\n",
    "        else:\n",
    "            logging.warning(\"Invalid AI service specified.\")\n",
    "            return \"\"\n",
    "        \n",
    "        logging.info(\"Explanation for Recommendation generated\")\n",
    "        return result\n",
    "    except Exception as e:\n",
    "        logging.warning(e)\n",
    "        raise Exception(\"Error generating explanation\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Converting the scripts to APIs to integrate with the frontend\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastapi import FastAPI, File, Form, UploadFile\n",
    "from fastapi.middleware.cors import CORSMiddleware\n",
    "from typing import Optional\n",
    "import uvicorn\n",
    "import os\n",
    "\n",
    "# Importing necessary classes and functions from project modules\n",
    "from jobTracker.recommendation import Recommendation\n",
    "from jobTracker.track import (\n",
    "    update_track_data,\n",
    "    get_track_data,\n",
    "    delete_track_data,\n",
    "    TrackFile,\n",
    ")\n",
    "from jobTracker.explain import explain_openai_gpt\n",
    "\n",
    "# Initialize FastAPI instance\n",
    "app = FastAPI()\n",
    "\n",
    "# CORS middleware configuration to allow requests from web applications hosted on different origins\n",
    "app.add_middleware(\n",
    "    CORSMiddleware,\n",
    "    allow_origins=[\"*\"],  # This allows all origins\n",
    "    allow_credentials=True,\n",
    "    allow_methods=[\"*\"],  # Allows all methods\n",
    "    allow_headers=[\"*\"],  # Allows all headers\n",
    ")\n",
    "\n",
    "# Endpoint for recommending jobs based on resume\n",
    "@app.post(\"/recommend-jobs/\")\n",
    "async def recommend_jobs(\n",
    "    resume: UploadFile = File(...),\n",
    "    ai_service: str = Form(...),  # AI service to use for recommendation\n",
    "    api_key: Optional[str] = Form(None),  # Optional API key\n",
    "    jobtitle: Optional[str] = Form(None),\n",
    "):\n",
    "    # Save temporary file to disk to be read by Document\n",
    "    temp_file_path = f\"temp_{resume.filename}\"\n",
    "    with open(temp_file_path, \"wb\") as buffer:\n",
    "        buffer.write(await resume.read())\n",
    "\n",
    "    # Initialize the Recommendation class with provided parameters\n",
    "    recommendation_engine = Recommendation(\n",
    "        resume=temp_file_path, ai_service=ai_service, api_key=api_key, jobtitle=jobtitle\n",
    "    )\n",
    "\n",
    "    # Generate job description based on the resume\n",
    "    generated_jd = recommendation_engine.get_generated_jd()\n",
    "\n",
    "    # Clean up the temporary file\n",
    "    os.remove(temp_file_path)\n",
    "\n",
    "    if not generated_jd:\n",
    "        return {\"error\": \"Failed to generate job description from the resume.\"}\n",
    "\n",
    "    # Search for job recommendations based on the generated job description\n",
    "    job_recommendations = recommendation_engine.search_jd(generated_jd, k=20)\n",
    "\n",
    "    return {\"generated_jd\": generated_jd, \"job_recommendations\": job_recommendations}\n",
    "\n",
    "# Endpoint for searching jobs based on job description\n",
    "@app.post(\"/search-jobs/\")\n",
    "def search_jobs(\n",
    "    jd: str = Form(...),\n",
    "    ai_service: str = Form(\"openai\"),  # Default to 'openai' for backward compatibility\n",
    "    api_key: Optional[str] = Form(None),  # Optional API key\n",
    "):\n",
    "    # Initialize with minimal params as this method only uses the search functionality\n",
    "    recommendation_engine = Recommendation(\n",
    "        resume=\"./\", ai_service=ai_service, api_key=api_key\n",
    "    )\n",
    "\n",
    "    # Search for job recommendations based on the provided job description\n",
    "    job_recommendations = recommendation_engine.search_jd(jd, k=10)\n",
    "\n",
    "    return {\"job_recommendations\": job_recommendations}\n",
    "\n",
    "# Endpoint for explaining job recommendation based on job description and resume\n",
    "@app.post(\"/explain-record/\")\n",
    "def explain_record(\n",
    "    jd: str = Form(...),\n",
    "    resume_text: str = Form(...),\n",
    "    ai_service: str = Form(\"llama\"),  # Default to 'llama'\n",
    "    api_key: Optional[str] = Form(None),  # Optional API key\n",
    "):\n",
    "    # Explain the job recommendation using the specified AI service\n",
    "    explain_data = explain_openai_gpt(\n",
    "        jd_text=jd, res=resume_text, ai_service=ai_service, api_key=api_key\n",
    "    )\n",
    "    return explain_data\n",
    "\n",
    "# Endpoint for updating track records based on emails\n",
    "@app.post(\"/update-records/\")\n",
    "async def update_track(\n",
    "    gmail_username: str = Form(...),\n",
    "    gmail_password: str = Form(...),\n",
    "):\n",
    "    # Update track data based on new emails\n",
    "    track_data = update_track_data(\n",
    "        gmail_username=gmail_username,\n",
    "        gmail_api_key=gmail_password,\n",
    "    )\n",
    "    return track_data\n",
    "\n",
    "# Endpoint for getting track records\n",
    "@app.get(\"/get-records/\")\n",
    "async def get_track():\n",
    "    # Get track data\n",
    "    track_data = get_track_data()\n",
    "    return track_data\n",
    "\n",
    "# Endpoint for deleting a track record by index\n",
    "@app.post(\"/delete-record/\")\n",
    "def delete_track(index: int):\n",
    "    # Delete a track record based on the provided index\n",
    "    track_data = delete_track_data(index)\n",
    "    return track_data\n",
    "\n",
    "# Endpoint for adding a new track record\n",
    "@app.post(\"/add-record/\")\n",
    "def add_track(\n",
    "    title: str,\n",
    "    company: str,\n",
    "    stage: str,\n",
    "    location: str,\n",
    "):\n",
    "    # Add a new track record with provided details\n",
    "    stage_map = {\"Applied\": 0, \"OA\": 1, \"Interview\": 2, \"Offer\": 3, \"Rejection\": 4}\n",
    "    stage = stage_map[stage]\n",
    "    rejected = 1 if stage == 4 else 0\n",
    "    track = TrackFile()\n",
    "    track = track.add_record(\n",
    "        title=title, company=company, stage=stage, rejected=rejected, location=location\n",
    "    )\n",
    "    return track.to_dict(orient=\"records\")\n",
    "\n",
    "# Run the FastAPI app\n",
    "if __name__ == \"__main__\":\n",
    "    uvicorn.run(app, host=\"0.0.0.0\", port=8000)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Training for Job Email Classification\n",
    "## Note: All the Training Data is trained using Generative AI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# Load dataset from CSV file\n",
    "dataset = load_dataset(\"csv\", data_files=\"./train_data/Job_Email_Data.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers.losses import CosineSimilarityLoss\n",
    "\n",
    "from setfit import SetFitModel, SetFitTrainer, sample_dataset\n",
    "\n",
    "# Load a SetFit model from Hub\n",
    "model = SetFitModel.from_pretrained(\"sentence-transformers/paraphrase-mpnet-base-v2\")\n",
    "# Create trainer\n",
    "trainer = SetFitTrainer(\n",
    "    model=model,\n",
    "    train_dataset=dataset['train'],\n",
    "    loss_class=CosineSimilarityLoss,\n",
    "    metric=\"accuracy\",\n",
    "    batch_size=4,\n",
    "    num_iterations=20, # The number of text pairs to generate for contrastive learning\n",
    "    num_epochs=1, # The number of epochs to use for contrastive learning\n",
    "    column_mapping={\"text\": \"text\", \"label\": \"label\"} # Map dataset columns to text/label expected by trainer\n",
    ")\n",
    "\n",
    "# Train and evaluate\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.model._save_pretrained(\"job_email_classification_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Training for Job Stage Classification\n",
    "## Note: All the Training Data is trained using Generative AI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "dataset = load_dataset(\"csv\", data_files=\"./train_data/email_classification_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset['train'].to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers.losses import CosineSimilarityLoss\n",
    "\n",
    "from setfit import SetFitModel, SetFitTrainer, sample_dataset\n",
    "\n",
    "\n",
    "\n",
    "# Load a SetFit model from Hub\n",
    "model = SetFitModel.from_pretrained(\"sentence-transformers/paraphrase-mpnet-base-v2\")\n",
    "\n",
    "# Create trainer\n",
    "trainer = SetFitTrainer(\n",
    "    model=model,\n",
    "    train_dataset=dataset['train'],\n",
    "    loss_class=CosineSimilarityLoss,\n",
    "    metric=\"accuracy\",\n",
    "    batch_size=4,\n",
    "    num_iterations=20, # The number of text pairs to generate for contrastive learning\n",
    "    num_epochs=1, # The number of epochs to use for contrastive learning\n",
    "    column_mapping={\"sentence\": \"text\", \"label\": \"label\"} # Map dataset columns to text/label expected by trainer\n",
    ")\n",
    "\n",
    "# Train and evaluate\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.model._save_pretrained(\"job_stage_classification_model\") #Both the models are saved in google drive and used for the above functions"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}