sentiment_analysis_context.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0125a4c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytube import YouTube, Channel\n",
    "import whisper\n",
    "import os\n",
    "os.environ[\"IMAGEIO_FFMPEG_EXE\"] = \"/usr/bin/ffmpeg\"\n",
    "from moviepy.editor import AudioFileClip\n",
    "from googleapiclient.discovery import build\n",
    "from datetime import datetime, timedelta\n",
    "from dotenv import load_dotenv\n",
    "from transformers import AutoTokenizer \n",
    "from transformers import AutoModelForSequenceClassification\n",
    "from transformers import pipeline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c2964823",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytube import YouTube, Channel\n",
    "import whisper\n",
    "import os\n",
    "os.environ[\"IMAGEIO_FFMPEG_EXE\"] = \"/usr/bin/ffmpeg\"\n",
    "from moviepy.editor import AudioFileClip\n",
    "from googleapiclient.discovery import build\n",
    "from datetime import datetime, timedelta\n",
    "from dotenv import load_dotenv\n",
    "from transformers import AutoTokenizer \n",
    "from transformers import AutoModelForSequenceClassification\n",
    "from transformers import pipeline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f1bbf5de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the above libraries are for the other parts of the project as well. \n",
    "# This is how we get our function called final_df\n",
    "\n",
    "import pandas as pd\n",
    "import openai\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline\n",
    "\n",
    "# Initialize OpenAI API\n",
    "openai.api_key = '' # use your api_key \n",
    "\n",
    "#  selected Huggingface models with their corresponding paths. We pick FinancialBert and ProsusAI/finbert\n",
    "models = {\n",
    "    \"FinancialBERT\": \"yiyanghkust/finbert-tone\",\n",
    "    #\"bert-base-uncased\": \"bert-base-uncased\", we take this model out since it's a binary statement as opposed to\n",
    "    # multiclass\n",
    "    \"ProsusAI/finbert\": \"ProsusAI/finbert\"\n",
    "}\n",
    "\n",
    "def analyze_sentiment_gpt35turbo(text):\n",
    "    \"\"\"\n",
    "    Analyze sentiment of a given text using GPT-3.5-turbo.\n",
    "\n",
    "    Parameters:\n",
    "    - text (str): The input text to analyze.\n",
    "\n",
    "    Returns:\n",
    "    - sentiment (str): The sentiment label of the text.\n",
    "    \"\"\"\n",
    "    if not text:\n",
    "        return \"We need text\"  \n",
    "\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": \"You are a sentiment analysis assistant.\"},\n",
    "        {\"role\": \"user\", \"content\": f\"Please analyze the sentiment of the following text and classify it as Positive, Negative, or Neutral:\\n\\n{text}\"}\n",
    "    ]\n",
    "\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=\"gpt-3.5-turbo\",\n",
    "        messages=messages\n",
    "    )\n",
    "\n",
    "    sentiment = response.choices[0].message['content'].strip()\n",
    "    return sentiment\n",
    "\n",
    "def analyze_sentiment_huggingface(text, model_path):\n",
    "    \"\"\"\n",
    "    Analyze sentiment of a given text using a specified pre-trained model from Hugging Face.\n",
    "\n",
    "    Parameters:\n",
    "    - text (str): The input text to analyze.\n",
    "    - model_path (str): The path to the pre-trained model.\n",
    "\n",
    "    Returns:\n",
    "    - label (str): The sentiment label of the text.\n",
    "    \"\"\"\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(model_path)\n",
    "    sentiment_analyzer = pipeline(\"sentiment-analysis\", model=model, tokenizer=tokenizer)\n",
    "    \n",
    "    result = sentiment_analyzer(text)[0]\n",
    "    return result['label']\n",
    "\n",
    "def analyze_csv_sentiments(input_csv, output_csv):\n",
    "    \"\"\"\n",
    "    Analyze sentiments of text data in a CSV file using multiple pre-trained models and\n",
    "    store the results in separate columns for each model.\n",
    "\n",
    "    Parameters:\n",
    "    - input_csv (str): Path to the input CSV file containing text data.\n",
    "    - output_csv (str): Path to save the output CSV file with sentiment analysis results.\n",
    "\n",
    "    Returns:\n",
    "    - df (DataFrame): The updated DataFrame with sentiment analysis results.\n",
    "    \"\"\"\n",
    "    # Load the CSV file into a DataFrame\n",
    "    df = pd.read_csv(input_csv)\n",
    "\n",
    "    # Ensure there is a column to analyze\n",
    "    if 'text' not in df.columns:\n",
    "        raise ValueError(\"The CSV file must contain a 'text' column for analysis.\")\n",
    "\n",
    "    # Analyze sentiment for each row using each model and add results to new columns\n",
    "    df['gpt-3.5-turbo_sentiment'] = df['text'].apply(analyze_sentiment_gpt35turbo)\n",
    "    \n",
    "    for model_name, model_path in models.items():\n",
    "        df[f'{model_name}_sentiment'] = df['text'].apply(lambda text: analyze_sentiment_huggingface(text, model_path))\n",
    "\n",
    "    # Save the updated DataFrame to a new CSV file\n",
    "    df.to_csv(output_csv, index=False)\n",
    "    \n",
    "    # Return the updated DataFrame\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d42abd1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This will take a while and this is how we generate our file with the above function\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "input_csv = \"test_file.csv\"  # Path to your input CSV file\n",
    "output_csv = \"output_with_sentiments1.csv\"  # Path to save the output CSV file with sentiments\n",
    "result_df = analyze_csv_sentiments(input_csv, output_csv)\n",
    "print(result_df.head())  # Print the first few rows of the updated DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ef014ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is for simplifying the answer of gpt 3.5 turboso we only get the positive, neutral and negative from the \n",
    "# entire sentiment sentence. \n",
    "\n",
    "def find_sentiment_words(input_csv, output_csv, column_name):\n",
    "    df = pd.read_csv(input_csv)\n",
    "    \n",
    "    if column_name not in df.columns:\n",
    "        raise ValueError(f\"The column {column_name} does not in the CSV file.\")\n",
    "        \n",
    "    sentiment_words = [\"Positive\",\"Neutral\",\"Negative\"]\n",
    "    \n",
    "    df['chatgpt_sentiment'] = None\n",
    "    \n",
    "    for index,row in df.iterrows():\n",
    "        text = row[column_name]\n",
    "        \n",
    "        for word in sentiment_words:\n",
    "            if pd.notnull(text) and word.lower() in text.lower():\n",
    "                df.at[index, 'chatgpt_sentiment'] = word\n",
    "                break\n",
    "    \n",
    "    df.to_csv(output_csv, index = False)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "540c51ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_csv = \"output_with_sentiments1.csv\"\n",
    "output_csv = \"updated_sentiments.csv\"\n",
    "column_name = \"gpt-3.5-turbo_sentiment\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b1997524",
   "metadata": {},
   "outputs": [],
   "source": [
    "# updated version of our file with chatgpt_sentiment column added \n",
    "final_df = find_sentiment_words(input_csv,output_csv, column_name)\n",
    "final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "14b557a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Again just some processing on the columns like doing upper case so all match. \n",
    "\n",
    "        \n",
    "replacements = {\"positive\":\"Positive\", \n",
    "                \"neutral\":\"Neutral\",\n",
    "               \"negative\":\"Negative\"\n",
    "    }\n",
    "for old_word, new_word in replacements.items(): \n",
    "        final_df['FinancialBERT_sentiment'].replace(old_word, new_word, inplace=True)\n",
    "        \n",
    "        # print(final_df)        \n",
    "replacements = {\"positive\":\"Positive\", \n",
    "                \"neutral\":\"Neutral\",\n",
    "               \"negative\":\"Negative\"\n",
    "    }\n",
    "for old_word, new_word in replacements.items(): \n",
    "        final_df['ProsusAI/finbert_sentiment'].replace(old_word, new_word, inplace=True) \n",
    "        \n",
    "       # print(final_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61657917",
   "metadata": {},
   "outputs": [],
   "source": [
    "# (4846, 6) # kaggle row number is checked \n",
    "final_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a34c3492",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the file that we have the sentiment model results in, call it \"test_models\"\n",
    "\n",
    "final_df.to_csv('test_models.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "623037fb",
   "metadata": {},
   "source": [
    "### We now need to check these sentiment labels against the Kaggle dataframe: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d4bb736",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Kaggle datafram is stored in test: \n",
    "\n",
    "test_file = \"/Users/nazanin.komeilizadeh/Desktop/test.csv\"\n",
    "test = pd.read_csv(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6efd5e7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "replacements = {\"positive\":\"Positive\", \n",
    "                \"neutral\":\"Neutral\",\n",
    "               \"negative\":\"Negative\"\n",
    "    }\n",
    "for old_word, new_word in replacements.items(): \n",
    "        test['kaggle_label'].replace(old_word, new_word, inplace=True)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6265c2dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "test.kaggle_label.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42118bd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = test.drop(['all_data'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2965cc63",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save into a csv file \n",
    "test.to_csv('kaggle.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cc0eca9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23d0ea2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# here, we want to check the sentiment analysis of each model against the Kaggle label \n",
    "\n",
    "# columns to compare\n",
    "columns_final_df = ['FinancialBERT_sentiment','ProsusAI/finbert_sentiment', 'chatgpt_sentiment']\n",
    "column_test = 'kaggle_label'\n",
    "\n",
    "# making sure DataFrames have the same length for row-wise comparison\n",
    "if len(final_df) != len(test):\n",
    "    raise ValueError(\"DataFrames must have the same number of rows for comparison.\")\n",
    "\n",
    "# Create new columns in final_df to store the match results\n",
    "for col in columns_final_df:\n",
    "    final_df[f'{col}_match'] = final_df[col] == test[column_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b19affe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('match.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f2fc240",
   "metadata": {},
   "outputs": [],
   "source": [
    "# slicing the datafram to only the last 3 comparision columns\n",
    "\n",
    "accuracy = final_df[final_df.columns[-3:]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36a352c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy.columns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "392170f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy['FinancialBERT_sentiment_match'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5572a243",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy['ProsusAI/finbert_sentiment_match'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "830fe32d",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy['chatgpt_sentiment_match'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "720dc4b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "accu_finbert = len(accuracy[accuracy['FinancialBERT_sentiment_match']==1])/len(accuracy)\n",
    "# 0.79"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "44c8d2f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# turns out it's the best model for accuracy\n",
    "accu_prosus = len(accuracy[accuracy['ProsusAI/finbert_sentiment_match']==True])/len(accuracy)\n",
    "# 0.8895"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "62137c6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "accu_chatgpt = len(accuracy[accuracy['chatgpt_sentiment_match']==True])/len(accuracy)\n",
    "# 0.712"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2c0d774",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "ed867fad",
   "metadata": {},
   "source": [
    "# END "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}