bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from transformers import BertTokenizer, BertModel\n",
    "from scipy.stats import pearsonr\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from tqdm import tqdm\n",
    "from utils import get_sorted_tweets, get_target_words\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Corpus-level LSC detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = 'cuda'\n",
    "else:\n",
    "    device = 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load BERT model and tokenizer\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertModel.from_pretrained(model_name).to(device)\n",
    "tweets = get_sorted_tweets()\n",
    "target_words = get_target_words()\n",
    "tsv_file_path = 'data/annotator.tsv'\n",
    "# Read the TSV file into a pandas DataFrame\n",
    "df = pd.read_csv(tsv_file_path, sep=' ', header=None).to_numpy()\n",
    "annotator = {item[0]: item[1] for item in df}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_vector_from_context(word, text):\n",
    "    tok_w = tokenizer(word, return_tensors='pt', add_special_tokens=False)\n",
    "    tok = int(tok_w['input_ids'].flatten()[0])\n",
    "    len_tok = len(tok_w['input_ids'].flatten())\n",
    "    tok_t = tokenizer(text, return_tensors='pt', padding='max_length')\n",
    "    ids = tok_t['input_ids'].flatten().tolist()\n",
    "    if tok in ids:\n",
    "        idx = ids.index(tok)\n",
    "    else:\n",
    "        raise ValueError(f'{tok} from {tok_w} not in list {ids}. \\n text: {text} word {word} \\n tokenizer decode: {tokenizer.decode(ids)}')\n",
    "    for item in tok_t:\n",
    "        tok_t[item] = tok_t[item].to(device)\n",
    "    vec = model(**tok_t)['last_hidden_state'].squeeze(0)[idx:idx+len_tok].cpu().detach().numpy()\n",
    "    vec = np.average(vec, axis=0)\n",
    "    return vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg_vector_by_year(year):\n",
    "    target_word_vectors = {wrd: [] for wrd in target_words}\n",
    "    data = tweets[year]\n",
    "    for t in data:\n",
    "        word = t['word']\n",
    "        text = t['text']\n",
    "        try:\n",
    "            vec = generate_vector_from_context(word, text)\n",
    "        except ValueError:\n",
    "            pass\n",
    "        target_word_vectors[word].append(vec)\n",
    "    for wrd in target_words:\n",
    "        vecs = np.array(target_word_vectors[wrd])\n",
    "        target_word_vectors[wrd] = np.average(vecs, axis=0)\n",
    "    \n",
    "    return target_word_vectors\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[13], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m vecs_2019 \u001b[38;5;241m=\u001b[39m \u001b[43mavg_vector_by_year\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m2019\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      2\u001b[0m vecs_2020 \u001b[38;5;241m=\u001b[39m avg_vector_by_year(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m2020\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      3\u001b[0m vecs_2021 \u001b[38;5;241m=\u001b[39m avg_vector_by_year(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m2021\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "Cell \u001b[0;32mIn[12], line 8\u001b[0m, in \u001b[0;36mavg_vector_by_year\u001b[0;34m(year)\u001b[0m\n\u001b[1;32m      6\u001b[0m text \u001b[38;5;241m=\u001b[39m t[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m----> 8\u001b[0m     vec \u001b[38;5;241m=\u001b[39m \u001b[43mgenerate_vector_from_context\u001b[49m\u001b[43m(\u001b[49m\u001b[43mword\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m:\n\u001b[1;32m     10\u001b[0m     \u001b[38;5;28;01mpass\u001b[39;00m\n",
      "Cell \u001b[0;32mIn[11], line 13\u001b[0m, in \u001b[0;36mgenerate_vector_from_context\u001b[0;34m(word, text)\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m tok_t:\n\u001b[1;32m     12\u001b[0m     tok_t[item] \u001b[38;5;241m=\u001b[39m tok_t[item]\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m---> 13\u001b[0m vec \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mtok_t\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlast_hidden_state\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m0\u001b[39m)[idx:idx\u001b[38;5;241m+\u001b[39mlen_tok]\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[1;32m     14\u001b[0m vec \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39maverage(vec, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m vec\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py:1013\u001b[0m, in \u001b[0;36mBertModel.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1004\u001b[0m head_mask \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_head_mask(head_mask, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mnum_hidden_layers)\n\u001b[1;32m   1006\u001b[0m embedding_output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membeddings(\n\u001b[1;32m   1007\u001b[0m     input_ids\u001b[39m=\u001b[39minput_ids,\n\u001b[1;32m   1008\u001b[0m     position_ids\u001b[39m=\u001b[39mposition_ids,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1011\u001b[0m     past_key_values_length\u001b[39m=\u001b[39mpast_key_values_length,\n\u001b[1;32m   1012\u001b[0m )\n\u001b[0;32m-> 1013\u001b[0m encoder_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mencoder(\n\u001b[1;32m   1014\u001b[0m     embedding_output,\n\u001b[1;32m   1015\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mextended_attention_mask,\n\u001b[1;32m   1016\u001b[0m     head_mask\u001b[39m=\u001b[39;49mhead_mask,\n\u001b[1;32m   1017\u001b[0m     encoder_hidden_states\u001b[39m=\u001b[39;49mencoder_hidden_states,\n\u001b[1;32m   1018\u001b[0m     encoder_attention_mask\u001b[39m=\u001b[39;49mencoder_extended_attention_mask,\n\u001b[1;32m   1019\u001b[0m     past_key_values\u001b[39m=\u001b[39;49mpast_key_values,\n\u001b[1;32m   1020\u001b[0m     use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m   1021\u001b[0m     output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m   1022\u001b[0m     output_hidden_states\u001b[39m=\u001b[39;49moutput_hidden_states,\n\u001b[1;32m   1023\u001b[0m     return_dict\u001b[39m=\u001b[39;49mreturn_dict,\n\u001b[1;32m   1024\u001b[0m )\n\u001b[1;32m   1025\u001b[0m sequence_output \u001b[39m=\u001b[39m encoder_outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m   1026\u001b[0m pooled_output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpooler(sequence_output) \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpooler \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py:607\u001b[0m, in \u001b[0;36mBertEncoder.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    596\u001b[0m     layer_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    597\u001b[0m         layer_module\u001b[39m.\u001b[39m\u001b[39m__call__\u001b[39m,\n\u001b[1;32m    598\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    604\u001b[0m         output_attentions,\n\u001b[1;32m    605\u001b[0m     )\n\u001b[1;32m    606\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 607\u001b[0m     layer_outputs \u001b[39m=\u001b[39m layer_module(\n\u001b[1;32m    608\u001b[0m         hidden_states,\n\u001b[1;32m    609\u001b[0m         attention_mask,\n\u001b[1;32m    610\u001b[0m         layer_head_mask,\n\u001b[1;32m    611\u001b[0m         encoder_hidden_states,\n\u001b[1;32m    612\u001b[0m         encoder_attention_mask,\n\u001b[1;32m    613\u001b[0m         past_key_value,\n\u001b[1;32m    614\u001b[0m         output_attentions,\n\u001b[1;32m    615\u001b[0m     )\n\u001b[1;32m    617\u001b[0m hidden_states \u001b[39m=\u001b[39m layer_outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m    618\u001b[0m \u001b[39mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py:539\u001b[0m, in \u001b[0;36mBertLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    536\u001b[0m     cross_attn_present_key_value \u001b[39m=\u001b[39m cross_attention_outputs[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n\u001b[1;32m    537\u001b[0m     present_key_value \u001b[39m=\u001b[39m present_key_value \u001b[39m+\u001b[39m cross_attn_present_key_value\n\u001b[0;32m--> 539\u001b[0m layer_output \u001b[39m=\u001b[39m apply_chunking_to_forward(\n\u001b[1;32m    540\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfeed_forward_chunk, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mchunk_size_feed_forward, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mseq_len_dim, attention_output\n\u001b[1;32m    541\u001b[0m )\n\u001b[1;32m    542\u001b[0m outputs \u001b[39m=\u001b[39m (layer_output,) \u001b[39m+\u001b[39m outputs\n\u001b[1;32m    544\u001b[0m \u001b[39m# if decoder, return the attn key/values as the last output\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/transformers/pytorch_utils.py:241\u001b[0m, in \u001b[0;36mapply_chunking_to_forward\u001b[0;34m(forward_fn, chunk_size, chunk_dim, *input_tensors)\u001b[0m\n\u001b[1;32m    238\u001b[0m     \u001b[39m# concatenate output at same dimension\u001b[39;00m\n\u001b[1;32m    239\u001b[0m     \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39mcat(output_chunks, dim\u001b[39m=\u001b[39mchunk_dim)\n\u001b[0;32m--> 241\u001b[0m \u001b[39mreturn\u001b[39;00m forward_fn(\u001b[39m*\u001b[39;49minput_tensors)\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py:552\u001b[0m, in \u001b[0;36mBertLayer.feed_forward_chunk\u001b[0;34m(self, attention_output)\u001b[0m\n\u001b[1;32m    550\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mfeed_forward_chunk\u001b[39m(\u001b[39mself\u001b[39m, attention_output):\n\u001b[1;32m    551\u001b[0m     intermediate_output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mintermediate(attention_output)\n\u001b[0;32m--> 552\u001b[0m     layer_output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49moutput(intermediate_output, attention_output)\n\u001b[1;32m    553\u001b[0m     \u001b[39mreturn\u001b[39;00m layer_output\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py:464\u001b[0m, in \u001b[0;36mBertOutput.forward\u001b[0;34m(self, hidden_states, input_tensor)\u001b[0m\n\u001b[1;32m    463\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, hidden_states: torch\u001b[39m.\u001b[39mTensor, input_tensor: torch\u001b[39m.\u001b[39mTensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m torch\u001b[39m.\u001b[39mTensor:\n\u001b[0;32m--> 464\u001b[0m     hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdense(hidden_states)\n\u001b[1;32m    465\u001b[0m     hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout(hidden_states)\n\u001b[1;32m    466\u001b[0m     hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mLayerNorm(hidden_states \u001b[39m+\u001b[39m input_tensor)\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/2611final/lib/python3.11/site-packages/torch/nn/modules/linear.py:114\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> 114\u001b[0m     \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49mlinear(\u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbias)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "vecs_2019 = avg_vector_by_year('2019')\n",
    "vecs_2020 = avg_vector_by_year('2020')\n",
    "vecs_2021 = avg_vector_by_year('2021')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_correlation(A, B):\n",
    "    bert_pred = []\n",
    "    ground_truth = []\n",
    "    for key in A:\n",
    "        a = A[key]\n",
    "        b = B[key]\n",
    "        if type(a) != np.ndarray or type(b) != np.ndarray:\n",
    "            continue\n",
    "        cos_sim = float(cosine_similarity([a], [b]).flatten()[0])\n",
    "        cos_dist = 1 - cos_sim\n",
    "        bert_pred.append(cos_dist)\n",
    "        ground_truth.append(annotator[key])\n",
    "    return pearsonr(bert_pred, ground_truth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PearsonRResult(statistic=0.6280822488122723, pvalue=0.012166736321230068)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_correlation(vecs_2020, vecs_2021)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PearsonRResult(statistic=0.6377667878884206, pvalue=0.003305909229633156)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_correlation(vecs_2019, vecs_2020)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Instance-level meaning shift detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(data_path, labels_path):\n",
    "    # Load tweet instances\n",
    "    with open(data_path, 'r', encoding='utf-8') as file:\n",
    "        data_instances = [json.loads(line) for line in file]\n",
    "\n",
    "    # Load labels\n",
    "    with open(labels_path, 'r', encoding='utf-8') as file:\n",
    "        labels = dict(line.strip().split('\\t') for line in file)\n",
    "\n",
    "    return data_instances, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_path = 'data/train.data.jl'\n",
    "train_labels_path = 'data/train.labels.tsv'\n",
    "\n",
    "val_data_path = 'data/validation.data.jl'\n",
    "val_labels_path = 'data/validation.labels.tsv'\n",
    "\n",
    "test_data_path = 'data/trial.data.jl'\n",
    "test_labels_path = 'data/trial.gold.tsv'\n",
    "\n",
    "data_instances, labels = [], {}\n",
    "tr_data, tr_labels = load_data(train_data_path, train_labels_path)\n",
    "data_instances.extend(tr_data)\n",
    "labels = labels | tr_labels\n",
    "\n",
    "val_data, val_labels = load_data(val_data_path, val_labels_path)\n",
    "data_instances.extend(val_data)\n",
    "labels = labels | val_labels\n",
    "\n",
    "te_data, te_labels = load_data(test_data_path, test_labels_path)\n",
    "data_instances.extend(te_data)\n",
    "labels = labels | te_labels\n",
    "\n",
    "pairs = {item['id']: [item['tweet1']['text'], item['tweet2']['text'], item['word']] for item in data_instances}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_acc(threshold):\n",
    "    preds = []\n",
    "    gts = []\n",
    "    for key in pairs:\n",
    "        label = labels[key]\n",
    "        t1, t2, word = pairs[key]\n",
    "        try:\n",
    "            vec1 = generate_vector_from_context(word, t1)\n",
    "            vec2 = generate_vector_from_context(word, t2)\n",
    "        except ValueError:\n",
    "            continue\n",
    "        res = float(cosine_similarity([vec1], [vec2]).flatten()[0])\n",
    "        res = 1 if res > threshold else 0\n",
    "        gts.append(int(label))\n",
    "        preds.append(res)\n",
    "\n",
    "    return accuracy_score(gts, preds), f1_score(gts, preds)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6447225244831338, 0.6001224739742804)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_acc(0.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Results on each threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy and F1 score at threshold=0.0: (0.4515778019586507, 0.6221889055472264)\n",
      "Accuracy and F1 score at threshold=0.1: (0.4515778019586507, 0.6221889055472264)\n",
      "Accuracy and F1 score at threshold=0.2: (0.4515778019586507, 0.6221889055472264)\n",
      "Accuracy and F1 score at threshold=0.3: (0.4515778019586507, 0.6219054763690922)\n",
      "Accuracy and F1 score at threshold=0.4: (0.45973884657236125, 0.6242905788876276)\n",
      "Accuracy and F1 score at threshold=0.5: (0.47279651795429817, 0.6222222222222222)\n",
      "Accuracy and F1 score at threshold=0.6: (0.5337323177366703, 0.62129916040654)\n",
      "Accuracy and F1 score at threshold=0.7: (0.6447225244831338, 0.6001224739742804)\n",
      "Accuracy and F1 score at threshold=0.8: (0.6142546245919478, 0.30963972736124634)\n",
      "Accuracy and F1 score at threshold=0.9: (0.5522306855277476, 0.019070321811680575)\n"
     ]
    }
   ],
   "source": [
    "for threshold in range(10):\n",
    "    i = threshold / 10\n",
    "    print(f'Accuracy and F1 score at threshold={i}: {find_acc(i)}')\n",
    "# best threshold 0.7"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.11.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "4bbd96449fffa39e361fc66bcb5d1530b80999218b6fe704e78cc8f408c31850"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}