sgns.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import pickle\n",
    "from utils import get_sorted_tweets, get_target_words, load_annotator_labels\n",
    "from scipy.linalg import orthogonal_procrustes\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from scipy.stats import pearsonr\n",
    "from operator import itemgetter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sgns hyperparameters\n",
    "k_lst = [5, 7, 10, 12, 15]\n",
    "vector_size_lst = [100, 200, 300]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The following cell will take some amount of time to run since it is training 15 sgns models. However, these models are already saved so you can skip it if need be."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k: 5 vector_size: 100 year: 2019: Word2Vec<vocab=1271, vector_size=100, alpha=0.025>\n",
      "k: 5 vector_size: 100 year: 2020: Word2Vec<vocab=8861, vector_size=100, alpha=0.025>\n",
      "k: 5 vector_size: 100 year: 2021: Word2Vec<vocab=7591, vector_size=100, alpha=0.025>\n",
      "k: 5 vector_size: 200 year: 2019: Word2Vec<vocab=1271, vector_size=200, alpha=0.025>\n",
      "k: 5 vector_size: 200 year: 2020: Word2Vec<vocab=8861, vector_size=200, alpha=0.025>\n",
      "k: 5 vector_size: 200 year: 2021: Word2Vec<vocab=7591, vector_size=200, alpha=0.025>\n",
      "k: 5 vector_size: 300 year: 2019: Word2Vec<vocab=1271, vector_size=300, alpha=0.025>\n",
      "k: 5 vector_size: 300 year: 2020: Word2Vec<vocab=8861, vector_size=300, alpha=0.025>\n",
      "k: 5 vector_size: 300 year: 2021: Word2Vec<vocab=7591, vector_size=300, alpha=0.025>\n",
      "k: 7 vector_size: 100 year: 2019: Word2Vec<vocab=1271, vector_size=100, alpha=0.025>\n",
      "k: 7 vector_size: 100 year: 2020: Word2Vec<vocab=8861, vector_size=100, alpha=0.025>\n",
      "k: 7 vector_size: 100 year: 2021: Word2Vec<vocab=7591, vector_size=100, alpha=0.025>\n",
      "k: 7 vector_size: 200 year: 2019: Word2Vec<vocab=1271, vector_size=200, alpha=0.025>\n",
      "k: 7 vector_size: 200 year: 2020: Word2Vec<vocab=8861, vector_size=200, alpha=0.025>\n",
      "k: 7 vector_size: 200 year: 2021: Word2Vec<vocab=7591, vector_size=200, alpha=0.025>\n",
      "k: 7 vector_size: 300 year: 2019: Word2Vec<vocab=1271, vector_size=300, alpha=0.025>\n",
      "k: 7 vector_size: 300 year: 2020: Word2Vec<vocab=8861, vector_size=300, alpha=0.025>\n",
      "k: 7 vector_size: 300 year: 2021: Word2Vec<vocab=7591, vector_size=300, alpha=0.025>\n",
      "k: 10 vector_size: 100 year: 2019: Word2Vec<vocab=1271, vector_size=100, alpha=0.025>\n",
      "k: 10 vector_size: 100 year: 2020: Word2Vec<vocab=8861, vector_size=100, alpha=0.025>\n",
      "k: 10 vector_size: 100 year: 2021: Word2Vec<vocab=7591, vector_size=100, alpha=0.025>\n",
      "k: 10 vector_size: 200 year: 2019: Word2Vec<vocab=1271, vector_size=200, alpha=0.025>\n",
      "k: 10 vector_size: 200 year: 2020: Word2Vec<vocab=8861, vector_size=200, alpha=0.025>\n",
      "k: 10 vector_size: 200 year: 2021: Word2Vec<vocab=7591, vector_size=200, alpha=0.025>\n",
      "k: 10 vector_size: 300 year: 2019: Word2Vec<vocab=1271, vector_size=300, alpha=0.025>\n",
      "k: 10 vector_size: 300 year: 2020: Word2Vec<vocab=8861, vector_size=300, alpha=0.025>\n",
      "k: 10 vector_size: 300 year: 2021: Word2Vec<vocab=7591, vector_size=300, alpha=0.025>\n",
      "k: 12 vector_size: 100 year: 2019: Word2Vec<vocab=1271, vector_size=100, alpha=0.025>\n",
      "k: 12 vector_size: 100 year: 2020: Word2Vec<vocab=8861, vector_size=100, alpha=0.025>\n",
      "k: 12 vector_size: 100 year: 2021: Word2Vec<vocab=7591, vector_size=100, alpha=0.025>\n",
      "k: 12 vector_size: 200 year: 2019: Word2Vec<vocab=1271, vector_size=200, alpha=0.025>\n",
      "k: 12 vector_size: 200 year: 2020: Word2Vec<vocab=8861, vector_size=200, alpha=0.025>\n",
      "k: 12 vector_size: 200 year: 2021: Word2Vec<vocab=7591, vector_size=200, alpha=0.025>\n",
      "k: 12 vector_size: 300 year: 2019: Word2Vec<vocab=1271, vector_size=300, alpha=0.025>\n",
      "k: 12 vector_size: 300 year: 2020: Word2Vec<vocab=8861, vector_size=300, alpha=0.025>\n",
      "k: 12 vector_size: 300 year: 2021: Word2Vec<vocab=7591, vector_size=300, alpha=0.025>\n",
      "k: 15 vector_size: 100 year: 2019: Word2Vec<vocab=1271, vector_size=100, alpha=0.025>\n",
      "k: 15 vector_size: 100 year: 2020: Word2Vec<vocab=8861, vector_size=100, alpha=0.025>\n",
      "k: 15 vector_size: 100 year: 2021: Word2Vec<vocab=7591, vector_size=100, alpha=0.025>\n",
      "k: 15 vector_size: 200 year: 2019: Word2Vec<vocab=1271, vector_size=200, alpha=0.025>\n",
      "k: 15 vector_size: 200 year: 2020: Word2Vec<vocab=8861, vector_size=200, alpha=0.025>\n",
      "k: 15 vector_size: 200 year: 2021: Word2Vec<vocab=7591, vector_size=200, alpha=0.025>\n",
      "k: 15 vector_size: 300 year: 2019: Word2Vec<vocab=1271, vector_size=300, alpha=0.025>\n",
      "k: 15 vector_size: 300 year: 2020: Word2Vec<vocab=8861, vector_size=300, alpha=0.025>\n",
      "k: 15 vector_size: 300 year: 2021: Word2Vec<vocab=7591, vector_size=300, alpha=0.025>\n"
     ]
    }
   ],
   "source": [
    "date_dict = get_sorted_tweets()\n",
    "\n",
    "years = [\"2019\", \"2020\", \"2021\"]\n",
    "\n",
    "for k in k_lst:\n",
    "    for vector_size in vector_size_lst:\n",
    "        for year in years:\n",
    "\n",
    "            model = gensim.models.Word2Vec(\n",
    "                sg=1, # skipgram\n",
    "                hs=0, # negative sampling\n",
    "                negative=k, # number of negative samples\n",
    "                workers=4,\n",
    "                vector_size=vector_size\n",
    "            )\n",
    "            \n",
    "            sentence_list = []\n",
    "            data_year = date_dict[year]\n",
    "            for data in data_year:\n",
    "                sentence_list.append(data[\"tokens\"])\n",
    "\n",
    "            model.build_vocab(sentence_list)\n",
    "            model.train(sentence_list, total_examples=model.corpus_count, epochs=20)\n",
    "\n",
    "            print(f\"k: {k} vector_size: {vector_size} year: {year}: {model}\")\n",
    "\n",
    "            # # Save the vectors and the model\n",
    "            outpath = f'model_files/sngs_{year}_{k}_{vector_size}'\n",
    "            model.wv.save(outpath)\n",
    "            model.save(outpath + '.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_dist(v1, v2):\n",
    "    return 1 - metrics.pairwise.cosine_similarity(v1.reshape(1, -1), v2.reshape(1, -1))[0][0]\n",
    "\n",
    "def get_same_words(word_set1, word_set2):\n",
    "    return word_set1.intersection(word_set2)\n",
    "\n",
    "def create_matrices(word2vec1, word2vec2):\n",
    "    vocab1 = set(word2vec1.index_to_key)\n",
    "    vocab2 = set(word2vec2.index_to_key)\n",
    "\n",
    "    intersect = get_same_words(vocab1, vocab2)\n",
    "\n",
    "    mat1 = np.zeros((len(intersect), vector_size))\n",
    "    mat2 = np.zeros((len(intersect), vector_size))\n",
    "\n",
    "    for i, word in enumerate(intersect):\n",
    "        mat1[i] = word2vec1.get_vector(word)\n",
    "        mat2[i] = word2vec2.get_vector(word)\n",
    "    \n",
    "\n",
    "    return list(intersect), mat1, mat2\n",
    "\n",
    "def get_consine_distance(year1, year2, k, vector_size):\n",
    "    target_words = get_target_words()\n",
    "\n",
    "    word2vec_path1 = f\"model_files/sngs_{year1}_{k}_{vector_size}\"\n",
    "    word2vec_path2 = f\"model_files/sngs_{year2}_{k}_{vector_size}\"\n",
    "    word2vec1 = gensim.models.KeyedVectors.load(word2vec_path1)\n",
    "    word2vec2 = gensim.models.KeyedVectors.load(word2vec_path2)\n",
    "\n",
    "    intersect, A, B = create_matrices(word2vec1, word2vec2)\n",
    "\n",
    "    result = orthogonal_procrustes(A, B)\n",
    "    A_op = A @ result[0]\n",
    "    cosine_distances = {}\n",
    "    for target in target_words:\n",
    "        try:\n",
    "            idx = intersect.index(target)\n",
    "            cosine_distances[target] = cosine_dist(A_op[idx], B[idx])\n",
    "        except:\n",
    "            pass\n",
    "            # print(f\"target word {target} is not found in the intersect of both corpora\")\n",
    "    \n",
    "    return cosine_distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg pearson coeff and p-value for k=15, vector_size=100: 0.47284280816877333, 0.054630160601486316\n",
      "avg pearson coeff and p-value for k=12, vector_size=100: 0.4358772878640359, 0.07781644216479439\n",
      "avg pearson coeff and p-value for k=10, vector_size=100: 0.32453833437791024, 0.22861472972834954\n",
      "avg pearson coeff and p-value for k=5, vector_size=100: 0.23200998421737587, 0.3696732158956685\n",
      "avg pearson coeff and p-value for k=5, vector_size=200: 0.22400907981207732, 0.4069698064214717\n",
      "avg pearson coeff and p-value for k=7, vector_size=100: 0.21790866223737831, 0.3862896561031963\n",
      "avg pearson coeff and p-value for k=10, vector_size=200: 0.21504609104114925, 0.424653968576419\n",
      "avg pearson coeff and p-value for k=7, vector_size=200: 0.19233450259659673, 0.516732760503054\n",
      "avg pearson coeff and p-value for k=15, vector_size=300: 0.15635637619077497, 0.5054529001259831\n",
      "avg pearson coeff and p-value for k=12, vector_size=200: 0.1536200932435125, 0.5840827144359321\n",
      "avg pearson coeff and p-value for k=15, vector_size=200: 0.1417956707985871, 0.5925538380417571\n",
      "avg pearson coeff and p-value for k=10, vector_size=300: 0.11187835834992252, 0.672906719519427\n",
      "avg pearson coeff and p-value for k=12, vector_size=300: 0.09791066462614517, 0.5587836384515221\n",
      "avg pearson coeff and p-value for k=7, vector_size=300: 0.062416639154617794, 0.6813020819935387\n",
      "avg pearson coeff and p-value for k=5, vector_size=300: 0.060538380307854384, 0.8105500918916557\n"
     ]
    }
   ],
   "source": [
    "years = [(\"2019\", \"2020\"), (\"2020\", \"2021\")]\n",
    "\n",
    "scores = []\n",
    "avg_scores = []\n",
    "\n",
    "for k in k_lst:\n",
    "    for vector_size in vector_size_lst:\n",
    "        total_pearson = 0\n",
    "        total_p_value = 0\n",
    "        for year1, year2 in years:\n",
    "            cd = get_consine_distance(year1, year2, k, vector_size)\n",
    "            \n",
    "            labels = load_annotator_labels()\n",
    "\n",
    "            sgns_vec = []\n",
    "            annotator_vec = []\n",
    "\n",
    "            for key in cd.keys():\n",
    "                sgns_vec.append(float(cd[key]))\n",
    "                annotator_vec.append(float(labels[key]))\n",
    "\n",
    "            pearson, p_value = pearsonr(sgns_vec, annotator_vec)\n",
    "            total_pearson += pearson\n",
    "            total_p_value += p_value\n",
    "            scores.append((k, vector_size, pearson, p_value, year1, year2))\n",
    "        avg_scores.append((k, vector_size, total_pearson / 2, total_p_value / 2))\n",
    "\n",
    "sorted_avg_scores = sorted(avg_scores, key=itemgetter(2), reverse=True)\n",
    "\n",
    "for score in sorted_avg_scores:\n",
    "    k, vector_size, pearson, p_value = score\n",
    "    print(f\"avg pearson coeff and p-value for k={k}, vector_size={vector_size}: {pearson}, {p_value}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all scores\n",
      "pearson coeff and p-value for k=5, vector_size=100, year=2019-2020: 0.2978033729975836, 0.21560374317047998\n",
      "pearson coeff and p-value for k=5, vector_size=100, year=2020-2021: 0.16621659543716816, 0.5237426886208569\n",
      "pearson coeff and p-value for k=5, vector_size=200, year=2019-2020: 0.3259475466241949, 0.17324308119146234\n",
      "pearson coeff and p-value for k=5, vector_size=200, year=2020-2021: 0.12207061299995976, 0.640696531651481\n",
      "pearson coeff and p-value for k=5, vector_size=300, year=2019-2020: 0.11815032251905985, 0.6299939163568279\n",
      "pearson coeff and p-value for k=5, vector_size=300, year=2020-2021: 0.0029264380966489076, 0.9911062674264837\n",
      "pearson coeff and p-value for k=7, vector_size=100, year=2019-2020: 0.23569636575114633, 0.33134520910573345\n",
      "pearson coeff and p-value for k=7, vector_size=100, year=2020-2021: 0.20012095872361027, 0.44123410310065914\n",
      "pearson coeff and p-value for k=7, vector_size=200, year=2019-2020: 0.34816681976429426, 0.14408174721704506\n",
      "pearson coeff and p-value for k=7, vector_size=200, year=2020-2021: 0.03650218542889922, 0.8893837737890629\n",
      "pearson coeff and p-value for k=7, vector_size=300, year=2019-2020: 0.1677342019687706, 0.49246834640587633\n",
      "pearson coeff and p-value for k=7, vector_size=300, year=2020-2021: -0.042900923659535004, 0.870135817581201\n",
      "pearson coeff and p-value for k=10, vector_size=100, year=2019-2020: 0.42518678671633525, 0.06955970456144324\n",
      "pearson coeff and p-value for k=10, vector_size=100, year=2020-2021: 0.2238898820394853, 0.38766975489525585\n",
      "pearson coeff and p-value for k=10, vector_size=200, year=2019-2020: 0.31521453757554085, 0.18867039942006886\n",
      "pearson coeff and p-value for k=10, vector_size=200, year=2020-2021: 0.11487764450675766, 0.6606375377327691\n",
      "pearson coeff and p-value for k=10, vector_size=300, year=2019-2020: 0.2091902201165484, 0.39006292810836646\n",
      "pearson coeff and p-value for k=10, vector_size=300, year=2020-2021: 0.014566496583296652, 0.9557505109304876\n",
      "pearson coeff and p-value for k=12, vector_size=100, year=2019-2020: 0.47674019833465836, 0.03904123106190101\n",
      "pearson coeff and p-value for k=12, vector_size=100, year=2020-2021: 0.3950143773934135, 0.11659165326768775\n",
      "pearson coeff and p-value for k=12, vector_size=200, year=2019-2020: 0.28404149548278, 0.23858769669177618\n",
      "pearson coeff and p-value for k=12, vector_size=200, year=2020-2021: 0.02319869100424498, 0.929577732180088\n",
      "pearson coeff and p-value for k=12, vector_size=300, year=2019-2020: 0.2541892018587357, 0.2936542126042689\n",
      "pearson coeff and p-value for k=12, vector_size=300, year=2020-2021: -0.058367872606445395, 0.8239130642987753\n",
      "pearson coeff and p-value for k=15, vector_size=100, year=2019-2020: 0.5167164981210678, 0.023497841741837857\n",
      "pearson coeff and p-value for k=15, vector_size=100, year=2020-2021: 0.4289691182164789, 0.08576247946113477\n",
      "pearson coeff and p-value for k=15, vector_size=200, year=2019-2020: 0.23409804929273803, 0.3347303252077464\n",
      "pearson coeff and p-value for k=15, vector_size=200, year=2020-2021: 0.04949329230443618, 0.8503773508757677\n",
      "pearson coeff and p-value for k=15, vector_size=300, year=2019-2020: 0.35426457690024055, 0.13671715356499156\n",
      "pearson coeff and p-value for k=15, vector_size=300, year=2020-2021: -0.041551824518690585, 0.8741886466869746\n"
     ]
    }
   ],
   "source": [
    "print(\"all scores\")\n",
    "for score_obj in scores:\n",
    "    k, vector_size, pearson, p_value, year1, year2 = score_obj\n",
    "    print(f\"pearson coeff and p-value for k={k}, vector_size={vector_size}, year={year1}-{year2}: {pearson}, {p_value}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.11.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4bbd96449fffa39e361fc66bcb5d1530b80999218b6fe704e78cc8f408c31850"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}