next_item_probability_vis-steam-mfs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import math\n",
    "torch.cuda.empty_cache()\n",
    "from recbole.quick_start import load_data_and_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05 Aug 19:51    INFO  [Training]: train_batch_size = [128] negative sampling: [None]\n",
      "05 Aug 19:51    INFO  [Evaluation]: eval_batch_size = [4096] eval_args: [{'split': {'LS': 'valid_and_test'}, 'order': 'TO', 'mode': 'full', 'group_by': 'user'}]\n",
      "05 Aug 19:53    INFO  [Training]: train_batch_size = [128] negative sampling: [None]\n",
      "05 Aug 19:53    INFO  [Evaluation]: eval_batch_size = [4096] eval_args: [{'split': {'LS': 'valid_and_test'}, 'order': 'TO', 'mode': 'full', 'group_by': 'user'}]\n",
      "05 Aug 19:55    INFO  [Training]: train_batch_size = [128] negative sampling: [None]\n",
      "05 Aug 19:55    INFO  [Evaluation]: eval_batch_size = [4096] eval_args: [{'split': {'LS': 'valid_and_test'}, 'order': 'TO', 'mode': 'full', 'group_by': 'user'}]\n"
     ]
    }
   ],
   "source": [
    "# RepeatNet\n",
    "# config_rn, model_rn, dataset_rn, train_data_rn, valid_data_rn, test_data_rn = load_data_and_model(\"saved/RepeatNet-Jun-06-2022_13-20-15.pth\")\n",
    "# SARSRec MFS + CLR\n",
    "# config_ours, model_ours, dataset_ours, train_data_ours, valid_data_ours, test_data_ours = load_data_and_model(\"/work/hawshiuancha_umass_edu/projects/RecBole-1/saved/SASRec-Jul-21-2022_03-28-09.pth\")\n",
    "\n",
    "# RepeatNet\n",
    "config_rn, model_rn, dataset_rn, train_data_rn, valid_data_rn, test_data_rn = load_data_and_model(\"/work/hawshiuancha_umass_edu/projects/RecBole-1/saved/RepeatNet-Jul-23-2023_22-25-20.pth\")\n",
    "\n",
    "# Softmax + Mi\n",
    "config_smi, model_smi, dataset_smi, train_data_smi, valid_data_smi, test_data_smi = load_data_and_model(\"/work/hawshiuancha_umass_edu/projects/RecBole-1/saved/GRU4Rec_Ours-Jul-23-2023_20-33-59.pth\")\n",
    "\n",
    "# Softmax + CPR:100 + Mi\n",
    "config_scmi, model_scmi, dataset_scmi, train_data_scmi, valida_data_scmi, test_data_scmi = load_data_and_model(\"/work/hawshiuancha_umass_edu/projects/RecBole-1/saved/GRU4Rec_Ours-Jul-23-2023_20-32-45.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from recbole.utils import get_trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer_rn = get_trainer(config_rn['MODEL_TYPE'], config_rn['model'])(config_rn, model_rn)\n",
    "trainer_smi = get_trainer(config_smi['MODEL_TYPE'], config_smi['model'])(config_smi, model_smi)\n",
    "trainer_scmi = get_trainer(config_scmi['MODEL_TYPE'], config_scmi['model'])(config_scmi, model_scmi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_rn = trainer_rn.evaluationWithInteractions(test_data_rn, load_best_model=False, model_file=\"/work/hawshiuancha_umass_edu/projects/RecBole-1/saved/RepeatNet-Jul-23-2023_22-25-20.pth\")\n",
    "results_smi = trainer_smi.evaluationWithInteractions(test_data_smi, load_best_model=False, model_file=\"/work/hawshiuancha_umass_edu/projects/RecBole-1/saved/GRU4Rec_Ours-Jul-23-2023_20-33-59.pth\")\n",
    "results_scmi = trainer_scmi.evaluationWithInteractions(test_data_scmi, load_best_model=False, model_file=\"/work/hawshiuancha_umass_edu/projects/RecBole-1/saved/GRU4Rec_Ours-Jul-23-2023_20-32-45.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_rn, interactions_rn_array, scores_rn_array = results_rn\n",
    "result_smi, interactions_smi_array, scores_smi_array = results_smi\n",
    "result_scmi, interactions_scmi_array, scores_scmi_array = results_scmi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'recall@10': 0.2092,\n",
       " 'mrr@10': 0.1488,\n",
       " 'ndcg@10': 0.1628,\n",
       " 'hit@10': 0.2092,\n",
       " 'precision@10': 0.0209}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_rn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'recall@10': 0.0246,\n",
       " 'mrr@10': 0.0085,\n",
       " 'ndcg@10': 0.0122,\n",
       " 'hit@10': 0.0246,\n",
       " 'precision@10': 0.0025}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'recall@10': 0.0355,\n",
       " 'mrr@10': 0.0168,\n",
       " 'ndcg@10': 0.0212,\n",
       " 'hit@10': 0.0355,\n",
       " 'precision@10': 0.0036}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_scmi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input_data_rn, result_rn, interactions_rn, scores_rn, positive_u_rn, positive_i_rn = results_rn\n",
    "# input_data_smi, result_smi, interactions_smi, scores_smi, positive_u_smi, positive_i_smi = results_smi\n",
    "# input_data_scmi, result_scmi, interactions_scmi, scores_scmi, positive_u_scmi, positive_i_scmi = results_scmi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tokens_info(dataset_name):\n",
    "    file_path = \"dataset/\" + dataset_name + \"/\" + dataset_name + \".item\"\n",
    "    f = open(file_path, \"r\")\n",
    "    lines = f.readlines()\n",
    "    token_to_info_header = [\"app\", \"dev\", \"early\", \"genres\", \"meta\", \"price\", \"publisher\", \"sentiment\", \"specs\", \"tags\"]\n",
    "    token_to_info = {}\n",
    "    for line in lines[1:]:\n",
    "        try:\n",
    "            arr = line.split('\\t')\n",
    "            item_token = arr[4]\n",
    "            item_info = arr[:4] + arr[5:7] + arr[8:]\n",
    "            token_to_info[item_token] = item_info\n",
    "        except e:\n",
    "            print(\"Incorrect data format!\", line)\n",
    "    return token_to_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_to_info = get_tokens_info(\"steam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of batches: 82\n",
      "The batch_size of interaction: 4096\n",
      "    user_id, torch.Size([4096]), cpu, torch.int64\n",
      "    play_hours, torch.Size([4096]), cpu, torch.float32\n",
      "    product_id, torch.Size([4096]), cpu, torch.int64\n",
      "    timestamp, torch.Size([4096]), cpu, torch.float32\n",
      "    item_length, torch.Size([4096]), cpu, torch.int64\n",
      "    play_hours_list, torch.Size([4096, 50]), cpu, torch.float64\n",
      "    product_id_list, torch.Size([4096, 50]), cpu, torch.int64\n",
      "    timestamp_list, torch.Size([4096, 50]), cpu, torch.float64\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of batches:\", len(interactions_rn_array))\n",
    "print(interactions_rn_array[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prettytable import PrettyTable\n",
    "from scipy.special import softmax\n",
    "import numpy as np\n",
    "def visualize_top_k_suggestions(scores, token_to_info, id_to_token_map, k=5):\n",
    "    probs = softmax(scores.cpu().numpy())\n",
    "    top_k_ids = np.flip(probs.argsort())[:k]\n",
    "    results = PrettyTable([\"s/no.\", \"token\", \"app\", \"dev\", \"early\", \"genres\", \"meta\", \"price\", \"publisher\", \"sentiment\", \"specs\", \"tags\", \"probability\"])\n",
    "    results._max_width = {\"s/no.\" : 4, \"token\" : 4, \"app\": 4, \"dev\": 4, \"early\": 4, \"genres\": 5, \"meta\": 3, \"price\": 4, \"publisher\": 4, \"sentiment\": 4, \"specs\": 5, \"tags\":5}\n",
    "    for i in range(k):\n",
    "        identifier = top_k_ids[i]\n",
    "        token = id_to_token_map[identifier]\n",
    "        info = token_to_info[token]\n",
    "        row = ['N/A']*13\n",
    "        row[0] = identifier\n",
    "        row[1] = token\n",
    "        if len(info) >= 1:\n",
    "            row[2] = info[0]\n",
    "        if len(info) >= 2:\n",
    "            row[3] = info[1]\n",
    "        if len(info) >= 3:\n",
    "            row[4] = info[2]\n",
    "        if len(info) >= 4:\n",
    "            row[5] = info[3]\n",
    "        if len(info) >= 5:\n",
    "            row[6] = info[4]\n",
    "        if len(info) >= 6:\n",
    "            row[7] = info[5]\n",
    "        if len(info) >= 7:\n",
    "            row[8] = info[6]\n",
    "        if len(info) >= 8:\n",
    "            row[9] = info[7]\n",
    "        if len(info) >= 9:\n",
    "            row[10] = info[8]\n",
    "        if len(info) >= 10:\n",
    "            row[11] = info[9]\n",
    "        row[12] = probs[top_k_ids[i]]\n",
    "        results.add_row(row)\n",
    "    print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_input(identifier, token_to_info, id_to_token_map):\n",
    "    results = PrettyTable([\"s/no.\", \"token\", \"app\", \"dev\", \"early_access\", \"genres\", \"metascore\", \"price\", \"publisher\", \"sentiment\", \"specs\", \"tags\"])\n",
    "    results._max_width = {\"s/no.\" : 4, \"token\" : 4, \"app\": 4, \"dev\": 4, \"early\": 4, \"genres\": 5, \"meta\": 3, \"price\": 4, \"publisher\": 4, \"sentiment\": 4, \"specs\": 5, \"tags\":5}\n",
    "    token = id_to_token_map[identifier]\n",
    "    info = token_to_info[token]\n",
    "    row = ['N/A']*12\n",
    "    row[0] = identifier\n",
    "    row[1] = token\n",
    "    if len(info) >= 1:\n",
    "            row[2] = info[0]\n",
    "    if len(info) >= 2:\n",
    "        row[3] = info[1]\n",
    "    if len(info) >= 3:\n",
    "        row[4] = info[2]\n",
    "    if len(info) >= 4:\n",
    "        row[5] = info[3]\n",
    "    if len(info) >= 5:\n",
    "        row[6] = info[4]\n",
    "    if len(info) >= 6:\n",
    "        row[7] = info[5]\n",
    "    if len(info) >= 7:\n",
    "        row[8] = info[6]\n",
    "    if len(info) >= 8:\n",
    "        row[9] = info[7]\n",
    "    if len(info) >= 9:\n",
    "        row[10] = info[8]\n",
    "    if len(info) >= 10:\n",
    "        row[11] = info[9]\n",
    "    results.add_row(row)\n",
    "    print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RepeatNet losses:\n",
      "Average loss when actual item repeated: 1.433872209640739\n",
      "Average loss when actual item not repeated: 7.477564935989621\n"
     ]
    }
   ],
   "source": [
    "# RepeatNet results\n",
    "repeat_selection_losses_rn = []\n",
    "nonrepeat_selection_losses_rn = []\n",
    "for interactions_rn, scores_rn in zip(interactions_rn_array, scores_rn_array):\n",
    "    for i in range(len(interactions_rn['product_id'])):\n",
    "        if interactions_rn['product_id'][i] in interactions_rn['product_id_list'][i]:\n",
    "            repeat_selection_losses_rn.append(-math.log(scores_rn[i][int(interactions_rn['product_id'][i])]))\n",
    "        else:\n",
    "            nonrepeat_selection_losses_rn.append(-math.log(scores_rn[i][int(interactions_rn['product_id'][i])]))\n",
    "print(\"RepeatNet losses:\")\n",
    "print(f\"Average loss when actual item repeated: {np.mean(repeat_selection_losses_rn)}\")\n",
    "print(f\"Average loss when actual item not repeated: {np.mean(nonrepeat_selection_losses_rn)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Softmax + Mi losses:\n",
      "Average loss when actual item repeated: 8.274012296898263\n",
      "Average loss when actual item not repeated: 10.102628799457856\n"
     ]
    }
   ],
   "source": [
    "# Softmax + Mi results\n",
    "repeat_selection_losses_smi = []\n",
    "nonrepeat_selection_losses_smi = []\n",
    "for interactions_smi, scores_smi in zip(interactions_smi_array, scores_smi_array):\n",
    "    for i in range(len(interactions_smi['product_id'])):\n",
    "        if interactions_smi['product_id'][i] in interactions_smi['product_id_list'][i]:\n",
    "            repeat_selection_losses_smi.append(-math.log(scores_smi[i][int(interactions_smi['product_id'][i])]))\n",
    "        else:\n",
    "            nonrepeat_selection_losses_smi.append(-math.log(scores_smi[i][int(interactions_smi['product_id'][i])]))\n",
    "print(\"Softmax + Mi losses:\")\n",
    "print(f\"Average loss when actual item repeated: {np.mean(repeat_selection_losses_smi)}\")\n",
    "print(f\"Average loss when actual item not repeated: {np.mean(nonrepeat_selection_losses_smi)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Softmax + CPR:100 + Mi losses:\n",
      "Average loss when actual item repeated: 7.1270048418774\n",
      "Average loss when actual item not repeated: 9.967234585455143\n"
     ]
    }
   ],
   "source": [
    "# Softmax + CPR:100 + Mi results\n",
    "repeat_selection_losses_scmi = []\n",
    "nonrepeat_selection_losses_scmi = []\n",
    "for interactions_scmi, scores_scmi in zip(interactions_scmi_array, scores_scmi_array):\n",
    "    for i in range(len(interactions_scmi['product_id'])):\n",
    "        if interactions_scmi['product_id'][i] in interactions_scmi['product_id_list'][i]:\n",
    "            repeat_selection_losses_scmi.append(-math.log(scores_scmi[i][int(interactions_scmi['product_id'][i])]))\n",
    "        else:\n",
    "            nonrepeat_selection_losses_scmi.append(-math.log(scores_scmi[i][int(interactions_scmi['product_id'][i])]))\n",
    "print(\"Softmax + CPR:100 + Mi losses:\")\n",
    "print(f\"Average loss when actual item repeated: {np.mean(repeat_selection_losses_scmi)}\")\n",
    "print(f\"Average loss when actual item not repeated: {np.mean(nonrepeat_selection_losses_scmi)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_batches = len(interactions_rn_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # In this cell, we select examples where there are only 5 items in the history and Ground Truth is in history\n",
    "# num_batches = len(interactions_rn_array)\n",
    "# example_indexes = [] # this is a list of tuples to retrieve example\n",
    "# for batch_index in range(num_batches):\n",
    "#     interactions_rn = interactions_rn_array[batch_index]\n",
    "#     for i in range(len(interactions_rn)):\n",
    "#         if (interactions_rn['product_id_list'][i] != 0).sum().item() <= 5 and (interactions_rn['product_id_list'][i] == interactions_rn['product_id'][i].item()).sum().item() > 0:\n",
    "#             example_indexes.append((batch_index, i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In this cell, we select examples where there are only 5 items in the history\n",
    "num_batches = len(interactions_rn_array)\n",
    "example_indexes = [] # this is a list of tuples to retrieve example\n",
    "for batch_index in range(num_batches):\n",
    "    interactions_rn = interactions_rn_array[batch_index]\n",
    "    for i in range(len(interactions_rn)):\n",
    "        if (interactions_rn['product_id_list'][i] != 0).sum().item() <= 5:\n",
    "            example_indexes.append((batch_index, i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128852"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(example_indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in this cell, we select final examples where Softmax + CPR:100 + Mi has the GT in top 3 \n",
    "# but SoftMax + Mi, and Repeatnet don't \n",
    "# final_example_indexes = []\n",
    "# for i, j in example_indexes:\n",
    "#     interactions_rn = interactions_rn_array[i]\n",
    "#     scores_rn = scores_rn_array[i][j]\n",
    "#     scores_smi = scores_smi_array[i][j]\n",
    "#     scores_scmi = scores_scmi_array[i][j]\n",
    "#     sorted_scores_rn = scores_rn.sort(dim=0, descending=True).indices\n",
    "#     sorted_scores_smi = scores_smi.sort(dim=0, descending=True).indices\n",
    "#     sorted_scores_scmi = scores_scmi.sort(dim=0, descending=True).indices\n",
    "#     if (interactions_rn['product_id'][j] in sorted_scores_scmi[:3]) and (interactions_rn['product_id'][j] not in sorted_scores_smi[:3]) and (interactions_rn['product_id'][j] not in sorted_scores_rn[:3]):\n",
    "#         final_example_indexes.append((i, j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hasIntersection(list1, list2):\n",
    "    list3 = [value for value in list1 if value in list2]\n",
    "    return len(list3) > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in this cell, we select final examples where Softmax + CPR:100 + Mi has the GT in top 3 \n",
    "# and RepeatNet and SoftMax + Mi have some games from history in top 3 items\n",
    "final_example_indexes = []\n",
    "for i, j in example_indexes:\n",
    "    interactions_rn = interactions_rn_array[i]\n",
    "    scores_rn = scores_rn_array[i][j]\n",
    "    scores_smi = scores_smi_array[i][j]\n",
    "    scores_scmi = scores_scmi_array[i][j]\n",
    "    sorted_scores_rn = scores_rn.sort(dim=0, descending=True).indices\n",
    "    sorted_scores_smi = scores_smi.sort(dim=0, descending=True).indices\n",
    "    sorted_scores_scmi = scores_scmi.sort(dim=0, descending=True).indices\n",
    "    if (interactions_rn['product_id'][j] in sorted_scores_scmi[:3]) and (interactions_rn['product_id'][j] not in sorted_scores_smi[:3]) and (interactions_rn['product_id'][j] not in sorted_scores_rn[:3]):\n",
    "        if hasIntersection(interactions_rn['product_id_list'][j], sorted_scores_rn[:3]) and hasIntersection(interactions_rn['product_id_list'][j], sorted_scores_smi[:3]):\n",
    "            final_example_indexes.append((i, j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\n"
     ]
    }
   ],
   "source": [
    "print(len(final_example_indexes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "------History------\n",
      "\n",
      "The Elder Scrolls IV: Oblivion\\xae Game of the Year Edition\n",
      "Reus\n",
      "Home\n",
      "Rust\n",
      "\n",
      "---Ground truth----\n",
      "Chivalry: Medieval Warfare\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Rust 0.4181603193283081\n",
      "Starbound 0.007947315461933613\n",
      "Borderlands 2 0.007651458960026503\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.037949319928884506\n",
      "No More Room in Hell 0.031017225235700607\n",
      "Portal 2 0.025028159841895103\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.07268843054771423\n",
      "Mount & Blade: Warband 0.028470205143094063\n",
      "Saints Row: The Third 0.02274739183485508\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.004932015668600798\n",
      "Softmax + Mi:  0.005092207342386246\n",
      "Softmax + CPR:100 + Mi:  0.07268843054771423\n",
      "\n",
      "------History------\n",
      "\n",
      "Delver\n",
      "Rust\n",
      "The Ship: Murder Party\n",
      "STAR WARS\\u2122 - The Force Unleashed\\u2122 Ultimate Sith Edition\n",
      "\n",
      "---Ground truth----\n",
      "Mount & Blade: Warband\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "STAR WARS\\u2122 - The Force Unleashed\\u2122 Ultimate Sith Edition 0.030165234580636024\n",
      "Borderlands 2 0.015920547768473625\n",
      "STAR WARS\\u2122: The Force Unleashed\\u2122 II 0.015616906806826591\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03757643699645996\n",
      "No More Room in Hell 0.032080478966236115\n",
      "Portal 2 0.024475157260894775\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.06793413311243057\n",
      "Mount & Blade: Warband 0.027882639318704605\n",
      "ORION: Prelude 0.026462161913514137\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.008811384439468384\n",
      "Softmax + Mi:  0.007693079765886068\n",
      "Softmax + CPR:100 + Mi:  0.027882639318704605\n",
      "\n",
      "------History------\n",
      "\n",
      "STAR WARS\\u2122 - Knights of the Old Republic\\u2122\n",
      "Rust\n",
      "Rust\n",
      "Mount & Blade: Warband\n",
      "Mount & Blade: Warband\n",
      "\n",
      "---Ground truth----\n",
      "Just Cause 2\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Mount & Blade: Warband 0.270098477602005\n",
      "Starbound 0.010134339332580566\n",
      "Chivalry: Medieval Warfare 0.010092283599078655\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03561300411820412\n",
      "No More Room in Hell 0.027930503711104393\n",
      "Portal 2 0.0256364569067955\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.07918461412191391\n",
      "Just Cause 2 0.022710207849740982\n",
      "Saints Row: The Third 0.021471647545695305\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.0021637005265802145\n",
      "Softmax + Mi:  0.00405069999396801\n",
      "Softmax + CPR:100 + Mi:  0.022710207849740982\n",
      "\n",
      "------History------\n",
      "\n",
      "Rust\n",
      "Rust\n",
      "Chivalry: Medieval Warfare\n",
      "Chivalry: Medieval Warfare\n",
      "\n",
      "---Ground truth----\n",
      "Overgrowth\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.06141195818781853\n",
      "DayZ 0.01882323808968067\n",
      "Heroes & Generals 0.013957444578409195\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03598383069038391\n",
      "No More Room in Hell 0.02940618246793747\n",
      "Portal 2 0.025531884282827377\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Oracle 0.19510367512702942\n",
      "The Uncertain: Episode 1 - The Last Quiet Day 0.08366697281599045\n",
      "Overgrowth 0.046973440796136856\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.0018682186491787434\n",
      "Softmax + Mi:  0.0004705664177890867\n",
      "Softmax + CPR:100 + Mi:  0.046973440796136856\n",
      "\n",
      "------History------\n",
      "\n",
      "Contagion\n",
      "Euro Truck Simulator 2\n",
      "Euro Truck Simulator 2\n",
      "Rust\n",
      "\n",
      "---Ground truth----\n",
      "Chivalry: Medieval Warfare\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Rust 0.3101258873939514\n",
      "DayZ 0.012234905734658241\n",
      "Arma 3 0.010153246112167835\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.036671798676252365\n",
      "No More Room in Hell 0.029170583933591843\n",
      "Portal 2 0.02598622813820839\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.07570971548557281\n",
      "Mount & Blade: Warband 0.042736172676086426\n",
      "Just Cause 2 0.021935805678367615\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.006473115645349026\n",
      "Softmax + Mi:  0.0050484659150242805\n",
      "Softmax + CPR:100 + Mi:  0.07570971548557281\n",
      "\n",
      "------History------\n",
      "\n",
      "Worms Clan Wars\n",
      "Risk of Rain\n",
      "Rust\n",
      "Rust\n",
      "Rust\n",
      "\n",
      "---Ground truth----\n",
      "Chivalry: Medieval Warfare\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Rust 0.2321251481771469\n",
      "Team Fortress 2 0.012586977332830429\n",
      "Starbound 0.011681620962917805\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03775780275464058\n",
      "No More Room in Hell 0.030645549297332764\n",
      "Portal 2 0.025736099109053612\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.07090676575899124\n",
      "Mount & Blade: Warband 0.04217510297894478\n",
      "Just Cause 2 0.023003417998552322\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.006604206748306751\n",
      "Softmax + Mi:  0.005068211350589991\n",
      "Softmax + CPR:100 + Mi:  0.07090676575899124\n",
      "\n",
      "------History------\n",
      "\n",
      "Chivalry: Medieval Warfare\n",
      "APB Reloaded\n",
      "NEOTOKYO\n",
      "Rust\n",
      "\n",
      "---Ground truth----\n",
      "Overgrowth\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Rust 0.4601832628250122\n",
      "Heroes & Generals 0.010176020674407482\n",
      "Team Fortress 2 0.010160801000893116\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.034704793244600296\n",
      "No More Room in Hell 0.03119167871773243\n",
      "Portal 2 0.02711116336286068\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "HIT 0.053453780710697174\n",
      "Railroad Tycoon 3 0.034977663308382034\n",
      "Overgrowth 0.02807888761162758\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.0008048668969422579\n",
      "Softmax + Mi:  0.0004567589785438031\n",
      "Softmax + CPR:100 + Mi:  0.02807888761162758\n",
      "\n",
      "------History------\n",
      "\n",
      "Gunpoint\n",
      "Mark of the Ninja\n",
      "Rust\n",
      "Receiver\n",
      "\n",
      "---Ground truth----\n",
      "Overgrowth\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Receiver 0.028412872925400734\n",
      "Team Fortress 2 0.012138968333601952\n",
      "Hotline Miami 0.008631214499473572\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.034117043018341064\n",
      "No More Room in Hell 0.03218231722712517\n",
      "Portal 2 0.02491474337875843\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "The Uncertain: Episode 1 - The Last Quiet Day 0.05338260158896446\n",
      "Chivalry: Medieval Warfare 0.04844988137483597\n",
      "Overgrowth 0.048280470073223114\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.00410106685012579\n",
      "Softmax + Mi:  0.0005096423556096852\n",
      "Softmax + CPR:100 + Mi:  0.048280470073223114\n",
      "\n",
      "------History------\n",
      "\n",
      "Rust\n",
      "The Elder Scrolls III: Morrowind\\xae Game of the Year Edition\n",
      "Serious Sam Double D XXL\n",
      "u\"Wasteland 2: Directors Cut\"\n",
      "\n",
      "---Ground truth----\n",
      "Mount & Blade: Warband\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Team Fortress 2 0.01854977011680603\n",
      "u\"Wasteland 2: Directors Cut\" 0.014338587410748005\n",
      "Fallout 4 0.014047352597117424\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "The Binding of Isaac 0.029479151591658592\n",
      "Rust 0.024058053269982338\n",
      "Portal 2 0.022767027840018272\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.0713915079832077\n",
      "Mount & Blade: Warband 0.02813238650560379\n",
      "Warhammer\\xae 40,000: Dawn of War\\xae - Dark Crusade 0.025701187551021576\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.008262219838798046\n",
      "Softmax + Mi:  0.008992349728941917\n",
      "Softmax + CPR:100 + Mi:  0.02813238650560379\n",
      "\n",
      "------History------\n",
      "\n",
      "FTL: Faster Than Light\n",
      "Star Trek Online\n",
      "Endless Space\\xae - Collection\n",
      "Future Wars\n",
      "Rust\n",
      "\n",
      "---Ground truth----\n",
      "Mount & Blade: Warband\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Rust 0.4032537341117859\n",
      "Starbound 0.009269198402762413\n",
      "Kerbal Space Program 0.008252626284956932\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.035775791853666306\n",
      "No More Room in Hell 0.0279020294547081\n",
      "Portal 2 0.02489926479756832\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.06948374211788177\n",
      "Mount & Blade: Warband 0.029829025268554688\n",
      "ORION: Prelude 0.0239527877420187\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.006280767731368542\n",
      "Softmax + Mi:  0.00864835362881422\n",
      "Softmax + CPR:100 + Mi:  0.029829025268554688\n",
      "\n",
      "------History------\n",
      "\n",
      "STAR WARS\\u2122 - Knights of the Old Republic\\u2122\n",
      "Rust\n",
      "Outlast\n",
      "Outlast\n",
      "\n",
      "---Ground truth----\n",
      "Chivalry: Medieval Warfare\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Outlast 0.1879294216632843\n",
      "Borderlands 2 0.016865048557519913\n",
      "DARK SOULS\\u2122 II 0.011648684740066528\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03780833259224892\n",
      "No More Room in Hell 0.030636504292488098\n",
      "Portal 2 0.026706958189606667\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.07624580711126328\n",
      "Mount & Blade: Warband 0.038998547941446304\n",
      "Just Cause 2 0.022016091272234917\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.00864802673459053\n",
      "Softmax + Mi:  0.004916117526590824\n",
      "Softmax + CPR:100 + Mi:  0.07624580711126328\n",
      "\n",
      "------History------\n",
      "\n",
      "Counter-Strike: Source\n",
      "Saints Row: The Third\n",
      "SMITE\\xae\n",
      "Battlefield: Bad Company\\u2122 2\n",
      "Heroes & Generals\n",
      "\n",
      "---Ground truth----\n",
      "A.V.A. Alliance of Valiant Arms\\u2122\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Heroes & Generals 0.08041176199913025\n",
      "Team Fortress 2 0.017381086945533752\n",
      "Paladins\\xae 0.01694086380302906\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Battlefield: Bad Company\\u2122 2 0.01826363429427147\n",
      "Portal 2 0.018112879246473312\n",
      "Amnesia: The Dark Descent 0.015477348119020462\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Trove 0.049107618629932404\n",
      "Warfare Online 0.04111190512776375\n",
      "A.V.A. Alliance of Valiant Arms\\u2122 0.029287705197930336\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.001506573287770152\n",
      "Softmax + Mi:  0.002149666426703334\n",
      "Softmax + CPR:100 + Mi:  0.029287705197930336\n",
      "\n",
      "------History------\n",
      "\n",
      "Thief\\u2122 Gold\n",
      "Metro: Last Light Redux\n",
      "Rust\n",
      "Rust\n",
      "Wolfenstein: The New Order\n",
      "\n",
      "---Ground truth----\n",
      "Hitman: Absolution\\u2122\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Wolfenstein: The New Order 0.20742371678352356\n",
      "Fallout 4 0.016559462994337082\n",
      "DayZ 0.010794132016599178\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03133821114897728\n",
      "Portal 2 0.027726417407393456\n",
      "No More Room in Hell 0.02541544660925865\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.08745817840099335\n",
      "Zombie Panic! Source 0.04502014443278313\n",
      "Hitman: Absolution\\u2122 0.03411945700645447\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.0016036008019000292\n",
      "Softmax + Mi:  0.001481199054978788\n",
      "Softmax + CPR:100 + Mi:  0.03411945700645447\n",
      "\n",
      "------History------\n",
      "\n",
      "BLOCKADE 3D\n",
      "No More Room in Hell\n",
      "Warframe\n",
      "Warframe\n",
      "Tree of Savior (English Ver.)\n",
      "\n",
      "---Ground truth----\n",
      "Trove\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Tree of Savior (English Ver.) 0.03906697407364845\n",
      "Team Fortress 2 0.025597726926207542\n",
      "Brawlhalla 0.019648604094982147\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Evolve Stage 2 0.0603332445025444\n",
      "No More Room in Hell 0.051357343792915344\n",
      "How to Survive 2 0.026445530354976654\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Trove 0.03336038812994957\n",
      "Robocraft 0.028041761368513107\n",
      "Warframe 0.02242172509431839\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.018684998154640198\n",
      "Softmax + Mi:  0.0013563764514401555\n",
      "Softmax + CPR:100 + Mi:  0.03336038812994957\n",
      "\n",
      "------History------\n",
      "\n",
      "u\"Tom Clancys Splinter Cell Chaos Theory\\xae\"\n",
      "No More Room in Hell\n",
      "Infinity Wars: Animated Trading Card Game\n",
      "Emily is Away\n",
      "Town of Salem\n",
      "\n",
      "---Ground truth----\n",
      "Trove\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Town of Salem 0.3000623881816864\n",
      "Undertale 0.017627369612455368\n",
      "Team Fortress 2 0.01171074528247118\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "No More Room in Hell 0.07254170626401901\n",
      "Alien Swarm 0.03330584242939949\n",
      "Team Fortress 2 0.030175862833857536\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Trove 0.03593482822179794\n",
      "Chivalry: Medieval Warfare 0.029892481863498688\n",
      "Codename CURE 0.02763531729578972\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.0040516965091228485\n",
      "Softmax + Mi:  0.0008827141718938947\n",
      "Softmax + CPR:100 + Mi:  0.03593482822179794\n",
      "\n",
      "------History------\n",
      "\n",
      "Arma II: DayZ Mod\n",
      "APB Reloaded\n",
      "Rust\n",
      "Rust\n",
      "\n",
      "---Ground truth----\n",
      "STAR WARS\\u2122 - Knights of the Old Republic\\u2122\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Rust 0.24427735805511475\n",
      "DayZ 0.017162153497338295\n",
      "Heroes & Generals 0.014561383053660393\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03692203387618065\n",
      "No More Room in Hell 0.029166454449295998\n",
      "Portal 2 0.024959206581115723\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.07462680339813232\n",
      "Mount & Blade: Warband 0.038732655346393585\n",
      "STAR WARS\\u2122 - Knights of the Old Republic\\u2122 0.022138562053442\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.0003085133503191173\n",
      "Softmax + Mi:  0.0033967201597988605\n",
      "Softmax + CPR:100 + Mi:  0.022138562053442\n",
      "\n",
      "------History------\n",
      "\n",
      "Stranded Deep\n",
      "Reign Of Kings\n",
      "Amnesia: The Dark Descent\n",
      "Life is Feudal: Your Own\n",
      "\n",
      "---Ground truth----\n",
      "Team Fortress 2\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Life is Feudal: Your Own 0.06234632432460785\n",
      "DayZ 0.0335712805390358\n",
      "Arma 3 0.02132670022547245\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Amnesia: The Dark Descent 0.13170592486858368\n",
      "RIFT 0.06932537257671356\n",
      "Pirates, Vikings, and Knights II 0.024707458913326263\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.030819937586784363\n",
      "Team Fortress 2 0.01992524415254593\n",
      "Emily is Away 0.018582915887236595\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.008011844009160995\n",
      "Softmax + Mi:  0.012526382692158222\n",
      "Softmax + CPR:100 + Mi:  0.01992524415254593\n",
      "\n",
      "------History------\n",
      "\n",
      "u\"Americas Army: Proving Grounds\"\n",
      "Heroes & Generals\n",
      "Heroes & Generals\n",
      "No More Room in Hell\n",
      "\n",
      "---Ground truth----\n",
      "Chivalry: Medieval Warfare\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "No More Room in Hell 0.0950203463435173\n",
      "Warface 0.031175430864095688\n",
      "Team Fortress 2 0.026352088898420334\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Rust 0.03168245777487755\n",
      "Portal 2 0.02696983888745308\n",
      "No More Room in Hell 0.026340555399656296\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.08732341974973679\n",
      "Hitman: Absolution\\u2122 0.04965653270483017\n",
      "Overgrowth 0.03582070767879486\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.007760111708194017\n",
      "Softmax + Mi:  0.005187626462429762\n",
      "Softmax + CPR:100 + Mi:  0.08732341974973679\n",
      "\n",
      "------History------\n",
      "\n",
      "Tree of Savior (English Ver.)\n",
      "Fractured Space\n",
      "Evolve Stage 2\n",
      "PlanetSide 2\n",
      "\n",
      "---Ground truth----\n",
      "Robocraft\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "PlanetSide 2 0.03588074445724487\n",
      "Warframe 0.023051710799336433\n",
      "Team Fortress 2 0.01407952792942524\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Red Trigger 0.07086466997861862\n",
      "Evolve Stage 2 0.033268604427576065\n",
      "Princess Remedy in a World of Hurt 0.02074761688709259\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Robocraft 0.03857482969760895\n",
      "Warfare Online 0.03495484218001366\n",
      "Trove 0.03325923532247543\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.013944796286523342\n",
      "Softmax + Mi:  0.0015399284893646836\n",
      "Softmax + CPR:100 + Mi:  0.03857482969760895\n",
      "\n",
      "------History------\n",
      "\n",
      "Saints Row: The Third\n",
      "The Stanley Parable\n",
      "Warframe\n",
      "Spiral Knights\n",
      "\n",
      "---Ground truth----\n",
      "Trove\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Spiral Knights 0.09065191447734833\n",
      "Team Fortress 2 0.03676477074623108\n",
      "Undertale 0.02714509330689907\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "Amnesia: The Dark Descent 0.047815952450037\n",
      "No More Room in Hell 0.026047054678201675\n",
      "Saints Row: The Third 0.023226367309689522\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Trove 0.062408097088336945\n",
      "Warfare Online 0.03766673430800438\n",
      "A.V.A. Alliance of Valiant Arms\\u2122 0.02682037465274334\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.012678400613367558\n",
      "Softmax + Mi:  0.00042489689076319337\n",
      "Softmax + CPR:100 + Mi:  0.062408097088336945\n",
      "\n",
      "------History------\n",
      "\n",
      "Dead Rising 3 Apocalypse Edition\n",
      "Fallout 2: A Post Nuclear Role Playing Game\n",
      "Banished\n",
      "Banished\n",
      "Borderlands 2\n",
      "\n",
      "---Ground truth----\n",
      "Chivalry: Medieval Warfare\n",
      "\n",
      "---RepeatNet top 3 suggestions---\n",
      "Borderlands 2 0.3846040666103363\n",
      "Fallout 4 0.01091169286519289\n",
      "DayZ 0.010519731789827347\n",
      "\n",
      "---Softmax + Mi top 3 suggestions---\n",
      "The Binding of Isaac 0.01860935427248478\n",
      "Borderlands 2 0.014301617629826069\n",
      "Team Fortress 2 0.01260523684322834\n",
      "\n",
      "---Softmax + CPR:100 + Mi top 3 suggestions---\n",
      "Chivalry: Medieval Warfare 0.07249713689088821\n",
      "Commandos 2: Men of Courage 0.04429369419813156\n",
      "Hitman: Absolution\\u2122 0.0419779047369957\n",
      "\n",
      "---Ground truth scores---\n",
      "\n",
      "RepeatNet:  0.003436340019106865\n",
      "Softmax + Mi:  0.006980796344578266\n",
      "Softmax + CPR:100 + Mi:  0.07249713689088821\n"
     ]
    }
   ],
   "source": [
    "for i, j in final_example_indexes:\n",
    "    interactions_rn = interactions_rn_array[i]\n",
    "    print(\"\\n------History------\\n\")\n",
    "    id_to_token_map = dataset_rn.field2id_token['product_id']\n",
    "    for k in range(50):\n",
    "        token = id_to_token_map[interactions_rn['product_id_list'][j][k]]\n",
    "        if token == '[PAD]':\n",
    "            break\n",
    "        info = token_to_info[token]\n",
    "        app_name = info[0]\n",
    "        print(app_name)\n",
    "    rn_top_3_predictions = scores_rn_array[i][j].sort(dim=0, descending=True)\n",
    "    rn_top3_predictions_indices = rn_top_3_predictions.indices[:3]\n",
    "    rn_top3_predictions_scores = rn_top_3_predictions.values[:3]\n",
    "    smi_top_3_predictions = scores_smi_array[i][j].sort(dim=0, descending=True)\n",
    "    smi_top3_predictions_indices = smi_top_3_predictions.indices[:3]\n",
    "    smi_top3_predictions_scores = smi_top_3_predictions.values[:3]\n",
    "    scmi_top3_predictions = scores_scmi_array[i][j].sort(dim=0, descending=True)\n",
    "    scmi_top3_predictions_indices = scmi_top3_predictions.indices[:3]\n",
    "    scmi_top3_predictions_scores = scmi_top3_predictions.values[:3]\n",
    "    print(\"\\n---Ground truth----\")\n",
    "    print(token_to_info[id_to_token_map[interactions_rn['product_id'][j]]][0])\n",
    "    print(\"\\n---RepeatNet top 3 suggestions---\")\n",
    "    for k in range(3):\n",
    "        print(token_to_info[id_to_token_map[rn_top3_predictions_indices[k]]][0], rn_top3_predictions_scores[k].item())\n",
    "    print(\"\\n---Softmax + Mi top 3 suggestions---\")\n",
    "    for k in range(3):\n",
    "        print(token_to_info[id_to_token_map[smi_top3_predictions_indices[k]]][0], smi_top3_predictions_scores[k].item())\n",
    "    print(\"\\n---Softmax + CPR:100 + Mi top 3 suggestions---\")\n",
    "    for k in range(3):\n",
    "        print(token_to_info[id_to_token_map[scmi_top3_predictions_indices[k]]][0], scmi_top3_predictions_scores[k].item())    \n",
    "    print(\"\\n---Ground truth scores---\")\n",
    "    print(\"\\nRepeatNet: \", scores_rn_array[i][j][interactions_rn['product_id'][j]].item())\n",
    "    print(\"Softmax + Mi: \", scores_smi_array[i][j][interactions_rn['product_id'][j]].item())\n",
    "    print(\"Softmax + CPR:100 + Mi: \", scores_scmi_array[i][j][interactions_rn['product_id'][j]].item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_inputs(identifiers, token_to_info, id_to_token_map):\n",
    "    results = PrettyTable([\"s/no.\", \"token\", \"app\", \"dev\", \"early_access\", \"genres\", \"metascore\", \"price\", \"publisher\", \"sentiment\", \"specs\", \"tags\"])\n",
    "    results._max_width = {\"s/no.\" : 4, \"token\" : 4, \"app\": 4, \"dev\": 4, \"early\": 4, \"genres\": 5, \"meta\": 3, \"price\": 4, \"publisher\": 4, \"sentiment\": 4, \"specs\": 5, \"tags\":5}\n",
    "    for identifier in identifiers:\n",
    "        token = id_to_token_map[identifier]\n",
    "        info = token_to_info[token]\n",
    "        row = ['N/A']*12\n",
    "        row[0] = identifier\n",
    "        row[1] = token\n",
    "        if len(info) >= 1:\n",
    "                row[2] = info[0]\n",
    "        if len(info) >= 2:\n",
    "            row[3] = info[1]\n",
    "        if len(info) >= 3:\n",
    "            row[4] = info[2]\n",
    "        if len(info) >= 4:\n",
    "            row[5] = info[3]\n",
    "        if len(info) >= 5:\n",
    "            row[6] = info[4]\n",
    "        if len(info) >= 6:\n",
    "            row[7] = info[5]\n",
    "        if len(info) >= 7:\n",
    "            row[8] = info[6]\n",
    "        if len(info) >= 8:\n",
    "            row[9] = info[7]\n",
    "        if len(info) >= 9:\n",
    "            row[10] = info[8]\n",
    "        if len(info) >= 10:\n",
    "            row[11] = info[9]\n",
    "        results.add_row(row)\n",
    "    print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:anaconda3-nikhil]",
   "language": "python",
   "name": "conda-env-anaconda3-nikhil-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}