2. Pré processamento.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pré processamento\n",
    "Agora que filtramos nossos dados, queremos adicionar colunas que podem ser inferidas através dos atributos originais."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Arquivos ainda não processados\n",
    "\n",
    "users = pd.read_csv('original_data/users_cleaned.csv')\n",
    "animes = pd.read_csv('original_data/anime_cleaned.csv')\n",
    "user_animes = pd.read_csv('manipulated_data/user_anime.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adicionando coluna de idade nos usuários\n",
    "users['age'] = users['birth_date'].apply(lambda x: 2018 - int(x.split(\"-\")[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Criando dicionário de países e regiões\n",
    "\n",
    "with open('external_data/countries.json') as f:\n",
    "    data = json.load(f)\n",
    "names = []\n",
    "for i in data:\n",
    "    names.append(i['name'])\n",
    "dictnames = dict();\n",
    "for i in names:\n",
    "    dictnames[i['common']] = [i['common']]\n",
    "    dictnames[i['common']].append(i['official'])\n",
    "    for n in [i['native']]:\n",
    "        for j in n:\n",
    "            dictnames[i['common']].append(n[j]['common'])\n",
    "            dictnames[i['common']].append(n[j]['official'])\n",
    "alternativeus = ['US', 'USA', 'U.S.', 'U.S', 'U.S.A.', 'America']\n",
    "for i in alternativeus:\n",
    "    dictnames['United States'].append(i)\n",
    "dictnames['United Kingdom'].append('UK')\n",
    "dictnames['United Kingdom'].append('England')\n",
    "dictnames['Canada'].append('Montreal')\n",
    "dictnames['Canada'].append('Montréal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adicionando estados americanos no dicionário\n",
    "\n",
    "states = pd.read_csv('external_data/states.csv')\n",
    "states = states.to_dict(\"records\")\n",
    "for i in states:\n",
    "    dictnames['United States'].append(i['State'])\n",
    "    dictnames['United States'].append(i['Abbreviation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# Adicionando coluna de regiões, limpando elementos não desejados\n",
    "\n",
    "def findCountry(x):\n",
    "    global c\n",
    "    c = c + 1\n",
    "    if not isinstance(x, int) and not isinstance(x, float):\n",
    "        for i in dictnames:\n",
    "            for j in dictnames[i]:\n",
    "                if j in x: return i\n",
    "        return \"no data\"\n",
    "    else: return \"no data\"\n",
    "\n",
    "total = len(users.index)\n",
    "c = 0\n",
    "users['country'] = users['location'].apply(lambda x: findCountry(x))\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Realizando merge nas tabelas User, Anime e UserAnime\n",
    "merged = users.merge(user_animes, left_on='username', right_on='username', how='inner')\n",
    "merged = merged.merge(animes, left_on='anime_id', right_on='anime_id', how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adicionanod coluna de porcentagem de completude usuário-anime\n",
    "merged['my_completion'] = round((merged['my_watched_episodes']/merged['episodes']),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transformando dados pré-processados em CSV para leitura em outros notebooks\n",
    "users.to_csv('manipulated_data/users.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}