Steam-Data-Cleaner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MYsXB9KAatGZ",
    "outputId": "807e2be8-1978-424d-fd0f-8b6f22ba0666"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: textblob in /usr/local/lib/python3.6/dist-packages (0.15.3)\n",
      "Requirement already satisfied: nltk>=3.1 in /usr/local/lib/python3.6/dist-packages (from textblob) (3.2.5)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk>=3.1->textblob) (1.15.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install textblob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "210DuWB0RGDr"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from textblob import TextBlob\n",
    "import random \n",
    "from random import seed, sample, randint\n",
    "from random import random\n",
    "import requests\n",
    "import json\n",
    "import time\n",
    "from datetime import datetime\n",
    "import traceback\n",
    "import json\n",
    "from google.colab import drive\n",
    "\n",
    "now = datetime.now()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accessing UnClean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RX7IMykfRNdu"
   },
   "outputs": [],
   "source": [
    "url_p1=\"https://raw.githubusercontent.com/junkuse/steam_data/main/games\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jIrhN962RWWC"
   },
   "outputs": [],
   "source": [
    "sets=[]\n",
    "for i in range(9):\n",
    "  sets.append(pd.read_csv(url_p1+str(i+1)+'.csv'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7QwjO5s2SQKR",
    "outputId": "6fddb886-43ba-4278-9220-fdba7529a705"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['url', 'name', 'categories', 'img_url', 'user_reviews', 'all_reviews',\n",
       "       'date', 'developer', 'publisher', 'price', 'requirements', 'popu_tags',\n",
       "       'desc', 'full_desc', 'pegi', 'pegi_url'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final = pd.concat(sets, ignore_index=True)\n",
    "final.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions for Cleaning Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rlGV768DkUhT"
   },
   "outputs": [],
   "source": [
    "def ret_price(stringg):\n",
    "  q2=\"\"\n",
    "  row=stringg\n",
    "  row=re.sub(r'[^\\x00-\\x7f]',r'', row)\n",
    "  if (row.find(\"$\")!=-1):\n",
    "      count=0\n",
    "      for i in row:\n",
    "          if(count==row.find(\"$\")+1 or count==row.find(\"$\")+2 or count==row.find(\"$\")+3 or count==row.find(\"$\")+4):\n",
    "              if i.isdigit():\n",
    "                  q2=q2+str(i)\n",
    "          count=count+1\n",
    "  else:\n",
    "      q2=\"free\"\n",
    "  return q2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4qLY6O7ltnTA"
   },
   "outputs": [],
   "source": [
    "def desc_getter(stringg):\n",
    "  desc=\"\"\n",
    "  d1={}\n",
    "  q2=\"\"\n",
    "  sort=\"\"\n",
    "  row=stringg\n",
    "  row=re.sub(r'[^\\x00-\\x7f]',r'', row)\n",
    "  if(len(row)>10):\n",
    "      if(row[11]=='G'):\n",
    "          sort=\"game\"\n",
    "      elif(row[11]=='C'):\n",
    "          sort=\"content\"\n",
    "      elif(row[11]=='b'):\n",
    "          sort=\"bundle\"\n",
    "  count=0\n",
    "  for k in row:\n",
    "      if(sort==\"game\" and count==15):\n",
    "            desc=desc+\" \"+k\n",
    "      elif(sort==\"content\" and count==18):\n",
    "          desc=desc+\" \"+k\n",
    "      else:\n",
    "          desc=desc+k\n",
    "      count=count+1\n",
    "  d1[\"sort\"]=sort\n",
    "  d1[\"desc\"]=desc\n",
    "  return(d1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bwHSfncTXy2v"
   },
   "outputs": [],
   "source": [
    "#sys req\n",
    "rows=final[:10]['requirements']\n",
    "def reqs_getter(stringg):\n",
    "  row=stringg\n",
    "  values=[]\n",
    "  values=([m.start() for m in re.finditer('Minimum:OS:', row)])+([m.start() for m in re.finditer('Recommended:OS:', row)])\n",
    "  values.sort()\n",
    "  j=0\n",
    "  i=0\n",
    "  d2={}\n",
    "  d3={}\n",
    "  d4={}\n",
    "  for value1 in values:\n",
    "      data=\"\"\n",
    "      d1={}\n",
    "      if(i!=len(values)-1):\n",
    "          value2=values[i+1]\n",
    "          for j in row[value1:value2]:\n",
    "              data=data+j\n",
    "      else:\n",
    "          for j in row[value1:len(row)]:\n",
    "              data=data+j\n",
    "      i=i+1\n",
    "      w1=data.find(\"Processor\")\n",
    "      w2=data.find(\"Memory\")\n",
    "      w3=data.find(\"Graphics\")\n",
    "      w4=data.find(\"Storage\")\n",
    "      w5=data.find(\"DirectX\")\n",
    "      w6=data.find(\"Minimum:OS:\")\n",
    "      w7=data.find(\"Recommended:OS:\")\n",
    "      count=0\n",
    "      processor=\"\"\n",
    "      memory=\"\"\n",
    "      graphics=\"\"\n",
    "      minimum_os=\"\"\n",
    "      recommended_os=\"\"\n",
    "      for d in data:\n",
    "          if(w2==-1):\n",
    "              if(count>w1+9 and count<w1+20 and w1!=-1):\n",
    "                  processor=processor+d\n",
    "          else:\n",
    "              if(count>w1+9 and count<w2 and w1!=-1):\n",
    "                  processor=processor+d\n",
    "          if(w3==-1):\n",
    "              if(count>w2+6 and count<w2+10 and w2!=-1):\n",
    "                  memory=memory+d\n",
    "          else:\n",
    "              if(count>w2+6 and count<w3 and w2!=-1):\n",
    "                  memory=memory+d\n",
    "          if(w5==-1):\n",
    "              if(count>w3+8 and count<w4 and w3!=-1):\n",
    "                  graphics=graphics+d\n",
    "          else:\n",
    "              if(count>w3+8 and count<w5 and w3!=-1):\n",
    "                  graphics=graphics+d\n",
    "          if(w6!=-1):\n",
    "              if(w1!=-1):\n",
    "                  if(count>w6+10 and count<w1):\n",
    "                      minimum_os=minimum_os+d\n",
    "              else:\n",
    "                  if(count>w6+10 and count<w6+20):\n",
    "                      minimum_os=minimum_os+d\n",
    "          else:\n",
    "              if(w1!=-1):\n",
    "                  if(count>w7+15 and count<w1):\n",
    "                      recommended_os=recommended_os+d\n",
    "              else:\n",
    "                  if(count>w7+11 and count<w7+20):\n",
    "                      recommended_os=recommended_os+d       \n",
    "          count=count+1\n",
    "      d1[\"processor\"]=processor\n",
    "      d1[\"memory\"]=memory\n",
    "      d1[\"graphics\"]=graphics\n",
    "      if(len(minimum_os)==0):\n",
    "          d1[\"os\"]=recommended_os\n",
    "          if(recommended_os.find(\"Windows\")!=-1):\n",
    "              d4[\"windows\"]=d1\n",
    "          elif(recommended_os.find(\"macOS\")!=-1 or recommended_os.find(\"Mac Os\")!=-1):\n",
    "              d4[\"macOS\"]=d1\n",
    "          else:\n",
    "              d4[\"linux\"]=d1\n",
    "      else:\n",
    "          d1[\"os\"]=minimum_os\n",
    "          if(minimum_os.find(\"Windows\")!=-1):\n",
    "              d2[\"windows\"]=d1\n",
    "          elif(minimum_os.find(\"macOS\")!=-1 or minimum_os.find(\"Mac Os\")!=-1):\n",
    "              d2[\"macOS\"]=d1\n",
    "          else:\n",
    "              d2[\"linux\"]=d1\n",
    "      d3[\"minimum\"]=d2\n",
    "      d3[\"recommended\"]=d4\n",
    "  return(d3)\n",
    "l1=[]\n",
    "for row in rows:\n",
    "\n",
    "    l1.append(reqs_getter(row))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning Data and Storing in Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 229
    },
    "id": "vkMUS5j5ZZ6k",
    "outputId": "1585b951-a1a1-485d-a299-7453b30995b4"
   },
   "outputs": [],
   "source": [
    "#dictionary maker\n",
    "final_dict=[]\n",
    "test_data=final\n",
    "pegii_list=[]\n",
    "pricee_list=[]\n",
    "game_id=[]\n",
    "nameee=[]\n",
    "for i in range(len(test_data['url'])):\n",
    "  temp_dict=dict({})\n",
    "  if (test_data['img_url'][i]!='-'):\n",
    "    temp_dict['img_url']=test_data['img_url'][i]\n",
    "\n",
    "  if (test_data['date'][i]!='-'):\n",
    "    temp_dict['date']=test_data['date'][i]\n",
    "\n",
    "  if (test_data['developer'][i]!='-'):\n",
    "    temp_dict['developer']=test_data['developer'][i]\n",
    "\n",
    "  if (test_data['publisher'][i]!='-'):\n",
    "    temp_dict['publisher']=test_data['publisher'][i]\n",
    "\n",
    "  if (test_data['pegi'][i]!='-'):\n",
    "    try:\n",
    "      asd=[re.sub(r\"([A-Z][a-z]+)([A-Z][a-z]+)\", r\"\\1 \\2\",i) for i in test_data['pegi'][i].split(' ')]\n",
    "      asd=' '.join(asd)\n",
    "      asd=asd.split(' ')\n",
    "      if (test_data['pegi_url'][i]!='-'):\n",
    "        temp_dict['pegi']={'pegi_url':test_data['pegi_url'][i], 'pegi_tags':asd, 'age_limit':final['pegi_url'][10].split('/')[-1].split('.')[0]}\n",
    "        pegii_list.append(final['pegi_url'][10].split('/')[-1].split('.')[0])\n",
    "      else:\n",
    "        temp_dict['pegi']={'pegi_tags':asd, 'age_limit':final['pegi_url'][10].split('/')[-1].split('.')[0]}\n",
    "        pegii_list.append(final['pegi_url'][10].split('/')[-1].split('.')[0])\n",
    "    except:\n",
    "      pass\n",
    "    \n",
    "\n",
    "  if (test_data['full_desc'][i]!='-'):\n",
    "    temp_dict['full_desc']=desc_getter(test_data['full_desc'][i])\n",
    "\n",
    "  if (test_data['requirements'][i]!='-'):\n",
    "    temp_dict['requirements']=reqs_getter(test_data['requirements'][i])\n",
    "\n",
    "  if (test_data['popu_tags'][i]!='-'):\n",
    "    asd=re.findall('[A-Z][^A-Z]*',test_data['popu_tags'][i])\n",
    "    asd=' '.join(asd)\n",
    "    asd=re.sub(r'\\b\\w{1,3}\\b', '',asd).split(' ')\n",
    "    while(\"\" in asd) : \n",
    "        asd.remove(\"\")\n",
    "    while(\"-\" in asd) : \n",
    "        asd.remove(\"-\")\n",
    "    temp_dict['popu_tags']=asd\n",
    "\n",
    "\n",
    "  if (test_data['price'][i] != '-'):\n",
    "    temp_dict['price']=ret_price(test_data['price'][i])\n",
    "    pricee_list.append(ret_price(test_data['price'][i]))\n",
    "  temp_dict['url_info']={'url':test_data['url'][i],'id':test_data['url'][i].split('/')[4],\n",
    "                        'type':test_data['url'][i].split('/')[3],\n",
    "                        'url_name': ' '.join((test_data['url'][i].split('/')[5]).split('_'))}\n",
    "  game_id.append(test_data['url'][i].split('/')[4])\n",
    "  nameee.append(' '.join((test_data['url'][i].split('/')[5]).split('_')))\n",
    "  if (test_data['name'][i]!='-'):\n",
    "    temp_dict['name']=test_data['name'][i]\n",
    "  if (test_data['categories'][i]!='-'):\n",
    "    try:\n",
    "      asd=[re.sub(r\"([A-Z][a-z]+)([A-Z][a-z]+)\", r\"\\1 \\2\",i) for i in test_data['categories'][i].split(' ')]\n",
    "      yy=[]\n",
    "      for i in range(0,30,2):\n",
    "        try:\n",
    "          yy.append(asd[i]+' '+asd[i+1])\n",
    "        except:\n",
    "            pass\n",
    "      temp_dict['categories']=yy\n",
    "    except:\n",
    "      pass\n",
    "    final_dict.append(temp_dict)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exporting Dictionary as JSON and storing it in Google Drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4z-HfTSqLnXP"
   },
   "outputs": [],
   "source": [
    "listt=final_dict\n",
    "with open('game_review_data.json', 'w') as outfile:\n",
    "    json.dump(listt, outfile)\n",
    "drive.mount('/gdrive', force_remount=True)\n",
    "with open('/gdrive/My Drive/game_review_data.json', 'w') as outfile:\n",
    "  json.dump(listt, outfile)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Steam csv to json.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}