pre-processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import os\n",
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option(\"max_rows\", 999)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 다양한 명칭으로 표기 된 식재료들을 하나의 이름으로 mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ingredients.csv는 server에도 올라가고 pre-processing에서도 사용됨\n",
    "mapping_csv = pd.read_csv('server/ingredients_mapping.csv', encoding='cp949', header=None).rename(columns = {0:'original', 1:'general'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original</th>\n",
       "      <th>general</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>가다랑어</td>\n",
       "      <td>가다랑어포</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>가쓰오</td>\n",
       "      <td>가다랑어포</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>가다랭이포</td>\n",
       "      <td>가다랑어포</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>가다랑어포</td>\n",
       "      <td>가다랑어포</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>가다랑이포</td>\n",
       "      <td>가다랑어포</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3632</th>\n",
       "      <td>무김치</td>\n",
       "      <td>김치</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3633</th>\n",
       "      <td>파김치</td>\n",
       "      <td>김치</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3634</th>\n",
       "      <td>신김치</td>\n",
       "      <td>김치</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3635</th>\n",
       "      <td>포기김치용</td>\n",
       "      <td>김치</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3636</th>\n",
       "      <td>볶음김치용</td>\n",
       "      <td>김치</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3637 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     original general\n",
       "0        가다랑어   가다랑어포\n",
       "1         가쓰오   가다랑어포\n",
       "2       가다랭이포   가다랑어포\n",
       "3       가다랑어포   가다랑어포\n",
       "4       가다랑이포   가다랑어포\n",
       "...       ...     ...\n",
       "3632      무김치      김치\n",
       "3633      파김치      김치\n",
       "3634      신김치      김치\n",
       "3635    포기김치용      김치\n",
       "3636    볶음김치용      김치\n",
       "\n",
       "[3637 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapping_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array(['가다랑어','포기김치용','무김치'])\n",
    "my_dict = dict( zip(mapping_csv.original, mapping_csv.general))\n",
    "\n",
    "def to_general_name(x):\n",
    "    return np.vectorize(my_dict.get)(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['가다랑어포', '김치', '김치'], dtype='<U5')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_general_name(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 크롤링 데이터 Concat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"./\"\n",
    "file_list = os.listdir(path)\n",
    "file_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_list = ['tbRecipe_ilsang_1.csv',\n",
    " 'tbRecipe_간식야식.csv',\n",
    " 'tbRecipe_다이어트.csv',\n",
    " 'tbRecipe_도시락영양식.csv',\n",
    " 'tbRecipe_명절_이유식_기타.csv',\n",
    " 'tbRecipe_손님접대.csv',\n",
    " 'tbRecipe_일상2.csv',\n",
    " 'tbRecipe_초스피드.csv',\n",
    " 'tbRecipe_해장_푸드스타일링.csv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "\n",
    "for csv in csv_list:\n",
    "    temp = pd.read_csv(csv)\n",
    "    df = pd.concat([df, temp], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 요리재료 제대로 안 긁힌거 날려버리기 ( 크롤링 잘못된 거) \n",
    "duplicated_row_indices = [] #중복된 애들\n",
    "\n",
    "for row_index in range(len(df)-1):\n",
    "      if df.loc[row_index, '재료'] == df.loc[row_index + 1, '재료']:\n",
    "            duplicated_row_indices.append(row_index + 1)\n",
    "\n",
    "duplicated_row_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy(deep=True)\n",
    "df_copy.loc[duplicated_row_indices, ['재료']] = np.nan\n",
    "df_copy = df_copy.dropna(axis = 0)\n",
    "df_copy = df_copy.reset_index(drop=True)\n",
    "df = df_copy.copy(deep = True)   # 다 날려버림"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.to_csv('tbRecipe_all.csv', index = False, encoding = 'utf-8-sig')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ingredient_count = df.groupby('재료')[['id']].count().sort_values(by = 'id', ascending = False)\n",
    "ingredient_count.head(150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 재료 column 데이터 타입 str -> list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_to_list(x):\n",
    "    return x.split(', ')\n",
    "\n",
    "df['재료개수'] = df['재료'].apply(str_to_list).apply(lambda x: len(x))\n",
    "df = df[lambda x: x['재료개수']>1]    # 재료가 하나인 애들/ 레시피마다 달라요 없애기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ingredient_per_recipe_list = df['재료'].apply(str_to_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 전처리 및 TF 벡터화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ingredient_per_recipe_list_converted = []\n",
    "\n",
    "# 괄호, 한글 아닌거 날리기\n",
    "for i in tqdm(df['재료']):\n",
    "    processed_ingredients = []\n",
    "    tmplst = i.split(\",\")\n",
    "    for tmpfood in tmplst:\n",
    "        tmp = re.findall(\"[가-힣]+\",tmpfood)\n",
    "        if len(tmp) == 0: continue\n",
    "        processed_ingredients.append(tmp[0])\n",
    "        np_processed_ingredients = np.array(processed_ingredients)\n",
    "        \n",
    "    ingredient_per_recipe_list_converted.append(np_processed_ingredients)\n",
    "    \n",
    "ingredient_per_recipe_list = ingredient_per_recipe_list_converted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ingredient_per_recipe_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 재료 개수 줄여버리기~\n",
    "df['재료_processed'] = ingredient_per_recipe_list\n",
    "ingredient_per_recipe_list = df['재료_processed'].apply(to_general_name).apply(lambda x: x[x != np.array('None')]).apply(lambda x: x[x != np.array(None)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_ingredient_list = []   #우리가 크롤링한 재료에 존재하는 모오든 재료\n",
    "\n",
    "for ingredient in ingredient_per_recipe_list:\n",
    "    total_ingredient_list.extend(ingredient)\n",
    "\n",
    "before_len = len(total_ingredient_list)\n",
    "total_ingredient_list = list(set(total_ingredient_list))\n",
    "after_len = len(total_ingredient_list)\n",
    "\n",
    "print(f'before:: {before_len}, after:: {after_len}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.reset_index(drop = True)  # loc으로 검색하려면 index가 초기화 되어야 함\n",
    "\n",
    "np_total_ingredient_list = np.array(total_ingredient_list)\n",
    "recipe_vector_dictionary = {}\n",
    "counter = 0\n",
    "\n",
    "for recipe_ingredients in ingredient_per_recipe_list:  #\n",
    "    \n",
    "    recipe_id = df.loc[counter, 'id']\n",
    "    \n",
    "    ingredients_location_list = []\n",
    "    recipe_vector = np.zeros(len(np_total_ingredient_list))\n",
    "    for ingredient in recipe_ingredients:\n",
    "        ingredient_location = np.where(np_total_ingredient_list == ingredient)[0][0]\n",
    "        ingredients_location_list.append(ingredient_location)\n",
    "    recipe_vector[ ingredients_location_list ] = 1\n",
    "    \n",
    "    recipe_vector_dictionary[ recipe_id ] = recipe_vector \n",
    "    \n",
    "    counter += 1\n",
    "#     if counter>12:\n",
    "#         break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vector = pd.DataFrame(recipe_vector_dictionary).T\n",
    "df_vector.columns = total_ingredient_list\n",
    "df_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cosine Similarity 계산 및 추천"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "musts = ['두부', '김치', '스팸']\n",
    "my_ingredient = ['대파', '마늘', '고추장', '콩', '라면스프', '햄', '계란']\n",
    "musts = to_general_name(musts)\n",
    "my_ingredient = to_general_name(my_ingredient)\n",
    "print(musts, my_ingredient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vector_test = df_vector.copy(deep = True)\n",
    "for must in musts:\n",
    "    df_vector_test = df_vector_test[lambda x: x[must] != 0]\n",
    "\n",
    "filter_indices = df_vector_test.index\n",
    "df_vector = df_vector.loc[filter_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_vector = np.zeros(len(np_total_ingredient_list))\n",
    "ingredients_location_list = []\n",
    "not_in_list = []\n",
    "for ingredient in my_ingredient:\n",
    "    try:\n",
    "        ingredient_location = np.where(np_total_ingredient_list == ingredient)[0][0]\n",
    "        ingredients_location_list.append(ingredient_location)\n",
    "    except:\n",
    "        not_in_list.append(ingredient)\n",
    "    my_vector[ ingredients_location_list ] = 1\n",
    "\n",
    "print(not_in_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = df_vector.index\n",
    "from scipy import spatial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_similarity(a, b):\n",
    "    return 1 - spatial.distance.cosine(a,b)\n",
    "\n",
    "idx = df_vector.index\n",
    "a = my_vector\n",
    "index_dictionary = {}\n",
    "\n",
    "for i in tqdm(idx):\n",
    "    b = np.array(df_vector.loc[i])\n",
    "    index_dictionary[i] = cosine_similarity(a,b)\n",
    "    \n",
    "\n",
    "df_vector_copy = df_vector.reset_index()\n",
    "df_vector_copy['similarity'] = df_vector_copy['index'].map(index_dictionary)\n",
    "df_vector_copy = df_vector_copy.set_index('index')\n",
    "df_vector_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vector_copy['similarity'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "similar_recipe = df_vector_copy[df_vector_copy.similarity.isin(\n",
    "    df_vector_copy[['similarity']].sort_values(by='similarity', ascending=False)['similarity'].head(12)) ]\n",
    "df_similar_recipe = df[df.id.isin(similar_recipe.index)].sort_values(by='조회수', ascending=False).head(12)\n",
    "df_similar_recipe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "similar_recipe_vector = df_vector_copy[df_vector_copy.index.isin(df_similar_recipe.id)]\n",
    "similar_recipe_vector = similar_recipe_vector.drop('similarity', axis = 1)\n",
    "similar_recipe_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_similar_recipe = df_similar_recipe.reset_index(drop=True)\n",
    "df_similar_recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_recipe = {}\n",
    "for x in range(12):\n",
    "    temp = df_similar_recipe.iloc[x,:]\n",
    "    key = temp.id\n",
    "    selected_recipe[int(key)] = {'제목': temp[2],'재료': temp[7],'소요시간': temp[5], '이미지url': temp[8], '레시피url': temp[9]}\n",
    "selected_recipe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 사용한 재료, 더 필요한 재료 확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_ingredient(similar_recipe_vector, my_vector):\n",
    "    \n",
    "    ingredient_list = np.array(similar_recipe_vector.columns)\n",
    "    ingredient_check = similar_recipe_vector - my_vector #1이면 없는 재료, -1이면 안쓰는 재료\n",
    "    \n",
    "    necessary_dictionary = {}\n",
    "    unused_dictionary = {}\n",
    "    \n",
    "    for index in ingredient_check.index:\n",
    "        ingredient_check_per_recipe = np.array(ingredient_check.loc[index])\n",
    "        \n",
    "        necessary_ingredient_indices = np.where(ingredient_check_per_recipe == 1)[0]\n",
    "        unused_ingredient_indices = np.where(ingredient_check_per_recipe == -1)[0]\n",
    "        \n",
    "        necessary_ingredient = list(ingredient_list[necessary_ingredient_indices])\n",
    "        unused_ingredient = list(ingredient_list[unused_ingredient_indices])\n",
    "        \n",
    "        necessary_dictionary[index] = necessary_ingredient\n",
    "        unused_dictionary[index] = unused_ingredient\n",
    "    \n",
    "    return necessary_dictionary, unused_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a, b= check_ingredient(similar_recipe_vector, my_vector)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}