result_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Result Analysis\n",
    "Use this notebook to analyse the results of a KG evaluation with KGrEaT. In the tables below you can find an overview of the performance metrics per KG as well as mapping coverage per dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Merge Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify the paths to the results of the KGs here\n",
    "\n",
    "PATH_TO_HR_RESULTS = {\n",
    "    '0_DBpedia2016': './kg/dbpedia2016/result/run_HR',\n",
    "    '1_DBpedia2022': './kg/dbpedia2022/result/run_HR',\n",
    "    '2_YAGO': './kg/yago3/result/run_HR',\n",
    "    '3_Wikidata': './kg/wikidata/result/run_HR',\n",
    "    '4_CaLiGraph': './kg/caligraph/result/run_HR',\n",
    "    '5_DbkWik': './kg/dbkwik/result/run_HR',\n",
    "}\n",
    "\n",
    "PATH_TO_HP_RESULTS = {\n",
    "    '0_DBpedia2016': './kg/dbpedia2016/result/run_HP',\n",
    "    '1_DBpedia2022': './kg/dbpedia2022/result/run_HP',\n",
    "    '2_YAGO': './kg/yago3/result/run_HP',\n",
    "    '3_Wikidata': './kg/wikidata/result/run_HP',\n",
    "    '4_CaLiGraph': './kg/caligraph/result/run_HP',\n",
    "    '5_DbkWik': './kg/dbkwik/result/run_HP',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "def load_kg_results(kg_name: str, path_to_results: str) -> pd.DataFrame:\n",
    "    path_to_results = Path(path_to_results)\n",
    "    df = pd.concat([pd.read_csv(f, sep='\\t') for f in path_to_results.rglob('*.tsv')]).reset_index(drop=True)\n",
    "    df['kg_name'] = kg_name\n",
    "    return df\n",
    "\n",
    "def merge_kg_results(path_to_results) -> pd.DataFrame:\n",
    "    return pd.concat([load_kg_results(kg_name, result_path) for kg_name, result_path in path_to_results.items() if Path(result_path).exists()])\n",
    "\n",
    "df_hp = merge_kg_results(PATH_TO_HP_RESULTS)\n",
    "df_hp['eval_mode'] = df_hp['eval_mode'].map({'ALL': '1_PA', 'KNOWN': '0_PK'})\n",
    "df_hr = merge_kg_results(PATH_TO_HR_RESULTS)\n",
    "df_hr = df_hr[df_hr['eval_mode'] == 'ALL']\n",
    "df_hr['eval_mode'] = '2_R'\n",
    "df = pd.concat([df_hr, df_hp])\n",
    "df['entity_frac_known'] = 1 - (df['entities_missing'] / df['entities_total'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raw Result Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results Aggregated on Task Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dfs = df.sort_values(['kg_name', 'eval_mode'])\n",
    "pt = dfs.pivot_table(columns=['kg_name', 'eval_mode'], index=['task_type', 'metric'], values='score', sort=False)\n",
    "pt.round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Coverage of datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = df.sort_values(['kg_name', 'eval_mode'])\n",
    "pt = dfs.pivot_table(columns=['kg_name', 'eval_mode'], index=['task_type', 'dataset'], values='entity_frac_known', sort=False)\n",
    "pt.round(2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kgreat",
   "language": "python",
   "name": "kgreat"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}