new table

Author: heitor57

algorithms/lib/constants.py

@@ -23,7 +23,7 @@ class usg_constants:
 METRICS_PRETTY = {'precision':'Prec',
                   'recall':'Rec',
                   'ild': 'ILD',
-                  'gc': 'GC',
+                  'gc': 'Cov',
                   'pr':'PRg',
                   'epc':'EPC',
                   'ndcg':'NDCG',
@@ -40,7 +40,8 @@ class usg_constants:
     # "geocat": "DisCovER",
     "geocat": "DisCovER",
     "persongeocat": "PersonDisCovER",
-    "geodiv": "Geo-Div(PR)",
+    # "geodiv": "Geo-Div(PR)",
+    "geodiv": "GeoDiv",
     "ld": "LD",
     "binomial": "Binom",
     "pm2": "PM2",

algorithms/lib/utils.py

@@ -29,8 +29,8 @@ class StatisticResult(Enum):
 def statistic_test(x, y, p):
     # try:
     statistic, pvalue = scipy.stats.ttest_ind(x, y)
-    y_mean = np.mean(y)
     x_mean = np.mean(x)
+    y_mean = np.mean(y)
     if pvalue < p:
         if x_mean > y_mean:
             return StatisticResult.GAIN

algorithms/print_table.py

@@ -1,23 +1,38 @@
 
+import re
 from numpy.core import numeric
+from numpy.core.arrayprint import printoptions
 from lib.utils import StatisticResult, statistic_test
 import scipy.stats
+# import argparse
 from collections import defaultdict
 import pandas as pd
 import sys, os
 from typing import final
 sys.path.insert(0, os.path.abspath('lib'))
 from lib.RecRunner import NameType, RecRunner
-# rr=RecRunner("usg","geocat","madison",80,20,"../data")
-# print(rr.get_base_rec_file_name())
-# print(rr.get_final_rec_file_name())
-
-# rr.load_base()
-# rr.run_base_recommender()
-# rr.run_final_recommender()
 import inquirer
 from lib.constants import METRICS_PRETTY, RECS_PRETTY, experiment_constants, CITIES_PRETTY
 
+LATEX_HEADER = r"""\documentclass{article}
+\usepackage{graphicx}
+\usepackage[utf8]{inputenc}
+\usepackage{xcolor}
+\usepackage{amsmath}
+\usepackage{amssymb}
+\usepackage{underscore}
+\usepackage[margin=0.5in]{geometry}
+\usepackage{booktabs}
+\begin{document}
+"""
+
+LATEX_FOOT = r"""
+\end{document}"""
+
+# argparser = argparse.ArgumentParser()
+# argparser.add_argument('-f')
+# args= argparser.parse_args()
+
 # questions = [
 #   inquirer.Checkbox('city',
 #                     message="City to use",
@@ -40,94 +55,148 @@
 
 # cities = ['lasvegas', 'phoenix']
 cities = ['lasvegas']
-base_recs = ['geomf', 'usg']
-# base_recs = ['geomf']
-final_recs = ['geocat']
-# print(cities)
-# print(base_recs)
-# print(final_recs)
+# base_recs = [ 'usg','geosoca','geomf',]
+base_recs = ['geomf']
+final_recs = ['geodiv']
 final_rec_list_size = 20
 rr=RecRunner(base_recs[0],final_recs[0],cities[0],80,final_rec_list_size,"../data")
 
-# rr.print_latex_vert_cities_metrics_table(cities=city)
 metrics_k = experiment_constants.METRICS_K
 final_recs_metrics= defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict())))
 base_recs_metrics= defaultdict(lambda: defaultdict(lambda: defaultdict()))
 latex_table = ""
-num_metrics = None
+main_metrics = ['precision','recall','gc','ild','pr','epc']
+def get_metrics_renamed_order(METRICS_PRETTY_k):
+    METRICS_PRETTY_k = [METRICS_PRETTY_k[v] for v in main_metrics]
+    return METRICS_PRETTY_k
+def get_metrics_pretty_k(metric_k:int) -> dict:
+    return {k: v+f'@{metric_k}' for k, v in METRICS_PRETTY.items()}
+def df_format(df_unf:pd.DataFrame,metric_k):
+    # METRICS_PRETTY_k = {k: v+f'@{metric_k}' for k, v in METRICS_PRETTY.items()}
+    METRICS_PRETTY_k = get_metrics_pretty_k(metric_k)
+    df_unf=df_unf[main_metrics].rename(columns=METRICS_PRETTY_k)
+    # print(df_unf)
+    return df_unf
+
+def get_base_name(base_name):
+  return RECS_PRETTY[base_name]
+
+def get_final_name(base_name,final_name):
+  return get_base_name(base_name)+'+'+RECS_PRETTY[final_name]
+  
 for city in cities:
   for base_rec in base_recs:
     rr.city = city
     rr.base_rec = base_rec
     metrics = rr.load_metrics(
         base=True, name_type=NameType.PRETTY, METRICS_KS=metrics_k)
     for metric_k in metrics_k:
-      METRICS_PRETTY_k = {k:v+f'@{metric_k}' for k,v in METRICS_PRETTY.items()}
       base_recs_metrics[city][base_rec][metric_k] = pd.DataFrame(
-          metrics[metric_k]).rename(columns=METRICS_PRETTY_k)
+          metrics[metric_k])
+      base_recs_metrics[city][base_rec][metric_k]=df_format(base_recs_metrics[city][base_rec][metric_k],metric_k)
+      # base_recs_metrics[city][base_rec][metric_k]=base_recs_metrics[city][base_rec][metric_k].rename(columns=METRICS_PRETTY_k)
   # rr.final_rec_list_size = final_rec_list_size
 
     for final_rec in final_recs:
       rr.final_rec = final_rec
       metrics = rr.load_metrics(
           base=False, name_type=NameType.PRETTY, METRICS_KS=metrics_k)
       for metric_k in metrics_k:
-        METRICS_PRETTY_k = {k: v+f'@{metric_k}' for k, v in METRICS_PRETTY.items()}
         final_recs_metrics[city][base_rec][final_rec][metric_k] = pd.DataFrame(
-            metrics[metric_k]).rename(columns=METRICS_PRETTY_k)
-        
+            metrics[metric_k])
+        final_recs_metrics[city][base_rec][final_rec][metric_k] =df_format(final_recs_metrics[city][base_rec][final_rec][metric_k],metric_k)
 num_metrics = 6
+num_columns= num_metrics+1
 latex_table_header= """
 \\begin{{tabular}}{{{}}}
-""".format('c'*num_metrics)
+""".format('c'*(num_columns))
 latex_table_footer= r"""
 \end{tabular}
 """
-for city in cities:
-  latex_table += r'\toprule\n'
-  latex_table += '\\multicolumn{{{}}}{{l}}{}\\\\\n'.format(num_metrics,CITIES_PRETTY[city])
-  latex_table += r'\bottomrule\n'
+
+top_count = defaultdict(lambda:defaultdict(int))
+for count1, city in enumerate(cities):
+  if count1 == 0:
+    latex_table += '\\toprule\n'
+  latex_table += '\\multicolumn{{{}}}{{l}}{{{}}}\\\\\n'.format((num_columns),CITIES_PRETTY[city])
+  latex_table += '\\bottomrule\n'
   for metric_k in metrics_k:
     dfs = []
+    names_recs_in_order = []
     for base_rec in base_recs:
       current_metrics = {}
-      current_metrics[RECS_PRETTY[base_rec]] = base_recs_metrics[city][base_rec][metric_k].drop(columns='user_id')
+      current_metrics[get_base_name(base_rec)] = base_recs_metrics[city][base_rec][metric_k]
+      names_recs_in_order.append(get_base_name(base_rec))
       for final_rec in final_recs:
-        current_metrics[RECS_PRETTY[base_rec]+'+'+RECS_PRETTY[final_rec]] = final_recs_metrics[city][base_rec][final_rec][metric_k].drop(columns='user_id')
+        current_metrics[get_final_name(base_rec,final_rec)] = final_recs_metrics[city][base_rec][final_rec][metric_k]
+        names_recs_in_order.append(get_final_name(base_rec,final_rec))
       df = pd.concat(current_metrics, axis=1)
+      # print(df)
       dfs.append(df)
-    df = pd.concat(dfs)
+    df = pd.concat(dfs,axis=1)
+    # print(df)
 
     df_reordered = df.reorder_levels([1,0],axis=1)
+    # print(df_reordered)
+    df_reordered_means = df_reordered.mean()
     top_methods = df_reordered.mean().reset_index().set_index('level_1').groupby('level_0').idxmax().to_dict()[0]
     df_top2_methods = df_reordered.copy()
-
-    print(df_top2_methods)
     for k, v in top_methods.items():
-      print(k,v)
       df_top2_methods=df_top2_methods.drop((k,v),axis=1)
     top2_methods = df_top2_methods.mean().reset_index().set_index('level_1').groupby('level_0').idxmax().to_dict()[0]
-    print(top_methods)
-    print(top2_methods)
     highlight_elements = defaultdict(list)
+    metrics_og_name = {v: k for k,v in get_metrics_pretty_k(metric_k).items()}
+    # print(metrics_og_name)
+    names_recs_to_og = {}
+    names_recs_to_og[get_base_name(base_rec)] = base_rec
+    for final_rec in final_recs:
+      names_recs_to_og[get_final_name(base_rec,final_rec)] = (base_rec,final_rec)
     for k,v in top_methods.items():
       top1_values = df_reordered[k,v]
       v_top2 = top2_methods[k]
       top2_values =df_reordered[k,v_top2]
       statistic_result= statistic_test(top1_values,top2_values,0.05)
-      if statistic_test == StatisticResult.GAIN:
+      if statistic_result == StatisticResult.GAIN:
         highlight_elements[k].extend([v])
-      elif statistic_test == StatisticResult.TIE:
-        highlight_elements[k].extend([v,top2_values])
-      else:
-        highlight_elements[k].extend([top2_values])
+      elif statistic_result == StatisticResult.TIE:
+        highlight_elements[k].extend([v,v_top2])
+      elif statistic_result == StatisticResult.LOSS:
+        highlight_elements[k].extend([v_top2])
+      for hige in highlight_elements[k]:
+          top_count[metrics_og_name[k]][names_recs_to_og[hige]] += 1
 
-    table_df_result = df_reordered.mean().unstack(level=0)
-    table_df_result_latex = table_df_result.to_latex(header=False)
+    
+    df_reordered_means=df_reordered_means.map(lambda x: f'{x:.4f}')
+    for metric, methods in highlight_elements.items():
+      for method in methods:
+        df_reordered_means.at[metric,method] = '\\textbf{{{}}}'.format(df_reordered_means.at[metric,method])
+    table_df_result : pd.DataFrame = df_reordered_means.unstack(level=0)
+    table_df_result=table_df_result[get_metrics_renamed_order(get_metrics_pretty_k(metric_k))]
+    table_df_result=table_df_result.reindex(names_recs_in_order)
+
+    table_df_result_latex = table_df_result.to_latex(header=True,escape=False)
+    table_df_result_latex=re.sub('\{\}','Algorithm',table_df_result_latex)
     table_df_result_latex = table_df_result_latex.split('\n')[:-2][2:]
     table_df_result_latex = '\n'.join(table_df_result_latex)
     latex_table+=table_df_result_latex+'\n'
     # raise SystemError
 
-latex_table = latex_table_header+latex_table+latex_table_footer
-print(latex_table)
+latex_table = LATEX_HEADER+latex_table_header+latex_table+latex_table_footer+LATEX_FOOT
+with open('../data/result/util/main_benchmark_table.tex','w') as f:
+  f.write(latex_table)
+
+
+table_top_count = pd.DataFrame.from_dict(top_count).fillna(0)
+table_top_count=table_top_count[main_metrics]
+table_top_count.columns = [METRICS_PRETTY[i] for i in table_top_count.columns]
+recs_order = []
+for base_rec in base_recs:
+    recs_order.append(base_rec)
+    for final_rec in final_recs:
+        recs_order.append((base_rec,final_rec))
+    
+table_top_count = table_top_count.reindex(recs_order)
+# for i in table_top_count.index:
+
+# table_df_result=table_df_result.reindex(names_recs_in_order)
+# table_df_result=table_df_result.reindex(names_recs_in_order)