Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "59c3cb27",
   "metadata": {},
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "89333041-d172-4d95-b9b3-f2867ae037a1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "from pathlib import Path\n",
    "import torch\n",
    "sys.path.append(os.getcwd())\n",
    "sys.path.append(str(Path(os.getcwd()).parent))\n",
    "from utils import *\n",
    "from configuration import *\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "if not os.path.exists('./evaluation_date/'):\n",
    "    os.makedirs('./evaluation_date/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f071f4ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "args = parser.parse_args([])\n",
    "args.DEVICE = 'cpu'\n",
    "args.task = 'temporal'\n",
    "args.metric = 'temporal_acc'\n",
    "args.SoftEva = True\n",
    "args = FormulateArgs(args)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb529a4e",
   "metadata": {},
   "source": [
    "## Results on non-augmented test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa97109f",
   "metadata": {},
   "outputs": [],
   "source": [
    "args.augment = False\n",
    "args.augment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0ace529",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = torch.zeros([15])\n",
    "\n",
    "for ds in range(15):\n",
    "        args.DATASET = ds\n",
    "        seed = args.SEED\n",
    "        test_loader , datainfo = GetDataLoader(args, 'test',  path='../dataset/')\n",
    "        print(datainfo)\n",
    "\n",
    "        modelname = f\"pLF_data_{ds:02d}_{datainfo['dataname']}_seed_{seed:02d}.model\"\n",
    "        \n",
    "        model_exist = os.path.isfile(f'./models/{modelname}')\n",
    "        \n",
    "        if model_exist:  \n",
    "                          \n",
    "            model = torch.load(f'./models/{modelname}', map_location=args.DEVICE)\n",
    "            \n",
    "            model.UpdateArgs(args)\n",
    "            \n",
    "            SetSeed(args.SEED)\n",
    "\n",
    "            evaluator = Evaluator(args).to(args.DEVICE)\n",
    "\n",
    "            for x,y in test_loader:\n",
    "                X_test, y_test = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "                start_time = time.time()\n",
    "                model(X_test[:1, :, :])\n",
    "                end_time = time.time()\n",
    "                inference_time = end_time - start_time\n",
    "                results[ds] = inference_time\n",
    "                break\n",
    "\n",
    "        else:\n",
    "            results[ds] = float('nan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5beec48a",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(f\"./evaluation1/inf_time_2Order_LPF_acc.txt\", results.numpy(), delimiter=\"\\t\", fmt='%.5f')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f55baa31",
   "metadata": {},
   "source": [
    "## Count devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82e085a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = torch.zeros([15, 3])\n",
    "\n",
    "for ds in range(15):\n",
    "        args.DATASET = ds\n",
    "        seed = args.SEED\n",
    "        test_loader , datainfo = GetDataLoader(args, 'test',  path='../dataset/')\n",
    "        print(datainfo)\n",
    "\n",
    "        modelname = f\"pLF_data_{ds:02d}_{datainfo['dataname']}_seed_{seed:02d}.model\"\n",
    "        \n",
    "        model_exist = os.path.isfile(f'./models/{modelname}')\n",
    "\n",
    "        if model_exist:  \n",
    "                          \n",
    "            model = torch.load(f'./models/{modelname}', map_location=args.DEVICE)\n",
    "            \n",
    "            model.UpdateArgs(args)\n",
    "            \n",
    "            SetSeed(args.SEED)\n",
    "\n",
    "            evaluator = Evaluator(args).to(args.DEVICE)\n",
    "\n",
    "            state_dict = model.state_dict()\n",
    "            num_theta = 0\n",
    "            num_r = 0\n",
    "            num_c = 0\n",
    "            for name in state_dict:\n",
    "                if \"theta_\" in name:\n",
    "                    num_theta += state_dict[name].shape[0] * state_dict[name].shape[1]\n",
    "                elif \"R_\" in name:\n",
    "                    num_r +=1\n",
    "                elif \"C_\" in name:\n",
    "                    num_c += 1\n",
    "            \n",
    "            results[ds][0] = num_theta\n",
    "            results[ds][1] = num_r\n",
    "            results[ds][2] = num_c\n",
    "\n",
    "        else:\n",
    "            results[ds] = float('nan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "eefea9a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(f\"./evaluation1/count_device_2Order_LPF_acc.txt\", results.numpy(), delimiter=\"\\t\", fmt='%.5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0281ec0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = torch.zeros([15,10,6])\n",
    "\n",
    "for ds in range(15):\n",
    "    args.DATASET = ds\n",
    "    valid_loader, datainfo = GetDataLoader(args, 'valid', path='../dataset/')\n",
    "    test_loader , datainfo = GetDataLoader(args, 'test',  path='../dataset/')\n",
    "    print(datainfo)\n",
    "    for seed in range(10):\n",
    "        args.SEED = seed\n",
    "\n",
    "        modelname = f\"pLF_data_{ds:02d}_{datainfo['dataname']}_seed_{seed:02d}.model\"\n",
    "        \n",
    "        model_exist = os.path.isfile(f'./models/{modelname}')\n",
    "\n",
    "        if model_exist:  \n",
    "                          \n",
    "            model = torch.load(f'./models/{modelname}', map_location=args.DEVICE)\n",
    "            \n",
    "            model.UpdateArgs(args)\n",
    "            \n",
    "            SetSeed(args.SEED)\n",
    "\n",
    "            evaluator = Evaluator(args).to(args.DEVICE)\n",
    "\n",
    "            for x,y in valid_loader:\n",
    "                X_valid, y_valid = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "            for x,y in test_loader:\n",
    "                X_test, y_test = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "\n",
    "            acc_valid = evaluator(model, X_valid, y_valid)\n",
    "            acc_test   = evaluator(model, X_test,  y_test)\n",
    "            \n",
    "            results[ds,seed,0] = acc_valid\n",
    "            results[ds,seed,1] = acc_test\n",
    "            \n",
    "        else:\n",
    "            results[ds,seed,:] = float('nan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b0fb7fc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_selected_seeds = 3\n",
    "re_temp = torch.nan_to_num(results, nan=-10000.)\n",
    "values, indices = torch.topk(re_temp[:,:,1], k=N_selected_seeds, dim=1)\n",
    "mean_selected = torch.mean(torch.asarray(values), dim=1)\n",
    "var_selected = torch.std(torch.asarray(values), dim=1)\n",
    "selected_results = torch.cat([mean_selected.unsqueeze(-1), var_selected.unsqueeze(-1)], dim=1)\n",
    "\n",
    "np.savetxt(f\"./evaluation1/non_var_test_top_{N_selected_seeds}_2Order_LPF_acc.txt\", selected_results.numpy(), delimiter=\"\\t\", fmt='%.5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "556d6140",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_selected_seeds = 3\n",
    "re_temp = torch.nan_to_num(results, nan=-10000.)\n",
    "values, indices = torch.topk(re_temp[:,:,3], k=N_selected_seeds, dim=1)\n",
    "mean_selected = torch.mean(torch.asarray(values), dim=1)\n",
    "var_selected = torch.std(torch.asarray(values), dim=1)\n",
    "selected_results = torch.cat([mean_selected.unsqueeze(-1), var_selected.unsqueeze(-1)], dim=1)\n",
    "\n",
    "np.savetxt(f\"./evaluation1/non_var_test_top_{N_selected_seeds}_aug_LPF_acc.txt\", selected_results.numpy(), delimiter=\"\\t\", fmt='%.5f')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1ef12b8",
   "metadata": {},
   "source": [
    "## Results on augmented test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89e19907",
   "metadata": {},
   "outputs": [],
   "source": [
    "args.augment = True\n",
    "args.augment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35b5bc8e",
   "metadata": {},
   "source": [
    "## Proposed model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d483bb80",
   "metadata": {},
   "outputs": [],
   "source": [
    "import PowerPrintedLearnableFilter as pNN\n",
    "args.augment = True\n",
    "all_results = []\n",
    "results = torch.zeros([15,10,6])\n",
    "for ds in range(15):\n",
    "    args.DATASET = ds\n",
    "    for seed in range(10):\n",
    "        valid_loader, datainfo = GetDataLoader(args, 'valid', path='./dataset/')\n",
    "        modelname = f\"pLF_data_{ds:02d}_{datainfo['dataname']}_seed_{seed:02d}.model\"\n",
    "        model_exist = os.path.isfile(f'./ConstantLearnableFilters/models/{modelname}')\n",
    "        print(model_exist)\n",
    "        if model_exist:     \n",
    "            model = torch.load(f'./ConstantLearnableFilters/models/{modelname}', map_location=args.DEVICE)\n",
    "            model.UpdateArgs(args)\n",
    "        break\n",
    "    break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dc6ff3a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "pgmin = 1e-7\n",
    "def g_tilde(theta_):\n",
    "    # scaled conductances\n",
    "    g_initial = theta_.abs()\n",
    "    g_min = g_initial.min(dim=0, keepdim=True)[0]\n",
    "    scaler = pgmin / g_min\n",
    "    return g_initial * scaler\n",
    "\n",
    "Rmin = torch.tensor(1e5)    # 100kOhm\n",
    "Rmax = torch.tensor(1e7)    # 10MOhm\n",
    "Cmin = torch.tensor(1e-7)   # 100nF\n",
    "Cmax = torch.tensor(1e-4)   # 100uF\n",
    "\n",
    "def C(C_):\n",
    "    C_true = torch.sigmoid(C_) * (Cmax - Cmin) + Cmin\n",
    "    return C_true\n",
    "        \n",
    "def R(R_):  \n",
    "    R_true = torch.sigmoid(R_) * (Rmax - Rmin) + Rmin\n",
    "    return R_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cd3ad09",
   "metadata": {},
   "outputs": [],
   "source": [
    "def MAC_power(self, x, y):\n",
    "    # dimensions of x: (num_batch, B, D, T)\n",
    "    # the features are in last dimension\n",
    "    x = x.permute(0, 1, 3, 2)\n",
    "    x_extend = torch.cat([x,\n",
    "                          torch.ones([x.shape[0], x.shape[1], x.shape[2], 1]).to(self.device),\n",
    "                          torch.zeros([x.shape[0], x.shape[1], x.shape[2], 1]).to(self.device)], dim=3)\n",
    "    x_neg = self.INV(x_extend)\n",
    "    x_neg[:, :, :, -1] = 0.\n",
    "    F = x_extend.shape[0]\n",
    "    V = x_extend.shape[1]\n",
    "    E = x_extend.shape[2]\n",
    "    M = x_extend.shape[3]\n",
    "    N = y.shape[3]\n",
    "    positive = self.theta_noisy.clone().detach().to(self.device)\n",
    "    positive[positive >= 0] = 1.\n",
    "    positive[positive < 0] = 0.\n",
    "    negative = 1. - positive\n",
    "    Power = torch.tensor(0.).to(self.device)\n",
    "    for f in range(F):\n",
    "        for v in range(V):\n",
    "            for m in range(M):\n",
    "                for n in range(N):\n",
    "                    Power += self.g_tilde[m, n] * ((x_extend[f, v, :, m]*positive[f, v, m, n] +\n",
    "                                                   x_neg[f, v, :, m]*negative[f, v, m, n])-y[f, v, :, n]).pow(2.).sum()\n",
    "    Power = Power / E / V / F\n",
    "    return Power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bf31381",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_loader, datainfo = GetDataLoader(args, 'valid', path='./dataset/')\n",
    "test_loader , datainfo = GetDataLoader(args, 'test',  path='./dataset/')  \n",
    "for x, y in test_loader:\n",
    "    print(x.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8319d432",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to pad shorter lists with None or empty strings\n",
    "def pad_to_length(lists, target_length):\n",
    "    return [lst + [None] * (target_length - len(lst)) for lst in lists]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ef9077ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import PrintedLearnableFilter as pNN\n",
    "# 5uw * count_2nd_order\n",
    "# sum over every g_tild (the valid ones)\n",
    "# acts\n",
    "# invs \n",
    "\n",
    "# sum over them\n",
    "def document_model(model, dataset):\n",
    "    # Initialize powers\n",
    "    sum_inv_power = 0\n",
    "    sum_act_power = 0\n",
    "    sum_crossbar_power = 0\n",
    "    sum_lF_power = 0\n",
    "    \n",
    "    LF_POWER = 5e-6\n",
    "    \n",
    "    # Initialize counters\n",
    "    count_g = 0\n",
    "    count_act = 0\n",
    "    count_inv = 0\n",
    "    count_R = 0\n",
    "    count_C = 0\n",
    "    count_secorder_filter = 0\n",
    "    count_single_filter = 0\n",
    "\n",
    "    # Initialize lists to store values for the Excel file\n",
    "    theta_values = []\n",
    "    g_tilde_values = []\n",
    "    mac_values = []\n",
    "    inv_power_values = []\n",
    "    act_power_values = []\n",
    "    r_values = []\n",
    "    c_values = []\n",
    "\n",
    "    # Simulate iterating over the model (assuming 'model' and 'g_tilde' are defined elsewhere)\n",
    "    for name, module in model.model.named_modules():\n",
    "        if name == '0_pLayer' or name == '1_pLayer' or name == '2_pLayer':\n",
    "            for name_pl, module_pl in module.named_modules():\n",
    "                if name_pl == 'model.0_MAC' or name_pl == 'model.2_MAC':\n",
    "                    if hasattr(module_pl, 'theta_'):\n",
    "                        n_neuron = module_pl.theta_.shape[1]\n",
    "                        n_inp = module_pl.theta_.shape[0]\n",
    "                        theta_ = module_pl.theta_\n",
    "                        \n",
    "                        nonzero = module_pl.theta.clone().detach().abs()\n",
    "                        nonzero[nonzero > 0] = 1.\n",
    "                        num_theta = nonzero.sum()\n",
    "                        count_g += num_theta.item()\n",
    "                        g_tilde_ = g_tilde(theta_)\n",
    "                        \n",
    "                        \n",
    "                        g_tilde_[torch.isnan(g_tilde_)] = 0\n",
    "                        g_tilde_[torch.isinf(g_tilde_)] = 0\n",
    "                        sum_crossbar_power += g_tilde_.sum().item()\n",
    "\n",
    "                        # Store theta and g_tilde values using detach to avoid the RuntimeError\n",
    "                        theta_values.append(theta_.detach().numpy())\n",
    "                        g_tilde_values.append(g_tilde_.detach().numpy())\n",
    "\n",
    "                        for col in range(n_neuron):\n",
    "                            mac_values.append(g_tilde_[:, col].detach().numpy())\n",
    "\n",
    "                        if hasattr(module_pl, 'INV'):\n",
    "                            inv_power_values.append(module_pl.INV.power.detach().item())\n",
    "                            nonzero = module_pl.theta.clone().detach().abs()[:-2, :]\n",
    "                            nonzero[nonzero > 0] = 1.\n",
    "                            count_inv += nonzero.max(0)[0].sum().item()\n",
    "                            sum_inv_power += module_pl.INV.power.detach().item()\n",
    "\n",
    "                        if hasattr(module_pl, 'ACT'):\n",
    "                            act_power_values.append(module_pl.ACT.power.detach().item())\n",
    "                            positive = module_pl.theta.clone().detach()[:-2, :]\n",
    "                            positive[positive >= 0] = 1.\n",
    "                            positive[positive < 0] = 0.\n",
    "                            negative = 1. - positive\n",
    "                            count_act += negative.max(1)[0].sum().item()\n",
    "                            sum_act_power += module_pl.ACT.power.detach().item()\n",
    "\n",
    "                if name_pl == 'model.1_LF':\n",
    "                    # Access the FilterGroups within '1_LF'\n",
    "                    for group_name, filter_group in module_pl.FilterGroups.named_modules():\n",
    "                        if hasattr(filter_group, 'FilterGroup'):\n",
    "                            \n",
    "                            for filter_name, sec_learnable_filter in filter_group.FilterGroup.named_modules():\n",
    "                                if isinstance(sec_learnable_filter, pNN.SecOLearnableFilter):\n",
    "                                    # count_secorder_filter += 1\n",
    "                                    # print(count_secorder_filter)\n",
    "                                    if hasattr(sec_learnable_filter, 'LearnableFilters'):\n",
    "                                        num_lf = 0\n",
    "                                        for name, learnable_filter in sec_learnable_filter.LearnableFilters.named_modules():\n",
    "                                            if isinstance(learnable_filter, pNN.LearnableFilter):\n",
    "                                                if hasattr(learnable_filter, 'R_') and hasattr(learnable_filter, 'C_'):\n",
    "                                                    r = R(learnable_filter.R_).detach().numpy()\n",
    "                                                    r_values.append(r)\n",
    "                                                    nonzero = R(learnable_filter.R_).detach().abs()\n",
    "                                                    nonzero[nonzero > 0] = 1.\n",
    "                                                    num_r = nonzero.sum().item()\n",
    "                                                    count_R += num_r\n",
    "\n",
    "                                                    c_values.append(C(learnable_filter.C_).detach().numpy())\n",
    "                                                    nonzero = C(learnable_filter.C_).detach().abs()\n",
    "                                                    nonzero[nonzero > 0] = 1.\n",
    "                                                    num_c = nonzero.sum().item()\n",
    "                                                    count_C += num_c\n",
    "                                                    \n",
    "                                                    if num_r == 1 and num_c == 1:\n",
    "                                                        count_single_filter += 1\n",
    "                                                        num_lf += 1\n",
    "                                                    if num_lf == 2:\n",
    "                                                        count_secorder_filter += 1\n",
    "                                            # print(num_lf)\n",
    "                                            \n",
    "    sum_lF_power = LF_POWER * count_secorder_filter\n",
    "    \n",
    "    sum_power = sum_inv_power + sum_act_power + sum_crossbar_power + sum_lF_power\n",
    "                                                        \n",
    "\n",
    "    # Get the max length of the lists\n",
    "    max_length = max(len(theta_values), len(g_tilde_values), len(mac_values), len(inv_power_values), len(act_power_values), len(r_values), len(c_values))\n",
    "\n",
    "    # Pad all lists to the same length\n",
    "    theta_values, g_tilde_values, mac_values, inv_power_values, act_power_values, r_values, c_values, count_g_list,  count_inv_list, count_act_list, count_R_list, count_C_list, count_sec_filter_list, count_sin_filter_list, sum_inv_power_ls, sum_act_power_ls, sum_crossbar_power_ls, sum_lF_power_ls, sum_power_ls = pad_to_length([theta_values, g_tilde_values, mac_values, inv_power_values, act_power_values, r_values, c_values,\n",
    "        [count_g], [count_inv], [count_act], [count_R], [count_C], [count_secorder_filter], [count_single_filter],\n",
    "        [sum_inv_power], [sum_act_power], [sum_crossbar_power], [sum_lF_power], [sum_power]], max_length)\n",
    "\n",
    "    # # Broadcast counters to match the length of the lists\n",
    "    # count_g_list = [count_g] * max_length\n",
    "    # count_inv_list = [count_inv] * max_length\n",
    "    # count_act_list = [count_act] * max_length\n",
    "    # count_R_list = [count_R] * max_length\n",
    "    # count_C_list = [count_C] * max_length\n",
    "\n",
    "    # Create a DataFrame to store the extracted values\n",
    "    df = pd.DataFrame({\n",
    "        'g_tilde Layer wise': [str(v) for v in g_tilde_values],\n",
    "        'g_tilde MAC wise': [str(v) for v in mac_values],\n",
    "        'INV Power': inv_power_values,\n",
    "        'ACT Power': act_power_values,\n",
    "        'R Values': [str(v) for v in r_values],\n",
    "        'C Values': [str(v) for v in c_values],\n",
    "        'Count g': count_g_list,\n",
    "        'Count INV': count_inv_list,\n",
    "        'Count ACT': count_act_list,\n",
    "        'Count R': count_R_list,\n",
    "        'Count C': count_C_list,\n",
    "        'Count Sec Filter': count_sec_filter_list,\n",
    "        'Count Single Filter': count_sin_filter_list,\n",
    "        'Sum ACT Power': sum_act_power_ls,\n",
    "        'Sum INV Power': sum_inv_power_ls,\n",
    "        'Sum Crossbar Power': sum_crossbar_power_ls,\n",
    "        'Sum LF Power': sum_lF_power_ls,\n",
    "        'Sum Power': sum_power_ls\n",
    "    })\n",
    "\n",
    "    # Save to Excel\n",
    "    file_path = f'./{dataset}_model_parameters_fixed.xlsx'\n",
    "    df.to_excel(file_path, index=False)\n",
    "\n",
    "    # Display DataFrame to the user\n",
    "    # import ace_tools as tools; tools.display_dataframe_to_user(name=\"Model Parameters\", dataframe=df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "094bec7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95a82322",
   "metadata": {},
   "outputs": [],
   "source": [
    "count_single_filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cb5b3aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "'Theta Values', [str(v) for v in theta_values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abe1ae00-dcac-4410-bfbb-25287ab417ba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "args.augment = True\n",
    "all_results = []\n",
    "results = torch.zeros([15,10,6])\n",
    "for ds in range(15):\n",
    "    args.DATASET = ds\n",
    "    valid_loader, datainfo = GetDataLoader(args, 'valid', path='../dataset/')\n",
    "    test_loader , datainfo = GetDataLoader(args, 'test',  path='../dataset/')  \n",
    "    print(datainfo)\n",
    "    for seed in range(10):\n",
    "        args.SEED = seed\n",
    "\n",
    "\n",
    "        modelname = f\"pLF_data_{ds:02d}_{datainfo['dataname']}_seed_{seed:02d}.model\"\n",
    "        \n",
    "        model_exist = os.path.isfile(f'./models/{modelname}')\n",
    "\n",
    "        print(model_exist)\n",
    "        \n",
    "        \n",
    "        if model_exist:  \n",
    "                          \n",
    "            model = torch.load(f'./models/{modelname}', map_location=args.DEVICE)\n",
    "            model.UpdateArgs(args)\n",
    "\n",
    "            SetSeed(args.SEED)\n",
    "\n",
    "            evaluator = Evaluator(args).to(args.DEVICE)\n",
    "            # baseline_evaluator = baselineEvaluator(baseline_args).to(baseline_args.DEVICE)\n",
    "\n",
    "            for x,y in valid_loader:\n",
    "                X_valid, y_valid = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "            for x,y in test_loader:\n",
    "                X_test, y_test = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "\n",
    "            acc_valid = evaluator(model, X_valid, y_valid)\n",
    "            acc_test   = evaluator(model, X_test,  y_test)\n",
    "\n",
    "            results[ds,seed,0] = acc_valid\n",
    "            results[ds,seed,1] = acc_test\n",
    "            \n",
    "            inference_time = 0\n",
    "            \n",
    "            for x,y in test_loader:\n",
    "                X_test, y_test = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "                start_time = time.time()\n",
    "                model(X_test[:1, :, :])\n",
    "                end_time = time.time()\n",
    "                inference_time = end_time - start_time\n",
    "                break\n",
    "            results[ds, seed, 2] = inference_time\n",
    "            \n",
    "            # results[ds,seed,3] = area_test\n",
    "            # results[ds,seed,4] = power_test\n",
    "            \n",
    "        else:\n",
    "            results[ds,seed,:] = float('nan')\n",
    "            \n",
    "        temp_result = [datainfo['dataname'], seed, results[ds, seed, 0].item(), results[ds, seed, 1].item(), results[ds, seed, 2].item()]\n",
    "        all_results.append(temp_result)\n",
    "\n",
    "columns = ['dataset', 'seed', 'acc_valid', 'acc_test', 'average_time']\n",
    "all_results.sort(key=lambda x: x[0])            \n",
    "df = pd.DataFrame(all_results, columns=columns)\n",
    "# Save the DataFrame to an Excel file\n",
    "if not os.path.exists(f\"./evaluation2_date/\"):\n",
    "    os.makedirs(f\"./evaluation2_date/\")\n",
    "excel_filename = f\"./evaluation2_date/augment_{args.augment}_proposed_evaluation_results_analysis.xlsx\"\n",
    "df.to_excel(excel_filename, index=False)\n",
    "print(f\"Results have been saved to {excel_filename}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f1ed4310",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_selected_seeds = 3\n",
    "re_temp = torch.nan_to_num(results, nan=-10000.)\n",
    "values, indices = torch.topk(re_temp[:,:,1], k=N_selected_seeds, dim=1)\n",
    "mean_selected = torch.mean(torch.asarray(values), dim=1)\n",
    "var_selected = torch.std(torch.asarray(values), dim=1)\n",
    "selected_results = torch.cat([mean_selected.unsqueeze(-1), var_selected.unsqueeze(-1)], dim=1)\n",
    "\n",
    "np.savetxt(f\"./evaluation1/var_test_top_{N_selected_seeds}_var_LPF_acc.txt\", selected_results.numpy(), delimiter=\"\\t\", fmt='%.5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c967fa35",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_selected_seeds = 3\n",
    "re_temp = torch.nan_to_num(results, nan=-10000.)\n",
    "values, indices = torch.topk(re_temp[:,:,3], k=N_selected_seeds, dim=1)\n",
    "mean_selected = torch.mean(torch.asarray(values), dim=1)\n",
    "var_selected = torch.std(torch.asarray(values), dim=1)\n",
    "selected_results = torch.cat([mean_selected.unsqueeze(-1), var_selected.unsqueeze(-1)], dim=1)\n",
    "\n",
    "np.savetxt(f\"./evaluation1/var_test_top_{N_selected_seeds}_aug_LPF_acc.txt\", selected_results.numpy(), delimiter=\"\\t\", fmt='%.5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38e92949",
   "metadata": {},
   "outputs": [],
   "source": [
    "args.augment = True\n",
    "all_results = []\n",
    "results = torch.zeros([15,10,6])\n",
    "for ds in range(9):\n",
    "    args.DATASET = ds\n",
    "    valid_loader, datainfo = GetDataLoader(args, 'valid', path='../dataset/')\n",
    "    test_loader , datainfo = GetDataLoader(args, 'test',  path='../dataset/')  \n",
    "    print(datainfo)\n",
    "    for seed in range(10):\n",
    "        args.SEED = seed\n",
    "\n",
    "\n",
    "        modelname = f\"pLF_data_{ds:02d}_{datainfo['dataname']}_seed_{seed:02d}.model\"\n",
    "        \n",
    "        model_exist = os.path.isfile(f'./models/{modelname}')\n",
    "        # aug_model_exist = os.path.isfile(f'../AugLearnableFilters/models/{modelname}')\n",
    "        # baseline_model_exist = os.path.isfile(f'../LastBaseline/LearnableFilters/models/{modelname}')\n",
    "        \n",
    "        # print(model_exist and aug_model_exist and baseline_model_exist)\n",
    "        print(model_exist)\n",
    "        \n",
    "        \n",
    "        if model_exist:  \n",
    "                          \n",
    "            model = torch.load(f'./models/{modelname}', map_location=args.DEVICE)\n",
    "            # aug_model = torch.load(f'../AugLearnableFilters/models/{modelname}', map_location=args.DEVICE)\n",
    "            # baseline_model = torch.load(f'../LastBaseline/LearnableFilters/models/{modelname}', map_location=baseline_args.DEVICE)\n",
    "            \n",
    "            model.UpdateArgs(args)\n",
    "            # aug_model.UpdateArgs(args)\n",
    "            # baseline_model.UpdateArgs(baseline_args)\n",
    "            \n",
    "            SetSeed(args.SEED)\n",
    "\n",
    "            evaluator = Evaluator(args).to(args.DEVICE)\n",
    "            # baseline_evaluator = baselineEvaluator(baseline_args).to(baseline_args.DEVICE)\n",
    "\n",
    "            for x,y in valid_loader:\n",
    "                X_valid, y_valid = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "            for x,y in test_loader:\n",
    "                X_test, y_test = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "\n",
    "            acc_valid = evaluator(model, X_valid, y_valid)\n",
    "            acc_test   = evaluator(model, X_test,  y_test)\n",
    "            \n",
    "            # aug_acc_valid = evaluator(aug_model, X_valid, y_valid)\n",
    "            # aug_acc_test   = evaluator(aug_model, X_test,  y_test)\n",
    "            \n",
    "            # baseline_acc_valid = baseline_evaluator(baseline_model, X_valid, y_valid)\n",
    "            # baseline_acc_test   = baseline_evaluator(baseline_model, X_test,  y_test)\n",
    "\n",
    "            results[ds,seed,0] = acc_valid\n",
    "            results[ds,seed,1] = acc_test\n",
    "            \n",
    "            # results[ds,seed,2] = aug_acc_valid\n",
    "            # results[ds,seed,3] = aug_acc_test\n",
    "            \n",
    "            # results[ds,seed,4] = baseline_acc_valid\n",
    "            # results[ds,seed,5] = baseline_acc_test\n",
    "        else:\n",
    "            results[ds,seed,:] = float('nan')\n",
    "            \n",
    "        inference_time = 0\n",
    "            \n",
    "        for x,y in test_loader:\n",
    "            X_test, y_test = x.to(args.DEVICE), y.to(args.DEVICE)\n",
    "            start_time = time.time()\n",
    "            model(X_test[:1, :, :])\n",
    "            end_time = time.time()\n",
    "            inference_time = end_time - start_time\n",
    "            break\n",
    "        results[ds, seed, 2] = inference_time\n",
    "            \n",
    "        temp_result = [datainfo['dataname'], seed, results[ds, seed, 0].item(), results[ds, seed, 1].item(), results[ds, seed, 2].item()]\n",
    "        all_results.append(temp_result)\n",
    "\n",
    "columns = ['dataset', 'seed', 'acc_valid', 'acc_test', 'average_time']\n",
    "all_results.sort(key=lambda x: x[0])            \n",
    "df = pd.DataFrame(all_results, columns=columns)\n",
    "# Save the DataFrame to an Excel file\n",
    "if not os.path.exists(f\"./evaluation/\"):\n",
    "    os.makedirs(f\"./evaluation/\")\n",
    "excel_filename = f\"./evaluation/ELMAN_evaluation_results_analysis.xlsx\"\n",
    "df.to_excel(excel_filename, index=False)\n",
    "print(f\"Results have been saved to {excel_filename}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "train-model",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}