DataAnalysis-v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import numpy as np\n",
    "import statsmodels.api as sm\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler, PowerTransformer, Normalizer\n",
    "import os\n",
    "import glob\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking up the idle current readings for reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the idle power consumption with and without Fan and no perf running\n",
    "# i.e. # Idleworkload-MaxFan and \n",
    "\n",
    "def __get_stats__(path:str,field_prefix:str) -> pandas.DataFrame:\n",
    "    freqs=[]\n",
    "    ampere_mA_mean = []\n",
    "    watt_mW_mean = []\n",
    "    idle_datafname  = re.compile(r'.*CPUFreq\\-([.\\d]+)GHz')\n",
    "    _idle_results_maxfan_bcore = glob.glob(path)\n",
    "    ctr = 0\n",
    "    for freqstepdata in _idle_results_maxfan_bcore:\n",
    "        fname_match = idle_datafname.match(freqstepdata)\n",
    "        if (fname_match):\n",
    "            # print (freqstepdata + ': '+fname_match.group(1))\n",
    "            freqidlepd = pandas.read_csv(freqstepdata+'/Idling.powdata.csv')\n",
    "            freqidlepd.drop(columns=['utctime.1', 'ts_utc'], inplace=True)\n",
    "            freqidlepd['dev_ippwr-ch1-volts_mV'] = freqidlepd['dev_ippwr-ch1-volts_mV'].apply(lambda x: float(x[2:-1]) if isinstance(x, str) and x.startswith(\"b'\") else x)\n",
    "            freqidlepd['dev_ippwr-ch1-ampere_mA'] = freqidlepd['dev_ippwr-ch1-ampere_mA'].apply(lambda x: float(x[2:-1]) if isinstance(x, str) and x.startswith(\"b'\") else x)\n",
    "            freqidlepd['dev_ippwr-ch1-watt_mW'] = freqidlepd['dev_ippwr-ch1-watt_mW'].apply(lambda x: float(x[2:-1]) if isinstance(x, str) and x.startswith(\"b'\") else x)\n",
    "\n",
    "            df2_mean = freqidlepd[[\"dev_ippwr-ch1-ampere_mA\",\"dev_ippwr-ch1-watt_mW\",\"therm_cpu0\",\"therm_cpu1\", \"therm_cpu2\", \"therm_cpu4\"]].mean()\n",
    "            df2_median = freqidlepd[[\"dev_ippwr-ch1-ampere_mA\",\"dev_ippwr-ch1-watt_mW\",\"therm_cpu0\",\"therm_cpu1\", \"therm_cpu2\", \"therm_cpu4\"]].median()\n",
    "            # display(freqidlepd)\n",
    "            freqs.append(float(fname_match.group(1)))\n",
    "            ampere_mA_mean.append(df2_mean[\"dev_ippwr-ch1-ampere_mA\"])\n",
    "            watt_mW_mean.append(df2_mean[\"dev_ippwr-ch1-watt_mW\"])\n",
    "\n",
    "            # print (str(freqs[ctr])+': '+str(ampere_mA_mean[ctr])+', '+str(ampere_mA_median[ctr])+', '+str(watt_mW_mean[ctr])+', '+str(watt_mW_median[ctr]))\n",
    "            \n",
    "        else:\n",
    "            # print ('No match for: '+ freqstepdata)\n",
    "            pass\n",
    "        ctr += 1\n",
    "    df = pandas.DataFrame(list(zip(freqs, ampere_mA_mean, watt_mW_mean)),\n",
    "                        columns =['Frequency', field_prefix+'ampere_mA_mean', field_prefix+'watt_mW_mean'])\n",
    "    return df    \n",
    "\n",
    "def calculate_power_impacts():\n",
    "    ###########################################################################\n",
    "    # Compile the Big Core idling current data\n",
    "    df_maxfan_bigcore = __get_stats__('combined_dataset-2/01-Simple-Idling/MaxFan/*-BigCore-*','MaxFan_')\n",
    "    df_maxfan_bigcore.sort_values('Frequency', inplace=True)\n",
    "    # df_maxfan.index= df_maxfan['Frequency']\n",
    "\n",
    "    df_nofan_bigcore = __get_stats__('combined_dataset-2/01-Simple-Idling/NoFan/*-BigCore-*','NoFan_')\n",
    "    df_nofan_bigcore.sort_values('Frequency', inplace=True)\n",
    "    # df_nofan.index= df_nofan['Frequency']\n",
    "\n",
    "    newdf_bigcore= pandas.merge(left=df_maxfan_bigcore, right=df_nofan_bigcore, on='Frequency' )\n",
    "    newdf_bigcore.sort_values('Frequency', inplace=True)\n",
    "\n",
    "    ###########################################################################\n",
    "    # Compile the Lttle Core idling current data\n",
    "    df_maxfan_littlecore = __get_stats__('combined_dataset/01-Simple-Idling/MaxFan/*-LittleCore-*','MaxFan_')\n",
    "    df_maxfan_littlecore.sort_values('Frequency', inplace=True)\n",
    "    # df_maxfan.index= df_maxfan['Frequency']\n",
    "\n",
    "    df_nofan_littlecore = __get_stats__('combined_dataset/01-Simple-Idling/NoFan/*-LittleCore-*','NoFan_')\n",
    "    df_nofan_littlecore.sort_values('Frequency', inplace=True)\n",
    "    # df_nofan.index= df_nofan['Frequency']\n",
    "\n",
    "    newdf_littlecore = pandas.merge(left=df_maxfan_littlecore, right=df_nofan_littlecore, on='Frequency' )\n",
    "    newdf_littlecore.sort_values('Frequency', inplace=True)\n",
    "\n",
    "    ###########################################################################\n",
    "    # Calculate the power consumption by Fan\n",
    "    df_bigcore_fandiff = newdf_bigcore['MaxFan_ampere_mA_mean'] - newdf_bigcore['NoFan_ampere_mA_mean']\n",
    "    df_litcore_fandiff = newdf_littlecore['MaxFan_ampere_mA_mean'] - newdf_littlecore['NoFan_ampere_mA_mean']\n",
    "    fan_delta = (df_bigcore_fandiff.describe()['mean'] + df_litcore_fandiff.describe()['mean'])/2\n",
    "    print ('Current consumption impact on fan = '+ str(fan_delta) +' mA')\n",
    "\n",
    "    df_bigcore_fandiff = newdf_bigcore['MaxFan_watt_mW_mean'] - newdf_bigcore['NoFan_watt_mW_mean']\n",
    "    df_litcore_fandiff = newdf_littlecore['MaxFan_watt_mW_mean'] - newdf_littlecore['NoFan_watt_mW_mean']\n",
    "    fan_delta = (df_bigcore_fandiff.describe()['mean'] + df_litcore_fandiff.describe()['mean'])/2\n",
    "    print ('Power consumption impact on fan = '+ str(fan_delta) +' mW')\n",
    "    ###########################################################################\n",
    "    # Calculate the Power difference between each frequency ranges in idling\n",
    "    # df_bigcore_freqdiff = newdf_bigcore.diff()\n",
    "    display(newdf_bigcore[['Frequency','NoFan_watt_mW_mean']])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze Correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_correlation_data(filename:str):\n",
    "    pd_bzip2_enwik8 = pandas.read_csv(filename)\n",
    "\n",
    "    pd_bzip2_enwik8['dev_ippwr-ch1-volts_mV'] = pd_bzip2_enwik8['dev_ippwr-ch1-volts_mV'].apply(lambda x: float(x[2:-1]) if isinstance(x, str) and x.startswith(\"b'\") else x)\n",
    "    pd_bzip2_enwik8['dev_ippwr-ch1-ampere_mA'] = pd_bzip2_enwik8['dev_ippwr-ch1-ampere_mA'].apply(lambda x: float(x[2:-1]) if isinstance(x, str) and x.startswith(\"b'\") else x)\n",
    "    pd_bzip2_enwik8['dev_ippwr-ch1-watt_mW'] = pd_bzip2_enwik8['dev_ippwr-ch1-watt_mW'].apply(lambda x: float(x[2:-1]) if isinstance(x, str) and x.startswith(\"b'\") else x)\n",
    "\n",
    "    ### JUGAAD WARNING :) ### \n",
    "    # Jugaad method of deducing earlier power estimations of idle power at each frequency and fan consumption\n",
    "    # pd_bzip2_enwik8['dev_ippwr-ch1-watt_mW'] = pd_bzip2_enwik8['dev_ippwr-ch1-watt_mW'] - 2761.181154 - 409.6397807531478\n",
    "\n",
    "    pd_bzip2_enwik8.drop(pd_bzip2_enwik8.columns.difference([\n",
    "        'utctime',\n",
    "        'dev_ippwr-ch1-volts_mV','dev_ippwr-ch1-ampere_mA','dev_ippwr-ch1-watt_mW',\n",
    "        'S0-D0-C0_cpu-cycles','S0-D0-C0_instructions','S0-D0-C0_cache-misses', 'S0-D0-C0_cache-references', 'S0-D0-C0_branch-instructions', 'S0-D0-C0_branch-misses', 'S0-D0-C0_branch-load-misses', 'S0-D0-C0_branch-loads',\n",
    "        'S0-D0-C1_cpu-cycles','S0-D0-C1_instructions','S0-D0-C1_cache-misses', 'S0-D0-C1_cache-references', 'S0-D0-C1_branch-instructions', 'S0-D0-C1_branch-misses', 'S0-D0-C1_branch-load-misses', 'S0-D0-C1_branch-loads',\n",
    "        'S0-D0-C2_cpu-cycles','S0-D0-C2_instructions','S0-D0-C2_cache-misses', 'S0-D0-C2_cache-references', 'S0-D0-C2_branch-instructions', 'S0-D0-C2_branch-misses', 'S0-D0-C2_branch-load-misses', 'S0-D0-C2_branch-loads',\n",
    "        'S0-D0-C3_cpu-cycles','S0-D0-C3_instructions','S0-D0-C3_cache-misses', 'S0-D0-C3_cache-references', 'S0-D0-C3_branch-instructions', 'S0-D0-C3_branch-misses', 'S0-D0-C3_branch-load-misses', 'S0-D0-C3_branch-loads',\n",
    "        'S1-D0-C0_cpu-cycles','S1-D0-C0_instructions','S1-D0-C0_cache-misses', 'S1-D0-C0_cache-references', 'S1-D0-C0_branch-instructions', 'S1-D0-C0_branch-misses', 'S1-D0-C0_branch-load-misses', 'S1-D0-C0_branch-loads',\n",
    "        'S1-D0-C1_cpu-cycles','S1-D0-C1_instructions','S1-D0-C1_cache-misses', 'S1-D0-C1_cache-references', 'S1-D0-C1_branch-instructions', 'S1-D0-C1_branch-misses', 'S1-D0-C1_branch-load-misses', 'S1-D0-C1_branch-loads',\n",
    "        'S1-D0-C2_cpu-cycles','S1-D0-C2_instructions','S1-D0-C2_cache-misses', 'S1-D0-C2_cache-references', 'S1-D0-C2_branch-instructions', 'S1-D0-C2_branch-misses', 'S1-D0-C2_branch-load-misses', 'S1-D0-C2_branch-loads',\n",
    "        'S1-D0-C3_cpu-cycles','S1-D0-C3_instructions','S1-D0-C3_cache-misses', 'S1-D0-C3_cache-references', 'S1-D0-C3_branch-instructions', 'S1-D0-C3_branch-misses', 'S1-D0-C3_branch-load-misses', 'S1-D0-C3_branch-loads',\n",
    "        ]), axis=1, inplace=True)\n",
    "\n",
    "    ## Per core CPI/IPC & Misc. calculation for little cores\n",
    "    # S0-D0-C0 - Little Core #1\n",
    "    pd_bzip2_enwik8['S0-D0-C0_CPI'] = pd_bzip2_enwik8['S0-D0-C0_cpu-cycles']/pd_bzip2_enwik8['S0-D0-C0_instructions']  \n",
    "    pd_bzip2_enwik8['S0-D0-C0_IPC'] = 1/pd_bzip2_enwik8['S0-D0-C0_CPI']\n",
    "    pd_bzip2_enwik8['S0-D0-C0_CacheMissRatio'] = pd_bzip2_enwik8['S0-D0-C0_cache-misses']  / pd_bzip2_enwik8['S0-D0-C0_cache-references']\n",
    "\n",
    "    # S0-D0-C1 - Little Core #2\n",
    "    pd_bzip2_enwik8['S0-D0-C1_CPI'] = pd_bzip2_enwik8['S0-D0-C1_cpu-cycles']/pd_bzip2_enwik8['S0-D0-C1_instructions']\n",
    "    pd_bzip2_enwik8['S0-D0-C1_IPC'] = 1/pd_bzip2_enwik8['S0-D0-C1_CPI']\n",
    "    pd_bzip2_enwik8['S0-D0-C1_CacheMissRatio'] = pd_bzip2_enwik8['S0-D0-C1_cache-misses']  / pd_bzip2_enwik8['S0-D0-C1_cache-references']\n",
    "\n",
    "    # S0-D0-C2 - Little Core #3\n",
    "    pd_bzip2_enwik8['S0-D0-C2_CPI'] = pd_bzip2_enwik8['S0-D0-C2_cpu-cycles']/pd_bzip2_enwik8['S0-D0-C2_instructions']\n",
    "    pd_bzip2_enwik8['S0-D0-C2_IPC'] = 1/pd_bzip2_enwik8['S0-D0-C2_CPI']\n",
    "    pd_bzip2_enwik8['S0-D0-C2_CacheMissRatio'] = pd_bzip2_enwik8['S0-D0-C2_cache-misses']  / pd_bzip2_enwik8['S0-D0-C2_cache-references']\n",
    "\n",
    "    # S0-D0-C3 - Little Core #4\n",
    "    pd_bzip2_enwik8['S0-D0-C3_CPI'] = pd_bzip2_enwik8['S0-D0-C3_cpu-cycles']/pd_bzip2_enwik8['S0-D0-C3_instructions']\n",
    "    pd_bzip2_enwik8['S0-D0-C3_IPC'] = 1/pd_bzip2_enwik8['S0-D0-C3_CPI']\n",
    "    pd_bzip2_enwik8['S0-D0-C3_CacheMissRatio'] = pd_bzip2_enwik8['S0-D0-C3_cache-misses'] / pd_bzip2_enwik8['S0-D0-C3_cache-references']\n",
    "\n",
    "    ## Per core CPI/IPC & Misc. calculation for Big cores\n",
    "    # S1-D0-C0 - Big Core #1\n",
    "    pd_bzip2_enwik8['S1-D0-C0_CPI'] = pd_bzip2_enwik8['S1-D0-C0_cpu-cycles']/pd_bzip2_enwik8['S1-D0-C0_instructions']\n",
    "    pd_bzip2_enwik8['S1-D0-C0_IPC'] = 1/pd_bzip2_enwik8['S1-D0-C0_CPI']\n",
    "    pd_bzip2_enwik8['S1-D0-C0_CacheMissRatio'] = pd_bzip2_enwik8['S1-D0-C0_cache-misses']  / pd_bzip2_enwik8['S1-D0-C0_cache-references']\n",
    "\n",
    "    # S1-D0-C1 - Big Core #2\n",
    "    pd_bzip2_enwik8['S1-D0-C1_CPI'] = pd_bzip2_enwik8['S1-D0-C1_cpu-cycles']/pd_bzip2_enwik8['S1-D0-C1_instructions']\n",
    "    pd_bzip2_enwik8['S1-D0-C1_IPC'] = 1/pd_bzip2_enwik8['S1-D0-C1_CPI']\n",
    "    pd_bzip2_enwik8['S1-D0-C1_CacheMissRatio'] = pd_bzip2_enwik8['S1-D0-C1_cache-misses']  / pd_bzip2_enwik8['S1-D0-C1_cache-references']\n",
    "\n",
    "    # S1-D0-C2 - Big Core #3\n",
    "    pd_bzip2_enwik8['S1-D0-C2_CPI'] = pd_bzip2_enwik8['S1-D0-C2_cpu-cycles']/pd_bzip2_enwik8['S1-D0-C2_instructions']\n",
    "    pd_bzip2_enwik8['S1-D0-C2_IPC'] = 1/pd_bzip2_enwik8['S1-D0-C2_CPI']\n",
    "    pd_bzip2_enwik8['S1-D0-C2_CacheMissRatio'] = pd_bzip2_enwik8['S1-D0-C2_cache-misses'] / pd_bzip2_enwik8['S1-D0-C2_cache-references']\n",
    "\n",
    "    # S1-D0-C3 - Big Core #4\n",
    "    pd_bzip2_enwik8['S1-D0-C3_CPI'] = pd_bzip2_enwik8['S1-D0-C3_cpu-cycles']/pd_bzip2_enwik8['S1-D0-C3_instructions']\n",
    "    pd_bzip2_enwik8['S1-D0-C3_IPC'] = 1/pd_bzip2_enwik8['S1-D0-C3_CPI']\n",
    "    pd_bzip2_enwik8['S1-D0-C3_CacheMissRatio'] = pd_bzip2_enwik8['S1-D0-C3_cache-misses'] / pd_bzip2_enwik8['S1-D0-C3_cache-references']\n",
    "\n",
    "    # Drop all the NA data points\n",
    "    pd_bzip2_enwik8 = pd_bzip2_enwik8.dropna()\n",
    "\n",
    "    ## Obtain the aggregate numbers for cluster and chip for CPI\n",
    "    collist_cpi = ['S0-D0-C0_CPI', 'S0-D0-C1_CPI', 'S0-D0-C2_CPI', 'S0-D0-C3_CPI']\n",
    "    collist_ipc = ['S1-D0-C0_CPI', 'S1-D0-C1_CPI', 'S1-D0-C2_CPI', 'S1-D0-C3_CPI']\n",
    "    pd_bzip2_enwik8['Aggregate_CPI-Little'] = pd_bzip2_enwik8[collist_cpi].sum(axis=1)\n",
    "    pd_bzip2_enwik8['Aggregate_CPI-Big'] = pd_bzip2_enwik8[collist_ipc].sum(axis=1)\n",
    "    pd_bzip2_enwik8['Aggregate_CPI'] = pd_bzip2_enwik8[['Aggregate_CPI-Little', 'Aggregate_CPI-Big']].sum(axis=1)\n",
    "\n",
    "    ## Obtain the aggregate numbers for cluster and chip for IPC\n",
    "    collist_litc = ['S0-D0-C0_IPC', 'S0-D0-C1_IPC', 'S0-D0-C2_IPC', 'S0-D0-C3_IPC']\n",
    "    collist_bigc = ['S1-D0-C0_IPC', 'S1-D0-C1_IPC', 'S1-D0-C2_IPC', 'S1-D0-C3_IPC']\n",
    "    pd_bzip2_enwik8['Aggregate_IPC-Little'] = pd_bzip2_enwik8[collist_litc].sum(axis=1)\n",
    "    pd_bzip2_enwik8['Aggregate_IPC-Big'] = pd_bzip2_enwik8[collist_bigc].sum(axis=1)\n",
    "    pd_bzip2_enwik8['Aggregate_IPC'] = pd_bzip2_enwik8[['Aggregate_IPC-Little','Aggregate_IPC-Big']].sum(axis=1)\n",
    "\n",
    "    ## Obtain the aggregate numbers for cluster and chip for Cache miss ratio\n",
    "    collist_cmrat_little = ['S1-D0-C0_CacheMissRatio','S1-D0-C1_CacheMissRatio', 'S1-D0-C2_CacheMissRatio','S1-D0-C3_CacheMissRatio']\n",
    "    collist_cmrat_big = ['S0-D0-C0_CacheMissRatio','S0-D0-C1_CacheMissRatio', 'S0-D0-C2_CacheMissRatio','S0-D0-C3_CacheMissRatio']\n",
    "    pd_bzip2_enwik8['Aggregate_CacheMissRatio-Big'] = pd_bzip2_enwik8[collist_cmrat_big].sum(axis=1)\n",
    "    pd_bzip2_enwik8['Aggregate_CacheMissRatio-Little'] = pd_bzip2_enwik8[collist_cmrat_little].sum(axis=1)\n",
    "    pd_bzip2_enwik8['Aggregate_CacheMissRatio'] = pd_bzip2_enwik8[['Aggregate_CacheMissRatio-Little','Aggregate_CacheMissRatio-Big']].sum(axis=1)\n",
    "\n",
    "    # Compose final data frame with Aggregate IPC\n",
    "    cols = ['dev_ippwr-ch1-ampere_mA','Aggregate_IPC']\n",
    "    __pd_bzip2_enwik8 = pd_bzip2_enwik8[cols]\n",
    "    __pd_bzip2_enwik8= __pd_bzip2_enwik8.sort_values('dev_ippwr-ch1-ampere_mA')\n",
    "\n",
    "    # Normalize the final data frame to appropriately scale the values\n",
    "    scaler = Normalizer()\n",
    "    df_scaled = scaler.fit_transform(__pd_bzip2_enwik8.to_numpy())\n",
    "    df_scaled = pandas.DataFrame(df_scaled, columns=cols)\n",
    "\n",
    "    correlation = df_scaled.corrwith(df_scaled['dev_ippwr-ch1-ampere_mA']).sort_values(ascending=False)\n",
    "    # display (correlation[\"Aggregate_IPC\"])\n",
    "    return correlation[\"Aggregate_IPC\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compiling results for combined_dataset/02-Idling-PerfSleep/MaxFan/*\n",
      "Processing (1/32): 1.2GHz data... \n",
      "Processing (2/32): 0.7GHz data... \n",
      "Processing (3/32): 0.4GHz data... \n",
      "Processing (4/32): 1.1GHz data... \n",
      "Processing (5/32): 1.6GHz data... \n",
      "Processing (6/32): 1.4GHz data... \n",
      "Processing (7/32): 0.9GHz data... \n",
      "Processing (8/32): 1.7GHz data... \n",
      "Processing (9/32): 0.3GHz data... \n",
      "Processing (10/32): 1.8GHz data... \n",
      "Processing (11/32): 2.0GHz data... \n",
      "Processing (12/32): 0.5GHz data... \n",
      "Processing (13/32): 1.3GHz data... \n",
      "Processing (14/32): 0.9GHz data... \n",
      "Processing (15/32): 1.0GHz data... \n",
      "Processing (16/32): 1.0GHz data... \n",
      "Processing (17/32): 1.2GHz data... \n",
      "Processing (18/32): 0.7GHz data... \n",
      "Processing (19/32): 0.6GHz data... \n",
      "Processing (20/32): 0.8GHz data... \n",
      "Processing (21/32): 0.4GHz data... \n",
      "Processing (22/32): 1.5GHz data... \n",
      "Processing (23/32): 0.3GHz data... \n",
      "Processing (24/32): 0.6GHz data... \n",
      "Processing (25/32): 0.2GHz data... \n",
      "Processing (26/32): 1.4GHz data... \n",
      "Processing (27/32): 1.1GHz data... \n",
      "Processing (28/32): 0.8GHz data... \n",
      "Processing (29/32): 1.9GHz data... \n",
      "Processing (30/32): 0.5GHz data... \n",
      "Processing (31/32): 0.2GHz data... \n",
      "Processing (32/32): 1.3GHz data... \n",
      "Correlation Result summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>correlation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>64.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.949698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.025432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.994767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.970261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.952245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-0.929933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>-0.899205</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       correlation\n",
       "count    64.000000\n",
       "mean     -0.949698\n",
       "std       0.025432\n",
       "min      -0.994767\n",
       "25%      -0.970261\n",
       "50%      -0.952245\n",
       "75%      -0.929933\n",
       "max      -0.899205"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correlation Results:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Frequency(GHz)</th>\n",
       "      <th>Data</th>\n",
       "      <th>correlation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.2</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.984523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.2</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.939765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.7</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.927773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.7</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.954542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.4</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.953920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.5</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.919343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.2</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.899205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>0.2</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.987075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>1.3</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.916924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>1.3</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.977512</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>64 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Frequency(GHz)                       Data  correlation\n",
       "0             1.2  IdleSleep-perf-1.prof.csv    -0.984523\n",
       "1             1.2  IdleSleep-perf-2.prof.csv    -0.939765\n",
       "2             0.7  IdleSleep-perf-1.prof.csv    -0.927773\n",
       "3             0.7  IdleSleep-perf-2.prof.csv    -0.954542\n",
       "4             0.4  IdleSleep-perf-1.prof.csv    -0.953920\n",
       "..            ...                        ...          ...\n",
       "59            0.5  IdleSleep-perf-2.prof.csv    -0.919343\n",
       "60            0.2  IdleSleep-perf-1.prof.csv    -0.899205\n",
       "61            0.2  IdleSleep-perf-2.prof.csv    -0.987075\n",
       "62            1.3  IdleSleep-perf-1.prof.csv    -0.916924\n",
       "63            1.3  IdleSleep-perf-2.prof.csv    -0.977512\n",
       "\n",
       "[64 rows x 3 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compiling results for combined_dataset/02-Idling-PerfSleep/NoFan/*\n",
      "Processing (1/32): 0.7GHz data... \n",
      "Processing (2/32): 0.3GHz data... \n",
      "Processing (3/32): 1.0GHz data... \n",
      "Processing (4/32): 0.2GHz data... \n",
      "Processing (5/32): 0.8GHz data... \n",
      "Processing (6/32): 0.2GHz data... \n",
      "Processing (7/32): 1.4GHz data... \n",
      "Processing (8/32): 0.4GHz data... \n",
      "Processing (9/32): 0.5GHz data... \n",
      "Processing (10/32): 1.7GHz data... \n",
      "Processing (11/32): 1.2GHz data... \n",
      "Processing (12/32): 1.3GHz data... \n",
      "Processing (13/32): 0.6GHz data... \n",
      "Processing (14/32): 1.1GHz data... \n",
      "Processing (15/32): 0.3GHz data... \n",
      "Processing (16/32): 2.0GHz data... \n",
      "Processing (17/32): 1.8GHz data... \n",
      "Processing (18/32): 0.9GHz data... \n",
      "Processing (19/32): 0.9GHz data... \n",
      "Processing (20/32): 0.4GHz data... \n",
      "Processing (21/32): 0.5GHz data... \n",
      "Processing (22/32): 1.2GHz data... \n",
      "Processing (23/32): 1.4GHz data... \n",
      "Processing (24/32): 1.3GHz data... \n",
      "Processing (25/32): 0.7GHz data... \n",
      "Processing (26/32): 1.1GHz data... \n",
      "Processing (27/32): 1.6GHz data... \n",
      "Processing (28/32): 1.0GHz data... \n",
      "Processing (29/32): 1.9GHz data... \n",
      "Processing (30/32): 1.5GHz data... \n",
      "Processing (31/32): 0.6GHz data... \n",
      "Processing (32/32): 0.8GHz data... \n",
      "Correlation Result summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>correlation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>64.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.944891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.025880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.993093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.968371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.941996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-0.926515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>-0.888258</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       correlation\n",
       "count    64.000000\n",
       "mean     -0.944891\n",
       "std       0.025880\n",
       "min      -0.993093\n",
       "25%      -0.968371\n",
       "50%      -0.941996\n",
       "75%      -0.926515\n",
       "max      -0.888258"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correlation Results:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Frequency(GHz)</th>\n",
       "      <th>Data</th>\n",
       "      <th>correlation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.7</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.946036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.7</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.917225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.959020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.3</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.969958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.954580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>1.5</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.968946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.968180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>0.6</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.957117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>0.8</td>\n",
       "      <td>IdleSleep-perf-1.prof.csv</td>\n",
       "      <td>-0.926340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>0.8</td>\n",
       "      <td>IdleSleep-perf-2.prof.csv</td>\n",
       "      <td>-0.915378</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>64 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Frequency(GHz)                       Data  correlation\n",
       "0             0.7  IdleSleep-perf-1.prof.csv    -0.946036\n",
       "1             0.7  IdleSleep-perf-2.prof.csv    -0.917225\n",
       "2             0.3  IdleSleep-perf-1.prof.csv    -0.959020\n",
       "3             0.3  IdleSleep-perf-2.prof.csv    -0.969958\n",
       "4             1.0  IdleSleep-perf-1.prof.csv    -0.954580\n",
       "..            ...                        ...          ...\n",
       "59            1.5  IdleSleep-perf-2.prof.csv    -0.968946\n",
       "60            0.6  IdleSleep-perf-1.prof.csv    -0.968180\n",
       "61            0.6  IdleSleep-perf-2.prof.csv    -0.957117\n",
       "62            0.8  IdleSleep-perf-1.prof.csv    -0.926340\n",
       "63            0.8  IdleSleep-perf-2.prof.csv    -0.915378\n",
       "\n",
       "[64 rows x 3 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compiling results for combined_dataset/03-Workloads/*\n",
      "Processing (1/32): 0.4GHz data... \n",
      "Processing (2/32): 1.5GHz data... \n",
      "Processing (3/32): 1.2GHz data... \n",
      "Processing (4/32): 0.5GHz data... \n",
      "Processing (5/32): 0.3GHz data... \n",
      "Processing (6/32): 1.4GHz data... \n"
     ]
    }
   ],
   "source": [
    "\n",
    "def process_correlation():\n",
    "    freq_regex  = re.compile(r'.*CPUFreq\\-([.\\d]+)GHz')\n",
    "\n",
    "    for item in [\n",
    "            'combined_dataset/02-Idling-PerfSleep/MaxFan/*',\n",
    "            'combined_dataset/02-Idling-PerfSleep/NoFan/*',\n",
    "            'combined_dataset/03-Workloads/*' ]:\n",
    "        frequency=[]\n",
    "        testset=[]\n",
    "        correlation=[]\n",
    "        dirs_collection = glob.glob(item)\n",
    "        dirs_collection_len = len (dirs_collection)\n",
    "        ctr = 0\n",
    "        print ('Compiling results for '+item)\n",
    "        for dirs in dirs_collection:\n",
    "            freqval_match = freq_regex.match(dirs)\n",
    "            if (freqval_match):\n",
    "                print ('Processing ('+str(ctr+1)+'/'+str(dirs_collection_len)+'): '+freqval_match.group(1)+'GHz data... ')\n",
    "                dataset_collection = glob.glob(dirs+'/*.csv')\n",
    "                for data in dataset_collection:\n",
    "                    frequency.append(freqval_match.group(1))\n",
    "                    testset.append(os.path.basename(data))\n",
    "                    corr = fetch_correlation_data(data)\n",
    "                    correlation.append(corr)\n",
    "                    \n",
    "            else:\n",
    "                print ('Error in frequency identification for '+dirs)\n",
    "            ctr += 1\n",
    "\n",
    "        df = pandas.DataFrame(list(zip(frequency, testset, correlation)),\n",
    "                            columns =['Frequency(GHz)','Data', 'correlation'])\n",
    "        # pandas.set_option('display.max_rows', df.shape[0]+1)\n",
    "        print (\"Correlation Result summary:\")\n",
    "        display(df.describe())\n",
    "\n",
    "        print (\"Correlation Results:\")\n",
    "        display(df)\n",
    "\n",
    "process_correlation()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_filter_level1 = [\n",
    "    'utctime',\n",
    "    'dev_ippwr-ch1-ampere_mA','dev_ippwr-ch1-watt_mW',\n",
    "    'S0-D0-C0_cpu-cycles','S0-D0-C0_instructions',\n",
    "    'S0-D0-C1_cpu-cycles','S0-D0-C1_instructions',\n",
    "    'S0-D0-C2_cpu-cycles','S0-D0-C2_instructions',\n",
    "    'S0-D0-C3_cpu-cycles','S0-D0-C3_instructions',\n",
    "    'S1-D0-C0_cpu-cycles','S1-D0-C0_instructions',\n",
    "    'S1-D0-C1_cpu-cycles','S1-D0-C1_instructions',\n",
    "    'S1-D0-C2_cpu-cycles','S1-D0-C2_instructions',\n",
    "    'S1-D0-C3_cpu-cycles','S1-D0-C3_instructions',\n",
    "]\n",
    "\n",
    "cols_bytedata_fields = [\n",
    "    'dev_ippwr-ch1-volts_mV', 'dev_ippwr-ch1-ampere_mA', 'dev_ippwr-ch1-watt_mW'\n",
    "]\n",
    "\n",
    "cols_corespecific_fields_litc = [\n",
    "    'S0-D0-C0', 'S0-D0-C1', 'S0-D0-C2', 'S0-D0-C3'\n",
    "]\n",
    "\n",
    "cols_corespecific_fields_bigc = [\n",
    "    'S1-D0-C0', 'S1-D0-C1', 'S1-D0-C2', 'S1-D0-C3'\n",
    "]\n",
    "\n",
    "collist_litc = ['S0-D0-C0_IPC', 'S0-D0-C1_IPC', 'S0-D0-C2_IPC', 'S0-D0-C3_IPC']\n",
    "collist_bigc = ['S1-D0-C0_IPC', 'S1-D0-C1_IPC', 'S1-D0-C2_IPC', 'S1-D0-C3_IPC']\n",
    "\n",
    "def prepare_dataframes_from_csv(filename:str) -> pandas.DataFrame:\n",
    "    pd_df_raw_data = pandas.read_csv(filename)\n",
    "    # Clean up the byte fields\n",
    "    for fields in cols_bytedata_fields:\n",
    "        pd_df_raw_data[fields] = pd_df_raw_data[fields].apply(lambda x: float(x[2:-1]) if isinstance(x, str) and x.startswith(\"b'\") else x)\n",
    "\n",
    "    # Drop all the irrelevant fields\n",
    "    pd_df_raw_data.drop(pd_df_raw_data.columns.difference(cols_filter_level1), axis=1, inplace=True)\n",
    "    # Drop all the NA data points\n",
    "    pd_df_raw_data = pd_df_raw_data.dropna()\n",
    "    # display(pd_df_raw_data)\n",
    "\n",
    "    for corespec_data in (cols_corespecific_fields_litc + cols_corespecific_fields_bigc) :\n",
    "        pd_df_raw_data[corespec_data+'_IPC'] = pd_df_raw_data[corespec_data+'_instructions'] / pd_df_raw_data[corespec_data+'_cpu-cycles'] \n",
    "    \n",
    "\n",
    "    pd_df_raw_data['Aggregate_IPC-Little'] = pd_df_raw_data[collist_litc].sum(axis=1)\n",
    "    pd_df_raw_data['Aggregate_IPC-Big'] = pd_df_raw_data[collist_bigc].sum(axis=1)\n",
    "    pd_df_raw_data['Aggregate_IPC'] = pd_df_raw_data[['Aggregate_IPC-Little','Aggregate_IPC-Big']].sum(axis=1)\n",
    "\n",
    "    # Compose final data frame with Aggregate IPC\n",
    "    cols = ['dev_ippwr-ch1-ampere_mA','Aggregate_IPC']\n",
    "    __pd_df_raw_data = pd_df_raw_data[cols]\n",
    "    __pd_df_raw_data= __pd_df_raw_data.sort_values('dev_ippwr-ch1-ampere_mA')\n",
    "\n",
    "    # Normalize the final data frame to appropriately scale the values\n",
    "    scaler = Normalizer()\n",
    "    df_scaled = scaler.fit_transform(__pd_df_raw_data.to_numpy())\n",
    "    df_scaled = pandas.DataFrame(df_scaled, columns=cols)\n",
    "\n",
    "    # display (df_scaled)\n",
    "    return df_scaled\n",
    "\n",
    "def process_workload_datasets(freqval:float, isworkload=True):\n",
    "    idle_glob_path = [\n",
    "            'combined_dataset/02-Idling-PerfSleep/MaxFan/*BigCore*-CPUFreq-'+freqval+'GHz',\n",
    "            'combined_dataset/02-Idling-PerfSleep/NoFan/*BigCore*-CPUFreq-'+freqval+'GHz',\n",
    "            ]\n",
    "    workload_glob_path = [\n",
    "        'combined_dataset/03-Workloads/*BigCore*-CPUFreq-'+freqval+'GHz'\n",
    "        ]\n",
    "    if (isworkload):\n",
    "        glob_path= workload_glob_path\n",
    "    else:\n",
    "        glob_path = idle_glob_path\n",
    "    \n",
    "    total_freqdf = pandas.DataFrame({'dev_ippwr-ch1-ampere_mA':[],'Aggregate_IPC':[]})\n",
    "    for item in glob_path:\n",
    "        listings = glob.glob(item)\n",
    "        for dir in listings:\n",
    "            csvlistings = glob.glob(dir+'/*.csv')\n",
    "            for csv in csvlistings:\n",
    "                df = prepare_dataframes_from_csv(csv)\n",
    "                total_freqdf = pandas.concat([total_freqdf, df])\n",
    "    # display (total_freqdf)\n",
    "    return total_freqdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "big_core_freq = np.linspace(0.2, 2.0, num=19)\n",
    "lit_core_freq = np.linspace(0.2, 1.4, num=13)\n",
    "freqs_list = []\n",
    "m_list = []\n",
    "c_list = []\n",
    "r_squared = []\n",
    "adj_r_squared = []\n",
    "f_statistic = []\n",
    "\n",
    "for freq in big_core_freq:\n",
    "    str_freq = float(\"{:.1f}\".format(freq))\n",
    "    total_freqdf = pandas.DataFrame({'dev_ippwr-ch1-ampere_mA':[],'Aggregate_IPC':[]})\n",
    "    total_freqdf = process_workload_datasets(str(str_freq))\n",
    "    X = sm.add_constant(total_freqdf['Aggregate_IPC']) \n",
    "    model = sm.OLS(total_freqdf['dev_ippwr-ch1-ampere_mA'], X)\n",
    "    result = model.fit()\n",
    "    m,c = result.params\n",
    "    # print ('Freq '+str(str_freq)+': m = '+ str(m) +' , c = '+str(c))\n",
    "    # display(result.summary())\n",
    "\n",
    "    freqs_list.append(freq)\n",
    "    m_list.append(m)\n",
    "    c_list.append(c)\n",
    "    r_squared.append(result.rsquared)\n",
    "    adj_r_squared.append(result.rsquared_adj)\n",
    "    f_statistic.append(result.fvalue)\n",
    "\n",
    "print ('Big Core data based coefficient calcultion')\n",
    "result_df = pandas.DataFrame(list(zip(freqs_list, m_list, c_list, r_squared, adj_r_squared, f_statistic)),\n",
    "                            columns =['Frequency(GHz)','M', 'c', 'OLS R-Squared','Adj. R-squared', 'F-statistic' ])\n",
    "display (result_df)\n",
    "    \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}