harmonization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This file shall start with an input csv with the schema of full_data_correct_cols_vi.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "version=\"5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>00:00:00</th>\n",
       "      <th>01:00:00</th>\n",
       "      <th>02:00:00</th>\n",
       "      <th>03:00:00</th>\n",
       "      <th>04:00:00</th>\n",
       "      <th>05:00:00</th>\n",
       "      <th>06:00:00</th>\n",
       "      <th>07:00:00</th>\n",
       "      <th>08:00:00</th>\n",
       "      <th>09:00:00</th>\n",
       "      <th>...</th>\n",
       "      <th>17:00:00</th>\n",
       "      <th>18:00:00</th>\n",
       "      <th>19:00:00</th>\n",
       "      <th>20:00:00</th>\n",
       "      <th>21:00:00</th>\n",
       "      <th>22:00:00</th>\n",
       "      <th>23:00:00</th>\n",
       "      <th>ID</th>\n",
       "      <th>Timeseries ID</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001986</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001986</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.003970</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.004864</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003905</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.003971</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001986</td>\n",
       "      <td>Energy_Apiu_1_8_0_BBB6105.kWh</td>\n",
       "      <td>BBB6105</td>\n",
       "      <td>2022-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.353243</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>0.342391</td>\n",
       "      <td>0.320634</td>\n",
       "      <td>0.349195</td>\n",
       "      <td>0.353760</td>\n",
       "      <td>0.372046</td>\n",
       "      <td>0.367966</td>\n",
       "      <td>0.338227</td>\n",
       "      <td>0.314461</td>\n",
       "      <td>...</td>\n",
       "      <td>0.546875</td>\n",
       "      <td>0.542411</td>\n",
       "      <td>0.374942</td>\n",
       "      <td>0.358033</td>\n",
       "      <td>0.462785</td>\n",
       "      <td>0.306167</td>\n",
       "      <td>0.318604</td>\n",
       "      <td>Energy_Apiu_1_8_0_BBB6065.kWh</td>\n",
       "      <td>BBB6065</td>\n",
       "      <td>2022-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.440781</td>\n",
       "      <td>1.721667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6017.kWh</td>\n",
       "      <td>BBB6017</td>\n",
       "      <td>2022-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.333913</td>\n",
       "      <td>0.358832</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.379157</td>\n",
       "      <td>0.363782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Energy_Apiu_1_8_0_BBB6017.kWh</td>\n",
       "      <td>BBB6017</td>\n",
       "      <td>2022-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.272458</td>\n",
       "      <td>1.250000</td>\n",
       "      <td>1.261905</td>\n",
       "      <td>1.270764</td>\n",
       "      <td>1.214107</td>\n",
       "      <td>1.292249</td>\n",
       "      <td>1.296913</td>\n",
       "      <td>1.353062</td>\n",
       "      <td>1.334171</td>\n",
       "      <td>1.256536</td>\n",
       "      <td>...</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>2.069509</td>\n",
       "      <td>2.034516</td>\n",
       "      <td>2.078655</td>\n",
       "      <td>1.928294</td>\n",
       "      <td>2.081276</td>\n",
       "      <td>1.304400</td>\n",
       "      <td>Energy_Apiu_1_8_0_BBB6028.kWh</td>\n",
       "      <td>BBB6028</td>\n",
       "      <td>2022-01-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   00:00:00  01:00:00  02:00:00  03:00:00  04:00:00  05:00:00  06:00:00  \\\n",
       "0       NaN  0.001986       NaN  0.001986       NaN  0.003970  0.000000   \n",
       "1  0.353243  0.312500  0.342391  0.320634  0.349195  0.353760  0.372046   \n",
       "2       NaN       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "3       NaN       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "4  1.272458  1.250000  1.261905  1.270764  1.214107  1.292249  1.296913   \n",
       "\n",
       "   07:00:00  08:00:00  09:00:00  ...  17:00:00  18:00:00  19:00:00  20:00:00  \\\n",
       "0       NaN  0.004864       NaN  ...  0.003905       NaN       NaN  0.003971   \n",
       "1  0.367966  0.338227  0.314461  ...  0.546875  0.542411  0.374942  0.358033   \n",
       "2       NaN  0.440781  1.721667  ...       NaN       NaN       NaN       NaN   \n",
       "3  0.333913  0.358832       NaN  ...  0.379157  0.363782       NaN       NaN   \n",
       "4  1.353062  1.334171  1.256536  ...  2.250000  2.069509  2.034516  2.078655   \n",
       "\n",
       "   21:00:00  22:00:00  23:00:00                              ID  \\\n",
       "0  0.000000       NaN  0.001986   Energy_Apiu_1_8_0_BBB6105.kWh   \n",
       "1  0.462785  0.306167  0.318604   Energy_Apiu_1_8_0_BBB6065.kWh   \n",
       "2       NaN       NaN       NaN  Energy_Ameno_2_8_0_BBB6017.kWh   \n",
       "3       NaN       NaN       NaN   Energy_Apiu_1_8_0_BBB6017.kWh   \n",
       "4  1.928294  2.081276  1.304400   Energy_Apiu_1_8_0_BBB6028.kWh   \n",
       "\n",
       "   Timeseries ID       Date  \n",
       "0        BBB6105 2022-01-01  \n",
       "1        BBB6065 2022-01-01  \n",
       "2        BBB6017 2022-01-01  \n",
       "3        BBB6017 2022-01-01  \n",
       "4        BBB6028 2022-01-01  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(f'full_data_correct_cols_v{version}.csv', index_col=0)\n",
    "\n",
    "# Convert 'Date' column to datetime type\n",
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to replace values within the 21:00:00 to 05:00:00 interval with 0 for IDs containing the term \"Ameno\" (Only for PV productions)\n",
    "def replace_night_values_with_zero(df_group):\n",
    "    no_sunlight_hours = [\"20:00:00\", \"21:00:00\", \"22:00:00\", \"23:00:00\", \"00:00:00\", \"01:00:00\", \"02:00:00\", \"03:00:00\", \"04:00:00\", \"05:00:00\"]\n",
    "    for col in df_group.columns[:-3]:\n",
    "        if ('Ameno' in df_group['ID'].iloc[0]) and ('BBB6052' in df_group['Timeseries ID'].iloc[0] or 'BBB6062' in df_group['Timeseries ID'].iloc[0])  and col in no_sunlight_hours:\n",
    "            df_group[col].fillna(0, inplace=True)\n",
    "    return df_group\n",
    "\n",
    "# Function to interpolate missing values lasting 2 hours or less during the day (07:00-23:00)\n",
    "def interpolate_short_gaps(df_group):\n",
    "    for col in df_group.columns[:-3]:\n",
    "        # Interpolate missing values using linear interpolation along the entire time series\n",
    "        df_group[col] = df_group[col].interpolate(limit=2, limit_direction='both')\n",
    "    return df_group\n",
    "\n",
    "# Function to fill missing values during the night (00:00:00-06:00:00) with the average of the same day of the past 4 weeks for the same ID\n",
    "def fill_night_gaps(df_group):\n",
    "    night_hours = [\"00:00:00\", \"01:00:00\", \"02:00:00\", \"03:00:00\", \"04:00:00\", \"05:00:00\", \"06:00:00\"]\n",
    "    for col in df_group.columns[:-3]:\n",
    "        if df_group[col].isnull().all() and col in night_hours:\n",
    "            # If all values are missing for the column during night hours\n",
    "            # and there is adjacent data, fill with the average of the same day of the past 4 weeks\n",
    "            adjacent_data = df_group[col].shift(-1).notnull() | df_group[col].shift(1).notnull()\n",
    "            if adjacent_data.any():\n",
    "                end_date = df_group['Date'].iloc[0]\n",
    "                start_date = end_date - timedelta(weeks=4)\n",
    "                previous_weeks = df[df['ID'] == df_group['ID'].iloc[0]]\n",
    "                previous_weeks = previous_weeks[\n",
    "                    (previous_weeks['Date'].dt.weekday == end_date.weekday()) &\n",
    "                    (previous_weeks['Date'].dt.time == end_date.time()) &\n",
    "                    (previous_weeks['Date'] >= start_date) &\n",
    "                    (previous_weeks['Date'] < end_date)\n",
    "                ]\n",
    "                df_group[col].fillna(previous_weeks[col].mean(), inplace=True)\n",
    "\n",
    "    return df_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply operations separately for each ID after resetting the index\n",
    "df_imputed = df.groupby('ID').apply(replace_night_values_with_zero).reset_index(drop=True)\n",
    "df_imputed = df_imputed.groupby('ID').apply(interpolate_short_gaps).reset_index(drop=True)\n",
    "df_imputed = df_imputed.groupby('ID').apply(fill_night_gaps).reset_index(drop=True)\n",
    "\n",
    "# Drop empty DataFrames (corresponding to days removed)\n",
    "df_imputed = df_imputed.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>00:00:00</th>\n",
       "      <th>01:00:00</th>\n",
       "      <th>02:00:00</th>\n",
       "      <th>03:00:00</th>\n",
       "      <th>04:00:00</th>\n",
       "      <th>05:00:00</th>\n",
       "      <th>06:00:00</th>\n",
       "      <th>07:00:00</th>\n",
       "      <th>08:00:00</th>\n",
       "      <th>09:00:00</th>\n",
       "      <th>...</th>\n",
       "      <th>17:00:00</th>\n",
       "      <th>18:00:00</th>\n",
       "      <th>19:00:00</th>\n",
       "      <th>20:00:00</th>\n",
       "      <th>21:00:00</th>\n",
       "      <th>22:00:00</th>\n",
       "      <th>23:00:00</th>\n",
       "      <th>ID</th>\n",
       "      <th>Timeseries ID</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2720</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>3.905642</td>\n",
       "      <td>12.698714</td>\n",
       "      <td>21.544399</td>\n",
       "      <td>31.162299</td>\n",
       "      <td>...</td>\n",
       "      <td>23.994628</td>\n",
       "      <td>14.441627</td>\n",
       "      <td>5.416202</td>\n",
       "      <td>0.999167</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.Wh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-06-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2721</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>3.860530</td>\n",
       "      <td>10.604298</td>\n",
       "      <td>20.021780</td>\n",
       "      <td>28.216426</td>\n",
       "      <td>...</td>\n",
       "      <td>23.811811</td>\n",
       "      <td>14.797325</td>\n",
       "      <td>5.174210</td>\n",
       "      <td>1.104600</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.Wh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-07-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2722</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>4.206939</td>\n",
       "      <td>10.916573</td>\n",
       "      <td>21.115469</td>\n",
       "      <td>30.877382</td>\n",
       "      <td>...</td>\n",
       "      <td>24.277444</td>\n",
       "      <td>13.523329</td>\n",
       "      <td>4.514278</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.Wh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-07-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2723</th>\n",
       "      <td>0.75</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.751291</td>\n",
       "      <td>10.149034</td>\n",
       "      <td>20.239885</td>\n",
       "      <td>29.554858</td>\n",
       "      <td>...</td>\n",
       "      <td>23.497236</td>\n",
       "      <td>13.288151</td>\n",
       "      <td>5.447399</td>\n",
       "      <td>0.833111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.Wh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-07-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2724</th>\n",
       "      <td>0.75</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>3.122557</td>\n",
       "      <td>10.296946</td>\n",
       "      <td>19.648335</td>\n",
       "      <td>29.126942</td>\n",
       "      <td>...</td>\n",
       "      <td>22.892829</td>\n",
       "      <td>13.594143</td>\n",
       "      <td>4.601687</td>\n",
       "      <td>0.867263</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.Wh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-07-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3365</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5996</td>\n",
       "      <td>1.643761</td>\n",
       "      <td>5.378791</td>\n",
       "      <td>8.446211</td>\n",
       "      <td>14.184737</td>\n",
       "      <td>...</td>\n",
       "      <td>25.331957</td>\n",
       "      <td>14.463161</td>\n",
       "      <td>6.464116</td>\n",
       "      <td>1.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.kWh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-06-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3366</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>3.990646</td>\n",
       "      <td>12.111817</td>\n",
       "      <td>21.460000</td>\n",
       "      <td>30.257409</td>\n",
       "      <td>...</td>\n",
       "      <td>22.764389</td>\n",
       "      <td>13.920706</td>\n",
       "      <td>4.346685</td>\n",
       "      <td>0.967208</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.kWh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-06-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3367</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>3.264897</td>\n",
       "      <td>10.992416</td>\n",
       "      <td>15.175821</td>\n",
       "      <td>22.857445</td>\n",
       "      <td>...</td>\n",
       "      <td>0.012690</td>\n",
       "      <td>0.006218</td>\n",
       "      <td>0.001314</td>\n",
       "      <td>0.000500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.kWh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-06-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3368</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.002392</td>\n",
       "      <td>0.010102</td>\n",
       "      <td>0.019782</td>\n",
       "      <td>0.030842</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013139</td>\n",
       "      <td>0.010756</td>\n",
       "      <td>0.004000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.kWh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-06-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3369</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.002800</td>\n",
       "      <td>0.010420</td>\n",
       "      <td>0.017684</td>\n",
       "      <td>0.031111</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013139</td>\n",
       "      <td>0.010756</td>\n",
       "      <td>0.004000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Energy_Ameno_2_8_0_BBB6052.kWh</td>\n",
       "      <td>BBB6052</td>\n",
       "      <td>2022-06-29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>263 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      00:00:00  01:00:00  02:00:00  03:00:00  04:00:00  05:00:00  06:00:00  \\\n",
       "2720      0.00       0.0       0.0       0.0       0.0    0.0000  3.905642   \n",
       "2721      0.00       0.0       0.0       0.0       0.0    0.0000  3.860530   \n",
       "2722      0.00       0.0       0.0       0.0       0.0    0.0000  4.206939   \n",
       "2723      0.75       0.0       0.0       0.0       0.0    0.0000  2.751291   \n",
       "2724      0.75       0.0       0.0       0.0       0.0    0.0000  3.122557   \n",
       "...        ...       ...       ...       ...       ...       ...       ...   \n",
       "3365      0.00       0.0       0.0       0.0       0.0    0.5996  1.643761   \n",
       "3366      0.00       0.0       0.0       0.0       0.0    0.0000  3.990646   \n",
       "3367      0.00       0.0       0.0       0.0       0.0    0.0000  3.264897   \n",
       "3368      0.00       0.0       0.0       0.0       0.0    0.0000  0.002392   \n",
       "3369      0.00       0.0       0.0       0.0       0.0    0.0000  0.002800   \n",
       "\n",
       "       07:00:00   08:00:00   09:00:00  ...   17:00:00   18:00:00  19:00:00  \\\n",
       "2720  12.698714  21.544399  31.162299  ...  23.994628  14.441627  5.416202   \n",
       "2721  10.604298  20.021780  28.216426  ...  23.811811  14.797325  5.174210   \n",
       "2722  10.916573  21.115469  30.877382  ...  24.277444  13.523329  4.514278   \n",
       "2723  10.149034  20.239885  29.554858  ...  23.497236  13.288151  5.447399   \n",
       "2724  10.296946  19.648335  29.126942  ...  22.892829  13.594143  4.601687   \n",
       "...         ...        ...        ...  ...        ...        ...       ...   \n",
       "3365   5.378791   8.446211  14.184737  ...  25.331957  14.463161  6.464116   \n",
       "3366  12.111817  21.460000  30.257409  ...  22.764389  13.920706  4.346685   \n",
       "3367  10.992416  15.175821  22.857445  ...   0.012690   0.006218  0.001314   \n",
       "3368   0.010102   0.019782   0.030842  ...   0.013139   0.010756  0.004000   \n",
       "3369   0.010420   0.017684   0.031111  ...   0.013139   0.010756  0.004000   \n",
       "\n",
       "      20:00:00  21:00:00  22:00:00  23:00:00                              ID  \\\n",
       "2720  0.999167       0.0       0.0       0.0   Energy_Ameno_2_8_0_BBB6052.Wh   \n",
       "2721  1.104600       0.0       0.0       0.0   Energy_Ameno_2_8_0_BBB6052.Wh   \n",
       "2722  0.375000       0.0       0.0       0.0   Energy_Ameno_2_8_0_BBB6052.Wh   \n",
       "2723  0.833111       0.0       0.0       0.0   Energy_Ameno_2_8_0_BBB6052.Wh   \n",
       "2724  0.867263       0.0       0.0       0.0   Energy_Ameno_2_8_0_BBB6052.Wh   \n",
       "...        ...       ...       ...       ...                             ...   \n",
       "3365  1.090909       0.0       0.0       0.0  Energy_Ameno_2_8_0_BBB6052.kWh   \n",
       "3366  0.967208       0.0       0.0       0.0  Energy_Ameno_2_8_0_BBB6052.kWh   \n",
       "3367  0.000500       0.0       0.0       0.0  Energy_Ameno_2_8_0_BBB6052.kWh   \n",
       "3368  0.000000       0.0       0.0       0.0  Energy_Ameno_2_8_0_BBB6052.kWh   \n",
       "3369  0.000000       0.0       0.0       0.0  Energy_Ameno_2_8_0_BBB6052.kWh   \n",
       "\n",
       "      Timeseries ID       Date  \n",
       "2720        BBB6052 2022-06-30  \n",
       "2721        BBB6052 2022-07-01  \n",
       "2722        BBB6052 2022-07-02  \n",
       "2723        BBB6052 2022-07-03  \n",
       "2724        BBB6052 2022-07-04  \n",
       "...             ...        ...  \n",
       "3365        BBB6052 2022-06-25  \n",
       "3366        BBB6052 2022-06-26  \n",
       "3367        BBB6052 2022-06-27  \n",
       "3368        BBB6052 2022-06-28  \n",
       "3369        BBB6052 2022-06-29  \n",
       "\n",
       "[263 rows x 27 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_imputed[df_imputed[\"Timeseries ID\"]==\"BBB6052\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_imputed.to_csv(f\"full_data_harmonized_v{version}.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}