Analyse PS S2 correlations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Analyse correlations between PlanetScope and Sentinel-2\n",
    "\n",
    "Load and analyse correlation coefficients between PlanetScope and Sentinel-2 regarding the following variables:\n",
    "\n",
    "* Planet satellite\n",
    "* Land use\n",
    "* Area"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Prepare libraries and plotting environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Plot inline\n",
    "%matplotlib inline\n",
    "# Define figure size\n",
    "plt.rcParams[\"figure.figsize\"] = (10, 8)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Load data"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Correlations files\n",
    "coor_fn_list = {\n",
    "    1: \"./data/Izola_coeff.csv\",\n",
    "    2: \"./data/Jesenice_coeff.csv\",\n",
    "    3: \"./data/Kranj_coeff.csv\",\n",
    "    4: \"./data/Radenci_coeff.csv\",\n",
    "}\n",
    "\n",
    "# Aggregated filename\n",
    "corr_df_fn = \"./data/corr_df.csv\""
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Load all files\n",
    "li = []\n",
    "for area in coor_fn_list.keys():\n",
    "    df = pd.read_csv(\n",
    "        coor_fn_list[area],\n",
    "        index_col=0,\n",
    "        dtype={\"band\": \"int\"},\n",
    "        parse_dates=[\"PS_Date\", \"S2_Date\"],\n",
    "    )\n",
    "    df[\"Area\"] = area\n",
    "    li.append(df)\n",
    "corr_df = pd.concat(li, axis=0, ignore_index=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "corr_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "corr_df.dtypes"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Split PS_filename at _ and create new column PS_Sat from the second part\n",
    "corr_df[\"PS_Sat\"] = corr_df[\"PS_filename\"].str.split(\"_\").str[2]\n",
    "corr_df[\"PS_Sat\"] = corr_df[\"PS_Sat\"].astype(\"category\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Split S2_filename at _ and create new column S2_Sat from the second part\n",
    "corr_df[\"S2_Sat\"] = corr_df[\"S2_filename\"].str.split(\"_\").str[1]\n",
    "corr_df[\"S2_Sat\"] = corr_df[\"S2_Sat\"].astype(\"category\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "corr_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Save dataframe\n",
    "corr_df.to_csv(corr_df_fn)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Analyse number of PS and S2 images\n",
    "\n",
    "Analyse number of PS and S2 images, and satellites used for the analysis."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# print length of dataframe\n",
    "print(f\"In total {len(corr_df)} image pairs are used.\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Count unique number of PS_Sat\n",
    "print(f'Number of unique PS satellites: {len(corr_df[\"PS_Sat\"].unique())}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Count unique number of S2_Sat\n",
    "print(f'Number of unique S2 satellites: {len(corr_df[\"S2_Sat\"].unique())}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Group by PS_Sat\n",
    "ps_sat_count = corr_df.groupby(\"PS_Sat\").count()[\"m\"]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Sort\n",
    "ps_sat_count = ps_sat_count.sort_values(ascending=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "ps_sat_count.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Seaborn barplot, order by m\n",
    "sns.barplot(x=ps_sat_count.index, y=ps_sat_count.values, order=ps_sat_count.index)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Plot number of PS images per satellite\n",
    "ps_sat_count.plot(drawstyle=\"steps-post\")\n",
    "plt.title(\"Number of PS images per satellite\")\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# PLot histogram of S2_Sat\n",
    "sns.histplot(corr_df[\"S2_Sat\"])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Correlation analysis\n",
    "\n",
    "Use only selected columns for analysis."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Use only Area, PS_Date, band, PS_sat, and m\n",
    "corr_sub_df = corr_df[[\"Area\", \"PS_Date\", \"band\", \"PS_Sat\", \"m\"]]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "corr_sub_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Plot correlation coefficients by time and band"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Iterate over all bands\n",
    "for band in corr_sub_df[\"band\"].unique():\n",
    "    # Select only band\n",
    "    band_df = corr_sub_df[corr_sub_df[\"band\"] == band]\n",
    "    # Aggregate by week\n",
    "    band_df_agg = band_df.groupby([pd.Grouper(key=\"PS_Date\", freq=\"W\")]).mean()\n",
    "    band_df_agg[\"m\"].dropna().rolling(window=7).mean().plot(label=f\"Band {band}\")\n",
    "plt.legend()\n",
    "plt.title(\"Correlation coefficients by time and band by week\")\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Iterate over all bands\n",
    "for band in corr_sub_df[\"band\"].unique():\n",
    "    # Select only band\n",
    "    band_df = corr_sub_df[corr_sub_df[\"band\"] == band]\n",
    "    # Aggregate by week\n",
    "    band_df_agg = band_df.groupby([pd.Grouper(key=\"PS_Date\", freq=\"Q\")]).mean()\n",
    "    band_df_agg[\"m\"].plot(label=f\"Band {band}\")\n",
    "plt.legend()\n",
    "plt.title(\"Correlation coefficients by time and band by quarter\")\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Plot correlation coefficients by area, time and band"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Plot PS_Date vs m, for each area and band\n",
    "for area in np.unique(corr_sub_df[\"Area\"]):\n",
    "    # Get df for area, drop area column\n",
    "    area_df = corr_sub_df.loc[corr_sub_df[\"Area\"] == area, :].drop(columns=[\"Area\"])\n",
    "    # Convert to wide\n",
    "    df = pd.pivot_table(area_df, index=[\"PS_Date\"], columns=\"band\", values=\"m\")\n",
    "    # Plot\n",
    "    fig, ax = plt.subplots(figsize=(10, 10))\n",
    "    df.plot(ax=ax, alpha=0.5, marker=\"o\")\n",
    "    ax.set_title(f\"Area {area}\")\n",
    "    ax.set_xlabel(\"PS_Date\")\n",
    "    ax.set_ylabel(\"m\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Smooth data"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Plot PS_Date vs m, for each area and band\n",
    "for area in np.unique(corr_sub_df[\"Area\"]):\n",
    "    # Get df for area, drop area column\n",
    "    area_df = corr_sub_df.loc[corr_sub_df[\"Area\"] == area, :].drop(columns=[\"Area\"])\n",
    "    # Convert to wide\n",
    "    df = pd.pivot_table(area_df, index=[\"PS_Date\"], columns=\"band\", values=\"m\")\n",
    "    # Smooth df by rolling mean\n",
    "    df_smooth = df.rolling(window=21).mean()\n",
    "    # Plot\n",
    "    fig, ax = plt.subplots(figsize=(10, 10))\n",
    "    df_smooth.plot(ax=ax, alpha=0.5, marker=\"o\")\n",
    "    ax.set_title(f\"Area {area}\")\n",
    "    ax.set_xlabel(\"PS_Date\")\n",
    "    ax.set_ylabel(\"m\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Mean m by band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "corr_sub_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Compute mean m by band, ignore Area\n",
    "mean_m_by_band = corr_sub_df.groupby(\"band\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Drop Area column\n",
    "mean_m_by_band = mean_m_by_band.drop(columns=[\"Area\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "mean_m_by_band.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Plot mean m by band\n",
    "fig, ax = plt.subplots(figsize=(10, 10))\n",
    "mean_m_by_band.plot(ax=ax)\n",
    "ax.set_title(\"Mean m by band\")\n",
    "ax.set_xlabel(\"Band\")\n",
    "ax.set_ylabel(\"m\")\n",
    "# No legend\n",
    "ax.legend().set_visible(False)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Create violin plot\n",
    "sns.violinplot(data=corr_sub_df, x=\"band\", y=\"m\")\n",
    "plt.ylim(-0.00002, 0.0001)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Statistics of m by area and band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Histogram of Area\n",
    "fig, ax = plt.subplots()\n",
    "ax = sns.countplot(data=corr_sub_df, x=\"Area\")\n",
    "ax.set_xlabel(\"Area\")\n",
    "ax.set_ylabel(\"Count\")\n",
    "plt.xticks([0, 1, 2, 3], [\"Izola\", \"Jesenice\", \"Kranj\", \"Radenci\"])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "corr_sub_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Compute mean m by area and band\n",
    "mean_m_by_area_band = corr_sub_df.groupby([\"band\", \"Area\"]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "mean_m_by_area_band.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# To wide\n",
    "mean_m_by_area_band_wide = mean_m_by_area_band.unstack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "mean_m_by_area_band_wide.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Plot mean m by band\n",
    "fig, ax = plt.subplots(figsize=(10, 10))\n",
    "mean_m_by_area_band_wide.plot(ax=ax)\n",
    "ax.set_title(\"Mean m by band and area\")\n",
    "ax.set_xlabel(\"Area\")\n",
    "ax.set_ylabel(\"m\")\n",
    "plt.xticks([0, 1, 2, 3], [\"Izola\", \"Jesenice\", \"Kranj\", \"Radenci\"])\n",
    "# No legend\n",
    "ax.legend().set_visible(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Create violin plot\n",
    "sns.violinplot(data=corr_sub_df, x=\"band\", y=\"m\", hue=\"Area\")\n",
    "plt.ylim(-0.00002, 0.0001)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b2306d0452ba168735097b5b13681ae1daaf363bd41c0f176d48971c21425a4f"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}