mlbaseline.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1ksndBerUYL_"
      },
      "source": [
        "# Crop Yield Prediction - ML Baseline\n",
        "\n",
        "We use WOFOST crop growth indicators, weather variables, geographic information, soil data and remote sensing indicators to predict the yield."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RX-t0S8wUr5X"
      },
      "source": [
        "## Google Colab Notes\n",
        "\n",
        "**To run the script in Google Colab environment**\n",
        "1. Download the data directory and save it somewhere convenient.\n",
        "2. Open the notebook using Google Colaboratory.\n",
        "3. Create a copy of the notebook for yourself.\n",
        "4. Click connect on the right hand side of the bar below menu items. When you are connected to a machine, you will see a green tick mark and bars showing RAM and disk.\n",
        "5. Click the folder icon on the left sidebar and click upload. Upload the data files you downloaded. Click *Ok* when you see a warning saying the files will be deleted after the session is disconnected.\n",
        "6. Use *Runtime* -> *Run before* option to run all cells before **Set Configuration**.\n",
        "7. Run the remaining cells except **Python Script Main**. The configuration subsection allows you to change configuration and rerun experiments.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vP2t4BbJGJKZ"
      },
      "source": [
        "## Global Variables and Spark Installation/Initialization\n",
        "\n",
        "Initialize Spark session and global variables. Package installation is required only in Google Colab."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xcalFxFgYk6s",
        "outputId": "5ba2701d-dfce-4ff7-c755-e9550ce747ac"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "WARNING: apt does not have a stable CLI interface. Use with caution in scripts.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyspark/sql/context.py:79: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n",
            "  FutureWarning\n"
          ]
        }
      ],
      "source": [
        "#%%writefile globals.py\n",
        "test_env = 'notebook'\n",
        "# test_env = 'cluster'\n",
        "# test_env = 'pkg'\n",
        "\n",
        "# change to False to skip tests\n",
        "run_tests = False\n",
        "\n",
        "# NUTS levels\n",
        "nuts_levels = ['NUTS' + str(i) for i in range(4)]\n",
        "\n",
        "# country codes\n",
        "countries = ['BG', 'DE', 'ES', 'FR', 'HU', 'IT', 'NL', 'PL', 'RO']\n",
        "\n",
        "# debug levels\n",
        "debug_levels = [i for i in range(5)]\n",
        "\n",
        "# Keeping these two mappings inside CYPConfiguration leads to SPARK-5063 error\n",
        "# when lambda functions use them. Therefore, they are defined as globals now.\n",
        "\n",
        "# crop name to id mapping\n",
        "crop_id_dict = {\n",
        "    'grain maize': 2,\n",
        "    'sugar beet' : 6,\n",
        "    'sugarbeet' : 6,\n",
        "    'sugarbeets' : 6,\n",
        "    'sugar beets' : 6,\n",
        "    'total potatoes' : 7,\n",
        "    'potatoes' : 7,\n",
        "    'potato' : 7,\n",
        "    'winter wheat' : 90,\n",
        "    'soft wheat' : 90,\n",
        "    'sunflower' : 93,\n",
        "    'spring barley' : 95,\n",
        "}\n",
        "\n",
        "# crop id to name mapping\n",
        "crop_name_dict = {\n",
        "    2 : 'grain maize',\n",
        "    6 : 'sugarbeet',\n",
        "    7 : 'potatoes',\n",
        "    90 : 'soft wheat',\n",
        "    93 : 'sunflower',\n",
        "    95 : 'spring barley',\n",
        "}\n",
        "\n",
        "if (test_env == 'notebook'):\n",
        "  !pip install pyspark > /dev/null\n",
        "  !sudo apt update > /dev/null\n",
        "  !apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "  !pip install joblibspark > /dev/null\n",
        "\n",
        "  import os\n",
        "  os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "\n",
        "import pyspark\n",
        "from pyspark.sql import functions as SparkF\n",
        "from pyspark.sql import types as SparkT\n",
        "\n",
        "from pyspark import SparkContext\n",
        "from pyspark import SparkConf\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql import SQLContext\n",
        "\n",
        "SparkContext.setSystemProperty('spark.executor.memory', '12g')\n",
        "SparkContext.setSystemProperty('spark.driver.memory', '6g')\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()\n",
        "spark.conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n",
        "\n",
        "sc = SparkContext.getOrCreate()\n",
        "sqlContext = SQLContext(sc)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kXT68GaJ1jOc"
      },
      "source": [
        "## Utility Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gZtsU16F1jOn"
      },
      "outputs": [],
      "source": [
        "#%%writefile util.py\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# crop name and id mappings\n",
        "def cropNameToID(crop_id_dict, crop):\n",
        "  \"\"\"\n",
        "  Return id of given crop. Relies on crop_id_dict.\n",
        "  Return 0 if crop name is not in the dictionary.\n",
        "  \"\"\"\n",
        "  crop_lcase = crop.lower()\n",
        "  try:\n",
        "    crop_id = crop_id_dict[crop_lcase]\n",
        "  except KeyError as e:\n",
        "    crop_id = 0\n",
        "\n",
        "  return crop_id\n",
        "\n",
        "def cropIDToName(crop_name_dict, crop_id):\n",
        "  \"\"\"\n",
        "  Return crop name for given crop ID. Relies on crop_name_dict.\n",
        "  Return 'NA' if crop id is not found in the dictionary.\n",
        "  \"\"\"\n",
        "  try:\n",
        "    crop_name = crop_name_dict[crop_id]\n",
        "  except KeyError as e:\n",
        "    crop_name = 'NA'\n",
        "\n",
        "  return crop_name\n",
        "\n",
        "def getYear(date_str):\n",
        "  \"\"\"Extract year from date in yyyyMMdd or dd/MM/yyyy format.\"\"\"\n",
        "  return SparkF.when(SparkF.length(date_str) == 8,\n",
        "                     SparkF.year(SparkF.to_date(date_str, 'yyyyMMdd')))\\\n",
        "                     .otherwise(SparkF.year(SparkF.to_date(date_str, 'dd/MM/yyyy')))\n",
        "\n",
        "def getMonth(date_str):\n",
        "  \"\"\"Extract month from date in yyyyMMdd or dd/MM/yyyy format.\"\"\"\n",
        "  return SparkF.when(SparkF.length(date_str) == 8,\n",
        "                     SparkF.month(SparkF.to_date(date_str, 'yyyyMMdd')))\\\n",
        "                     .otherwise(SparkF.month(SparkF.to_date(date_str, 'dd/MM/yyyy')))\n",
        "\n",
        "def getDay(date_str):\n",
        "  \"\"\"Extract day from date in yyyyMMdd or dd/MM/yyyy format.\"\"\"\n",
        "  return SparkF.when(SparkF.length(date_str) == 8,\n",
        "                     SparkF.dayofmonth(SparkF.to_date(date_str, 'yyyyMMdd')))\\\n",
        "                     .otherwise(SparkF.dayofmonth(SparkF.to_date(date_str, 'dd/MM/yyyy')))\n",
        "\n",
        "# 1-10: Dekad 1\n",
        "# 11-20: Dekad 2\n",
        "# > 20 : Dekad 3\n",
        "def getDekad(date_str):\n",
        "  \"\"\"Extract dekad from date in YYYYMMDD format.\"\"\"\n",
        "  month = getMonth(date_str)\n",
        "  day = getDay(date_str)\n",
        "  return SparkF.when(day < 30, (month - 1)* 3 +\n",
        "                     SparkF.ceil(day/10)).otherwise((month - 1) * 3 + 3)\n",
        "\n",
        "# Machine Learning Utility Functions\n",
        "\n",
        "# This definition is from the suggested answer to:\n",
        "# https://stats.stackexchange.com/questions/58391/mean-absolute-percentage-error-mape-in-scikit-learn/294069#294069\n",
        "def mean_absolute_percentage_error(Y_true, Y_pred):\n",
        "  \"\"\"Mean Absolute Percentage Error\"\"\"\n",
        "  Y_true, Y_pred = np.array(Y_true), np.array(Y_pred)\n",
        "  return np.mean(np.abs((Y_true - Y_pred) / Y_true)) * 100\n",
        "\n",
        "def printFeatures(features, indices, log_fh=None):\n",
        "  \"\"\"Print names of features in groups of 5\"\"\"\n",
        "  num_features = len(indices)\n",
        "  groups = int(num_features/5) + 1\n",
        "\n",
        "  feature_str = '\\n'\n",
        "  for g in range(groups):\n",
        "    group_start = g * 5\n",
        "    group_end = (g + 1) * 5\n",
        "    if (group_end > num_features):\n",
        "      group_end = num_features\n",
        "\n",
        "    group_indices = indices[group_start:group_end]\n",
        "    for idx in group_indices:\n",
        "      if (idx == group_indices[-1]):\n",
        "        feature_str += str(idx+1) + ': ' + features[idx]\n",
        "      else:\n",
        "        feature_str += str(idx+1) + ': ' + features[idx] + ', '\n",
        "\n",
        "    feature_str += '\\n'\n",
        "\n",
        "  print(feature_str)\n",
        "  if (log_fh is not None):\n",
        "    log_fh.write(feature_str)\n",
        "\n",
        "def getPredictionScores(Y_true, Y_predicted, metrics):\n",
        "  \"\"\"Get values of metrics for given Y_predicted and Y_true\"\"\"\n",
        "  pred_scores = {}\n",
        "\n",
        "  for met in metrics:\n",
        "    score_function = metrics[met]\n",
        "    met_score = score_function(Y_true, Y_predicted)\n",
        "    # for RMSE, score_function is mean_squared_error, take square root\n",
        "    # normalize RMSE\n",
        "    if (met == 'RMSE'):\n",
        "      met_score = np.round(100*np.sqrt(met_score)/np.mean(Y_true), 2)\n",
        "      pred_scores['NRMSE'] = met_score\n",
        "    # normalize mean absolute errors except MAPE which is already a percentage\n",
        "    elif ((met == 'MAE') or (met == 'MdAE')):\n",
        "      met_score = np.round(100*met_score/np.mean(Y_true), 2)\n",
        "      pred_scores['N' + met] = met_score\n",
        "    # MAPE, R2, ... : no postprocessing\n",
        "    else:\n",
        "      met_score = np.round(met_score, 2)\n",
        "      pred_scores[met] = met_score\n",
        "\n",
        "  return pred_scores\n",
        "\n",
        "def getFilename(crop, country, yield_trend,\n",
        "                early_season, early_season_end, nuts_level=None):\n",
        "  \"\"\"Get filename based on input arguments\"\"\"\n",
        "  suffix = crop.replace(' ', '_')\n",
        "  suffix += '_' + country\n",
        "\n",
        "  if (nuts_level is not None):\n",
        "    suffix += '_' + nuts_level\n",
        "\n",
        "  if (yield_trend):\n",
        "    suffix += '_trend'\n",
        "  else:\n",
        "    suffix += '_notrend'\n",
        "\n",
        "  if (early_season):\n",
        "    suffix += '_early' + str(early_season_end)\n",
        "\n",
        "  return suffix\n",
        "\n",
        "def getLogFilename(crop, country, yield_trend,\n",
        "                   early_season, early_season_end):\n",
        "  \"\"\"Get filename for experiment log\"\"\"\n",
        "  log_file = getFilename(crop, country, yield_trend,\n",
        "                         early_season, early_season_end)\n",
        "  return log_file + '.log'\n",
        "\n",
        "def getFeatureFilename(crop, country, yield_trend,\n",
        "                       early_season, early_season_end):\n",
        "  \"\"\"Get unique filename for features\"\"\"\n",
        "  feature_file = 'ft_'\n",
        "  suffix = getFilename(crop, country, yield_trend, early_season, early_season_end)\n",
        "  feature_file += suffix\n",
        "  return feature_file\n",
        "\n",
        "def getPredictionFilename(crop, country, nuts_level, yield_trend,\n",
        "                          early_season, early_season_end):\n",
        "  \"\"\"Get unique filename for predictions\"\"\"\n",
        "  pred_file = 'pred_'\n",
        "  suffix = getFilename(crop, country, yield_trend,\n",
        "                       early_season, early_season_end, nuts_level)\n",
        "  pred_file += suffix\n",
        "  return pred_file\n",
        "\n",
        "def plotTrend(years, actual_values, trend_values, trend_label):\n",
        "  \"\"\"Plot a linear trend and scatter plot of actual values\"\"\"\n",
        "  plt.scatter(years, actual_values, color=\"blue\", marker=\"o\")\n",
        "  plt.plot(years, trend_values, '--')\n",
        "  plt.xticks(np.arange(years[0], years[-1] + 1, step=len(years)/5))\n",
        "  ax = plt.axes()\n",
        "  plt.xlabel(\"YEAR\")\n",
        "  plt.ylabel(trend_label)\n",
        "  plt.title(trend_label + ' Trend by YEAR')\n",
        "  plt.show()\n",
        "\n",
        "def plotTrueVSPredicted(actual, predicted):\n",
        "  \"\"\"Plot actual and predicted values\"\"\"\n",
        "  fig, ax = plt.subplots()\n",
        "  ax.scatter(np.asarray(actual), predicted)\n",
        "  ax.plot([actual.min(), actual.max()], [actual.min(), actual.max()], 'k--', lw=4)\n",
        "  ax.set_xlabel('Actual')\n",
        "  ax.set_ylabel('Predicted')\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l8UktClOS9Hs"
      },
      "source": [
        "## Configuration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cjP5SjODS-qS"
      },
      "outputs": [],
      "source": [
        "#%%writefile config.py\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.linear_model import Lasso\n",
        "from sklearn.linear_model import Ridge\n",
        "from sklearn.svm import SVR\n",
        "from sklearn.ensemble import ExtraTreesRegressor\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from sklearn.ensemble import GradientBoostingRegressor\n",
        "from sklearn.neighbors import KNeighborsRegressor\n",
        "from sklearn.neural_network import MLPRegressor\n",
        "\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.feature_selection import SelectKBest\n",
        "from sklearn.feature_selection import mutual_info_regression\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "from sklearn.feature_selection import RFE\n",
        "\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.preprocessing import RobustScaler\n",
        "\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.metrics import mean_absolute_error\n",
        "from sklearn.metrics import r2_score\n",
        "from sklearn.metrics import explained_variance_score\n",
        "from sklearn.metrics import median_absolute_error\n",
        "\n",
        "class CYPConfiguration:\n",
        "  def __init__(self, crop_name='potatoes', country_code='NL', season_cross='N'):\n",
        "    self.config = {\n",
        "        'crop_name' : crop_name,\n",
        "        'crop_id' : cropNameToID(crop_id_dict, crop_name),\n",
        "        'season_crosses_calendar_year' : season_cross,\n",
        "        'country_code' : country_code,\n",
        "        'nuts_level' : 'NUTS2',\n",
        "        'data_sources' : [ 'WOFOST', 'METEO_DAILY', 'SOIL', 'YIELD' ],\n",
        "        'use_yield_trend' : 'N',\n",
        "        'predict_yield_residuals' : 'N',\n",
        "        'find_optimal_trend_window' : 'N',\n",
        "        # set it to a list with one entry for fixed window\n",
        "        'trend_windows' : [5, 7, 10],\n",
        "        'use_centroids' : 'N',\n",
        "        'use_remote_sensing' : 'Y',\n",
        "        'early_season_prediction' : 'N',\n",
        "        'early_season_end_dekad' : 0,\n",
        "        'data_path' : '.',\n",
        "        'output_path' : '.',\n",
        "        'save_features' : 'N',\n",
        "        'use_saved_features' : 'N',\n",
        "        'save_predictions' : 'Y',\n",
        "        'use_saved_predictions' : 'N',\n",
        "        'compare_with_mcyfs' : 'N',\n",
        "        'debug_level' : 0,\n",
        "    }\n",
        "\n",
        "    # Description of configuration parameters\n",
        "    # This should be in sync with config above\n",
        "    self.config_desc = {\n",
        "        'crop_name' : 'Crop name',\n",
        "        'crop_id' : 'Crop ID',\n",
        "        'season_crosses_calendar_year' : 'Crop growing season crosses calendar year boundary',\n",
        "        'country_code' : 'Country code (e.g. NL)',\n",
        "        'nuts_level' : 'NUTS level for yield prediction',\n",
        "        'data_sources' : 'Input data sources',\n",
        "        'use_yield_trend' : 'Estimate and use yield trend',\n",
        "        'predict_yield_residuals' : 'Predict yield residuals instead of full yield',\n",
        "        'find_optimal_trend_window' : 'Find optimal trend window',\n",
        "        'trend_windows' : 'List of trend window lengths (number of years)',\n",
        "        'use_centroids' : 'Use centroid coordinates and distance to coast',\n",
        "        'use_remote_sensing' : 'Use remote sensing data (FAPAR)',\n",
        "        'early_season_prediction' : 'Predict yield early in the season',\n",
        "        'early_season_end_dekad' : 'Early season end dekad relative to harvest',\n",
        "        'data_path' : 'Path to all input data. Default is current directory.',\n",
        "        'output_path' : 'Path to all output files. Default is current directory.',\n",
        "        'save_features' : 'Save features to a CSV file',\n",
        "        'use_saved_features' : 'Use features from a CSV file',\n",
        "        'save_predictions' : 'Save predictions to a CSV file',\n",
        "        'use_saved_predictions' : 'Use predictions from a CSV file',\n",
        "        'compare_with_mcyfs' : 'Compare predictions with MARS Crop Yield Forecasting System',\n",
        "        'debug_level' : 'Debug level to control amount of debug information',\n",
        "    }\n",
        "\n",
        "    ########### Machine learning configuration ###########\n",
        "    # test fraction\n",
        "    self.test_fraction = 0.3\n",
        "\n",
        "    # scaler\n",
        "    self.scaler = StandardScaler()\n",
        "\n",
        "    # Feature selection algorithms. Initialized in getFeatureSelectors().\n",
        "    self.feature_selectors = {}\n",
        "\n",
        "    # prediction algorithms\n",
        "    self.estimators = {\n",
        "        # linear model\n",
        "        'Ridge' : {\n",
        "            'estimator' : Ridge(alpha=1, random_state=42, max_iter=1000,\n",
        "                                copy_X=True, fit_intercept=True),\n",
        "            'fs_param_grid' : dict(estimator__alpha=[1e-1]),\n",
        "            'param_grid' : dict(estimator__alpha=[1e-5, 1e-2, 1e-1, 0.5, 1, 5, 10])\n",
        "        },\n",
        "        'KNN' : {\n",
        "            'estimator' : KNeighborsRegressor(weights='distance'),\n",
        "            'fs_param_grid' : dict(estimator__n_neighbors=[5]),\n",
        "            'param_grid' : dict(estimator__n_neighbors=[3, 5, 7, 9])\n",
        "        },\n",
        "        # SVM regression\n",
        "        'SVR' : {\n",
        "            'estimator' : SVR(kernel='rbf', gamma='scale', max_iter=-1,\n",
        "                              shrinking=True, tol=0.001),\n",
        "            'fs_param_grid' : dict(estimator__C=[10.0],\n",
        "                                   estimator__epsilon=[0.5]),\n",
        "            'param_grid' : dict(estimator__C=[1e-1, 5e-1, 1.0, 5.0, 10.0, 50.0, 100.0, 200.0],\n",
        "                                estimator__epsilon=[1e-2, 1e-1, 0.5, 1.0, 5.0]),\n",
        "        },\n",
        "        # random forest\n",
        "        #'RF' : {\n",
        "        #    'estimator' : RandomForestRegressor(bootstrap=True, random_state=42,\n",
        "        #                                        oob_score=True, min_samples_leaf=5),\n",
        "        #    'fs_param_grid' : dict(estimator__max_depth=[7],\n",
        "        #                           estimator__n_estimators=[100]),\n",
        "        #    'param_grid' : dict(estimator__max_depth=[5, 7],\n",
        "        #                        estimator__n_estimators=[100, 500])\n",
        "        #},\n",
        "        # extra randomized trees\n",
        "        #'ERT' : {\n",
        "        #    'estimator' : ExtraTreesRegressor(bootstrap=True, random_state=42,\n",
        "        #                                      oob_score=True, min_samples_leaf=5),\n",
        "        #    'fs_param_grid' : dict(estimator__max_depth=[7],\n",
        "        #                           estimator__n_estimators=[100]),\n",
        "        #    'param_grid' : dict(estimator__max_depth=[5, 7],\n",
        "        #                        estimator__n_estimators=[100, 500])\n",
        "        #},\n",
        "        # gradient boosted decision trees\n",
        "        'GBDT' : {\n",
        "            'estimator' : GradientBoostingRegressor(learning_rate=0.01,\n",
        "                                                    subsample=0.8, loss='huber',\n",
        "                                                    min_samples_leaf=5,\n",
        "                                                    random_state=42),\n",
        "            'fs_param_grid' : dict(estimator__max_depth=[5],\n",
        "                                   estimator__n_estimators=[100]),\n",
        "            'param_grid' : dict(estimator__max_depth=[5, 10, 15],\n",
        "                                estimator__n_estimators=[100, 500])\n",
        "        },\n",
        "        #'MLP' : {\n",
        "        #    'estimator' : MLPRegressor(batch_size='auto', learning_rate='adaptive',\n",
        "        #                               solver='sgd', activation='relu',\n",
        "        #                               learning_rate_init=0.01, power_t=0.5,\n",
        "        #                               max_iter=1000, shuffle=True,\n",
        "        #                               random_state=42, tol=0.001,\n",
        "        #                               verbose=False, warm_start=False,\n",
        "        #                               momentum=0.9, nesterovs_momentum=True,\n",
        "        #                               early_stopping=True,\n",
        "        #                               validation_fraction=0.4, beta_1=0.9,\n",
        "        #                               beta_2=0.999, epsilon=1e-08),\n",
        "        #    'fs_param_grid' : dict(estimator__hidden_layer_sizes=[(10, 10), (15,15)],\n",
        "        #                           estimator__alpha=[0.2, 0.3]),\n",
        "        #    'param_grid' : dict(estimator__hidden_layer_sizes=[(10, 10), (15, 15), (20, 20)],\n",
        "        #                        estimator__alpha=[0.1, 0.2, 0.3]),\n",
        "        #},\n",
        "   }\n",
        "\n",
        "    # k-fold validation metric for feature selection\n",
        "    self.fs_cv_metric = 'neg_mean_squared_error'\n",
        "    # k-fold validation metric for training\n",
        "    self.est_cv_metric = 'neg_mean_squared_error'\n",
        "\n",
        "    # Performance evaluation metrics:\n",
        "    # sklearn supports these metrics:\n",
        "    # 'explained_variance', 'max_error', 'neg_mean_absolute_error\n",
        "    # 'neg_mean_squared_error', 'neg_root_mean_squared_error'\n",
        "    # 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2'\n",
        "    self.eval_metrics = {\n",
        "        # EXP_VAR (y_true, y_obs) = 1 - ( var(y_true - y_obs) / var (y_true) )\n",
        "        #'EXP_VAR' : explained_variance_score,\n",
        "        # MAE (y_true, y_obs) = ( 1 / n ) * sum_i-n ( | y_true_i - y_obs_i | )\n",
        "        'MAE' : mean_absolute_error,\n",
        "        # MdAE (y_true, y_obs) = median ( | y_true_1 - y_obs_1 |, | y_true_2 - y_obs_2 |, ... )\n",
        "        #'MdAE' : median_absolute_error,\n",
        "        # MAPE (y_true, y_obs) = ( 1 / n ) * sum_i-n ( ( y_true_i - y_obs_i ) / y_true_i )\n",
        "        'MAPE' : mean_absolute_percentage_error,\n",
        "        # MSE (y_true, y_obs) = ( 1 / n ) * sum_i-n ( y_true_i - y_obs_i )^2\n",
        "        'RMSE' : mean_squared_error,\n",
        "        # R2 (y_true, y_obs) = 1 - ( ( sum_i-n ( y_true_i - y_obs_i )^2 )\n",
        "        #                           / sum_i-n ( y_true_i - mean(y_true) )^2)\n",
        "        'R2' : r2_score,\n",
        "    }\n",
        "\n",
        "  ########### Setters and getters ###########\n",
        "  def setCropName(self, crop_name):\n",
        "    \"\"\"Set the crop name\"\"\"\n",
        "    crop = crop_name.lower()\n",
        "    assert crop in crop_id_dict\n",
        "    self.config['crop_name'] = crop\n",
        "    self.config['crop_id'] = cropNameToID(crop_id_dict, crop)\n",
        "\n",
        "  def getCropName(self):\n",
        "    \"\"\"Return the crop name\"\"\"\n",
        "    return self.config['crop_name']\n",
        "\n",
        "  def setCropID(self, crop_id):\n",
        "    \"\"\"Set the crop ID\"\"\"\n",
        "    assert crop_id in crop_name_dict\n",
        "    self.config['crop_id'] = crop_id\n",
        "    self.config['crop_name'] = cropIDToName(crop_name_dict, crop_id)\n",
        "\n",
        "  def getCropID(self):\n",
        "    \"\"\"Return the crop ID\"\"\"\n",
        "    return self.config['crop_id']\n",
        "\n",
        "  def setSeasonCrossesCalendarYear(self, season_crosses):\n",
        "    \"\"\"Set whether the season crosses calendar year boundary\"\"\"\n",
        "    scross = season_crosses.upper()\n",
        "    assert scross in ['Y', 'N']\n",
        "    self.config['season_crosses_calendar_year'] = scross\n",
        "\n",
        "  def seasonCrossesCalendarYear(self):\n",
        "    \"\"\"Return whether the season crosses calendar year boundary\"\"\"\n",
        "    return (self.config['season_crosses_calendar_year'] == 'Y')\n",
        "\n",
        "  def setCountryCode(self, country_code):\n",
        "    \"\"\"Set the country code\"\"\"\n",
        "    if (country_code is None):\n",
        "      self.config['country_code'] = None\n",
        "    else:\n",
        "      ccode = country_code.upper()\n",
        "      assert len(ccode) == 2\n",
        "      assert ccode in countries\n",
        "      self.config['country_code'] = ccode\n",
        "\n",
        "  def getCountryCode(self):\n",
        "    \"\"\"Return the country code\"\"\"\n",
        "    return self.config['country_code']\n",
        "\n",
        "  def setNUTSLevel(self, nuts_level):\n",
        "    \"\"\"Set the NUTS level\"\"\"\n",
        "    nuts = nuts_level.upper()\n",
        "    assert nuts in nuts_levels\n",
        "    self.config['nuts_level'] = nuts\n",
        "\n",
        "  def getNUTSLevel(self):\n",
        "    \"\"\"Return the NUTS level\"\"\"\n",
        "    return self.config['nuts_level']\n",
        "\n",
        "  def setDataSources(self, data_sources):\n",
        "    \"\"\"Get the data sources\"\"\"\n",
        "    # TODO: some validation\n",
        "    self.config['data_sources'] = data_sources\n",
        "\n",
        "  def updateDataSources(self, data_src, include_src, nuts_level=None):\n",
        "    \"\"\"add or remove data_src from data sources\"\"\"\n",
        "    src_nuts = self.getNUTSLevel()\n",
        "    if (nuts_level is not None):\n",
        "      src_nuts = nuts_level\n",
        "\n",
        "    data_sources = self.config['data_sources']\n",
        "    # no update required\n",
        "    if (((include_src == 'Y') and (data_src in data_sources)) or\n",
        "        ((include_src == 'N') and (data_src not in data_sources))):\n",
        "      return\n",
        "\n",
        "    if (include_src == 'Y'):\n",
        "      if (isinstance(data_sources, dict)):\n",
        "        data_sources[data_src] = src_nuts\n",
        "      else:\n",
        "        data_sources.append(data_src)\n",
        "    else:\n",
        "      if (isinstance(data_sources, dict)):\n",
        "        del data_sources[data_src]\n",
        "      else:\n",
        "        data_sources.remove(data_src)\n",
        "\n",
        "    self.config['data_sources'] = data_sources\n",
        "\n",
        "  def getDataSources(self):\n",
        "    \"\"\"Return the data sources\"\"\"\n",
        "    return self.config['data_sources']\n",
        "\n",
        "  def setUseYieldTrend(self, use_trend):\n",
        "    \"\"\"Set whether to use yield trend\"\"\"\n",
        "    use_yt = use_trend.upper()\n",
        "    assert use_yt in ['Y', 'N']\n",
        "    self.config['use_yield_trend'] = use_yt\n",
        "\n",
        "  def useYieldTrend(self):\n",
        "    \"\"\"Return whether to use yield trend\"\"\"\n",
        "    return (self.config['use_yield_trend'] == 'Y')\n",
        "\n",
        "  def setPredictYieldResiduals(self, pred_res):\n",
        "    \"\"\"Set whether to use predict yield residuals\"\"\"\n",
        "    pred_yres = pred_res.upper()\n",
        "    assert pred_yres in ['Y', 'N']\n",
        "    self.config['predict_yield_residuals'] = pred_yres\n",
        "\n",
        "  def predictYieldResiduals(self):\n",
        "    \"\"\"Return whether to use predict yield residuals\"\"\"\n",
        "    return (self.config['predict_yield_residuals'] == 'Y')\n",
        "\n",
        "  def setFindOptimalTrendWindow(self, find_opt):\n",
        "    \"\"\"Set whether to find optimal trend window for each year\"\"\"\n",
        "    find_otw = find_opt.upper()\n",
        "    assert find_otw in ['Y', 'N']\n",
        "    self.config['find_optimal_trend_window'] = find_otw\n",
        "\n",
        "  def findOptimalTrendWindow(self):\n",
        "    \"\"\"Return whether to find optimal trend window for each year\"\"\"\n",
        "    return (self.config['find_optimal_trend_window'] == 'Y')\n",
        "\n",
        "  def setTrendWindows(self, trend_windows):\n",
        "    \"\"\"Set trend window lengths (years)\"\"\"\n",
        "    assert isinstance(trend_windows, list)\n",
        "    assert len(trend_windows) > 0\n",
        "\n",
        "    # trend windows less than 2 years do not make sense\n",
        "    for tw in trend_windows:\n",
        "      assert tw > 2\n",
        "\n",
        "    self.config['trend_windows'] = trend_windows\n",
        "\n",
        "  def getTrendWindows(self):\n",
        "    \"\"\"Return trend window lengths (years)\"\"\"\n",
        "    return self.config['trend_windows']\n",
        "\n",
        "  def setUseCentroids(self, use_centroids):\n",
        "    \"\"\"Set whether to use centroid coordinates and distance to coast\"\"\"\n",
        "    use_ct = use_centroids.upper()\n",
        "    assert use_ct in ['Y', 'N']\n",
        "    self.config['use_centroids'] = use_ct\n",
        "    self.updateDataSources('CENTROIDS', use_ct)\n",
        "\n",
        "  def useCentroids(self):\n",
        "    \"\"\"Return whether to use centroid coordinates and distance to coast\"\"\"\n",
        "    return (self.config['use_centroids'] == 'Y')\n",
        "\n",
        "  def setUseRemoteSensing(self, use_remote_sensing):\n",
        "    \"\"\"Set whether to use remote sensing data\"\"\"\n",
        "    use_rs = use_remote_sensing.upper()\n",
        "    assert use_rs in ['Y', 'N']\n",
        "    self.config['use_remote_sensing'] = use_rs\n",
        "    self.updateDataSources('REMOTE_SENSING', use_rs, 'NUTS2')\n",
        "\n",
        "  def useRemoteSensing(self):\n",
        "    \"\"\"Return whether to use remote sensing data\"\"\"\n",
        "    return (self.config['use_remote_sensing'] == 'Y')\n",
        "\n",
        "  def setEarlySeasonPrediction(self, early_season):\n",
        "    \"\"\"Set whether to do early season prediction\"\"\"\n",
        "    ep = early_season.upper()\n",
        "    assert ep in ['Y', 'N']\n",
        "    self.config['early_season_prediction'] = ep\n",
        "\n",
        "  def earlySeasonPrediction(self):\n",
        "    \"\"\"Return whether to do early season prediction\"\"\"\n",
        "    return (self.config['early_season_prediction'] == 'Y')\n",
        "\n",
        "  def setEarlySeasonEndDekad(self, end_dekad):\n",
        "    \"\"\"Set early season prediction dekad\"\"\"\n",
        "    dekads_range = [dek for dek in range(1, 37)]\n",
        "    assert end_dekad in dekads_range\n",
        "    self.config['early_season_end_dekad'] = end_dekad\n",
        "\n",
        "  def getEarlySeasonEndDekad(self):\n",
        "    \"\"\"Return early season prediction dekad\"\"\"\n",
        "    return self.config['early_season_end_dekad']\n",
        "\n",
        "  def setDataPath(self, data_path):\n",
        "    \"\"\"Set the data path\"\"\"\n",
        "    # TODO: some validation\n",
        "    self.config['data_path'] = data_path\n",
        "\n",
        "  def getDataPath(self):\n",
        "    \"\"\"Return the data path\"\"\"\n",
        "    return self.config['data_path']\n",
        "\n",
        "  def setOutputPath(self, out_path):\n",
        "    \"\"\"Set the path to output files. TODO: some validation.\"\"\"\n",
        "    self.config['output_path'] = out_path\n",
        "\n",
        "  def getOutputPath(self):\n",
        "    \"\"\"Return the path to output files.\"\"\"\n",
        "    return self.config['output_path']\n",
        "\n",
        "  def setSaveFeatures(self, save_ft):\n",
        "    \"\"\"Set whether to save features in a CSV file\"\"\"\n",
        "    sft = save_ft.upper()\n",
        "    assert sft in ['Y', 'N']\n",
        "    self.config['save_features'] = sft\n",
        "\n",
        "  def saveFeatures(self):\n",
        "    \"\"\"Return whether to save features in a CSV file\"\"\"\n",
        "    return (self.config['save_features'] == 'Y')\n",
        "\n",
        "  def setUseSavedFeatures(self, use_saved):\n",
        "    \"\"\"Set whether to use features from CSV file\"\"\"\n",
        "    saved = use_saved.upper()\n",
        "    assert saved in ['Y', 'N']\n",
        "    self.config['use_saved_features'] = saved\n",
        "\n",
        "  def useSavedFeatures(self):\n",
        "    \"\"\"Return whether to use to use features from CSV file\"\"\"\n",
        "    return (self.config['use_saved_features'] == 'Y')\n",
        "\n",
        "  def setSavePredictions(self, save_pred):\n",
        "    \"\"\"Set whether to save predictions in a CSV file\"\"\"\n",
        "    spd = save_pred.upper()\n",
        "    assert spd in ['Y', 'N']\n",
        "    self.config['save_predictions'] = spd\n",
        "\n",
        "  def savePredictions(self):\n",
        "    \"\"\"Return whether to save predictions in a CSV file\"\"\"\n",
        "    return (self.config['save_predictions'] == 'Y')\n",
        "\n",
        "  def setUseSavedPredictions(self, use_saved):\n",
        "    \"\"\"Set whether to use predictions from CSV file\"\"\"\n",
        "    saved = use_saved.upper()\n",
        "    assert saved in ['Y', 'N']\n",
        "    self.config['use_saved_predictions'] = saved\n",
        "\n",
        "  def useSavedPredictions(self):\n",
        "    \"\"\"Return whether to use to use predictions from CSV file\"\"\"\n",
        "    return (self.config['use_saved_predictions'] == 'Y')\n",
        "\n",
        "  def setCompareWithMCYFS(self, compare_mcyfs):\n",
        "    \"\"\"Set whether to compare predictions with MCYFS\"\"\"\n",
        "    comp_mcyfs = compare_mcyfs.upper()\n",
        "    assert comp_mcyfs in ['Y', 'N']\n",
        "    self.config['compare_with_mcyfs'] = comp_mcyfs\n",
        "\n",
        "  def compareWithMCYFS(self):\n",
        "    \"\"\"Return whether to compare predictions with MCYFS\"\"\"\n",
        "    return (self.config['compare_with_mcyfs'] == 'Y')\n",
        "\n",
        "  def setDebugLevel(self, debug_level):\n",
        "    \"\"\"Set the debug level\"\"\"\n",
        "    assert debug_level in debug_levels\n",
        "    self.config['debug_level'] = debug_level\n",
        "\n",
        "  def getDebugLevel(self):\n",
        "    \"\"\"Return the debug level\"\"\"\n",
        "    return self.config['debug_level']\n",
        "\n",
        "  def updateConfiguration(self, config_update):\n",
        "    \"\"\"Update configuration\"\"\"\n",
        "    assert isinstance(config_update, dict)\n",
        "    for k in config_update:\n",
        "      assert k in self.config\n",
        "\n",
        "      # keys that need special handling\n",
        "      special_cases = {\n",
        "          'crop_name' : self.setCropName,\n",
        "          'crop_id' : self.setCropID,\n",
        "          'use_centroids' : self.setUseCentroids,\n",
        "          'use_remote_sensing' : self.setUseRemoteSensing,\n",
        "      }\n",
        "\n",
        "      if (k not in special_cases):\n",
        "        self.config[k] = config_update[k]\n",
        "        continue\n",
        "\n",
        "      # special case\n",
        "      special_cases[k](config_update[k])\n",
        "\n",
        "  def printConfig(self, log_fh):\n",
        "    \"\"\"Print current configuration and write configuration to log file.\"\"\"\n",
        "    config_str = '\\nCurrent ML Baseline Configuration'\n",
        "    config_str += '\\n--------------------------------'\n",
        "    for k in self.config:\n",
        "      if (isinstance(self.config[k], dict)):\n",
        "        conf_keys = list(self.config[k].keys())\n",
        "        if (not isinstance(conf_keys[0], str)):\n",
        "          conf_keys = [str(k) for k in conf_keys]\n",
        "\n",
        "        config_str += '\\n' + self.config_desc[k] + ': ' + ', '.join(conf_keys)\n",
        "      elif (isinstance(self.config[k], list)):\n",
        "        conf_vals = self.config[k]\n",
        "        if (not isinstance(conf_vals[0], str)):\n",
        "          conf_vals = [str(k) for k in conf_vals]\n",
        "\n",
        "        config_str += '\\n' + self.config_desc[k] + ': ' + ', '.join(conf_vals)\n",
        "      else:\n",
        "        conf_val = self.config[k]\n",
        "        if (not isinstance(conf_val, str)):\n",
        "          conf_val = str(conf_val)\n",
        "\n",
        "        config_str += '\\n' + self.config_desc[k] + ': ' + conf_val\n",
        "\n",
        "    config_str += '\\n'\n",
        "    log_fh.write(config_str + '\\n')\n",
        "    print(config_str)\n",
        "\n",
        "  # Machine learning configuration\n",
        "  def getTestFraction(self):\n",
        "    \"\"\"Return test set fraction (of full dataset)\"\"\"\n",
        "    return self.test_fraction\n",
        "\n",
        "  def setTestFraction(self, test_fraction):\n",
        "    \"\"\"Set test set fraction (of full dataset)\"\"\"\n",
        "    assert (test_fraction > 0.0 and test_fraction < 1.0)\n",
        "    self.test_fraction = test_fraction\n",
        "\n",
        "  def getFeatureScaler(self):\n",
        "    \"\"\"Return feature scaling method\"\"\"\n",
        "    return self.scaler\n",
        "\n",
        "  def setFeatureScaler(self, scaler):\n",
        "    \"\"\"Set feature scaling method\"\"\"\n",
        "    assert (isinstance(scaler, MinMaxScaler) or isinstance(scaler, StandardScaler))\n",
        "    self.scaler = scaler\n",
        "\n",
        "  def getFeatureSelectionCVMetric(self):\n",
        "    \"\"\"Return metric for feature selection using K-fold validation\"\"\"\n",
        "    return self.fs_cv_metric\n",
        "\n",
        "  def setFeatureSelectionCVMetric(self, fs_metric):\n",
        "    \"\"\"Return metric for feature selection using K-fold validation\"\"\"\n",
        "    assert fs_metric in self.eval_metrics\n",
        "    self.fs_cv_metric = fs_metric\n",
        "\n",
        "  def getAlgorithmTrainingCVMetric(self):\n",
        "    \"\"\"Return metric for hyperparameter optimization using K-fold validation\"\"\"\n",
        "    return self.est_cv_metric\n",
        "\n",
        "  def setFeatureSelectionCVMetric(self, est_metric):\n",
        "    \"\"\"Return metric for hyperparameter optimization using K-fold validation\"\"\"\n",
        "    assert est_metric in self.eval_metrics\n",
        "    self.est_cv_metric = est_metric\n",
        "\n",
        "  def getFeatureSelectors(self, X_train, Y_train, num_features,\n",
        "                          custom_cv):\n",
        "    \"\"\"Feature selection methods\"\"\"\n",
        "    # already defined?\n",
        "    if (len(self.feature_selectors) > 0):\n",
        "      return self.feature_selectors\n",
        "\n",
        "    # NOTE: X_train, Y_train, custom_cv\n",
        "    # are for optimizing hyperparamters of rf and lasso used\n",
        "    # to define feature selectors. At the moment, we don't\n",
        "    # optimize hyperparameters.\n",
        "\n",
        "    # Early season prediction can have less than 10 features\n",
        "    min_features = 10 if num_features > 10 else num_features\n",
        "    max_features = [min_features]\n",
        "\n",
        "    if (num_features > 15):\n",
        "      max_features.append(15)\n",
        "    if (num_features > 20):\n",
        "      max_features.append(20)\n",
        "\n",
        "    use_yield_trend = self.useYieldTrend()\n",
        "    if ((num_features > 25) and (use_yield_trend)):\n",
        "      max_features.append(25)\n",
        "\n",
        "    rf = RandomForestRegressor(n_estimators=100, max_depth=5,\n",
        "                               bootstrap=True, random_state=42,\n",
        "                               oob_score=True, min_samples_leaf=5)\n",
        "\n",
        "    lasso = Lasso(alpha=0.1, copy_X=True, fit_intercept=True,\n",
        "                  random_state=42,selection='cyclic', tol=0.01)\n",
        "\n",
        "    self.feature_selectors = {\n",
        "      # random forest\n",
        "      'random_forest' : {\n",
        "          'selector' : SelectFromModel(rf, threshold='median'),\n",
        "          'param_grid' : dict(selector__max_features=max_features)\n",
        "      },\n",
        "      # recursive feature elimination using Lasso\n",
        "      'RFE_Lasso' : {\n",
        "          'selector' : RFE(lasso),\n",
        "          'param_grid' : dict(selector__n_features_to_select=max_features)\n",
        "      },\n",
        "      # NOTE: Mutual info raises an error when used with spark parallel backend.\n",
        "      # univariate feature selection\n",
        "      # 'mutual_info' : {\n",
        "      #     'selector' : SelectKBest(mutual_info_regression),\n",
        "      #     'param_grid' : dict(selector__k=max_features)\n",
        "      # },\n",
        "    }\n",
        "\n",
        "    return self.feature_selectors\n",
        "\n",
        "  def setFeatureSelectors(self, ft_sel):\n",
        "    \"\"\"Set feature selection algorithms\"\"\"\n",
        "    assert isinstance(ft_sel, dict)\n",
        "    assert len(ft_sel) > 0\n",
        "    for sel in ft_sel:\n",
        "      assert isinstance(sel, dict)\n",
        "      assert 'selector' in sel\n",
        "      assert 'param_grid' in sel\n",
        "      # add cases if other feature selection methods are used\n",
        "      assert (isinstance(sel['selector'], SelectKBest) or\n",
        "              isinstance(sel['selector'], SelectFromModel) or\n",
        "              isinstance(sel['selector'], RFE))\n",
        "      assert isinstance(sel['param_grid'], dict)\n",
        "\n",
        "    self.feature_selectors = ft_sel\n",
        "\n",
        "  def getEstimators(self):\n",
        "    \"\"\"Return machine learning algorithms for prediction\"\"\"\n",
        "    return self.estimators\n",
        "  \n",
        "  def setEstimators(self, estimators):\n",
        "    \"\"\"Set machine learning algorithms for prediction\"\"\"\n",
        "    assert isinstance(estimators, dict)\n",
        "    assert len(estimators) > 0\n",
        "    for est in estimators:\n",
        "      assert isinstance(est, dict)\n",
        "      assert 'estimator' in est\n",
        "      assert 'param_grid' in est\n",
        "      assert 'fs_param_grid' in est\n",
        "      assert isinstance(est['param_grid'], dict)\n",
        "      assert isinstance(est['fs_param_grid'], dict)\n",
        "\n",
        "    self.estimators = estimators\n",
        "  \n",
        "  def getEvaluationMetrics(self):\n",
        "    \"\"\"Return metrics to evaluate predictions of algorithms\"\"\"\n",
        "    return self.eval_metrics\n",
        "  \n",
        "  def setEvaluationMetrics(self, metrics):\n",
        "    assert isinstance(metrics, dict)\n",
        "    self.eval_metrics = metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WKLbsRZPj_la"
      },
      "source": [
        "## Workflow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KJOtnzSYQvPu"
      },
      "source": [
        "### Data Loading and Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mDlfa0-RtsA7"
      },
      "source": [
        "#### Data Loader Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nk-0JeNdQvP2"
      },
      "outputs": [],
      "source": [
        "#%%writefile data_loading.py\n",
        "class CYPDataLoader:\n",
        "  def __init__(self, spark, cyp_config):\n",
        "    self.spark = spark\n",
        "    self.data_path = cyp_config.getDataPath()\n",
        "    self.country_code = cyp_config.getCountryCode()\n",
        "    self.nuts_level = cyp_config.getNUTSLevel()\n",
        "    self.data_sources = cyp_config.getDataSources()\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "    self.data_dfs = {}\n",
        "\n",
        "  def loadFromCSVFile(self, data_path, src, nuts, country_code):\n",
        "    \"\"\"\n",
        "    The implied filename for each source is:\n",
        "    <data_source>_<nuts_level>_<country_code>.csv\n",
        "    Examples: CENTROIDS_NUTS2_NL.csv, WOFOST_NUTS2_NL.csv.\n",
        "    Schema is inferred from the file. We might want to specify the schema at some point.\n",
        "    \"\"\"\n",
        "    if (country_code is not None):\n",
        "      datafile = data_path + '/' + src  + '_' + nuts + '_' + country_code + '.csv'\n",
        "    else:\n",
        "      datafile = data_path + '/' + src  + '_' + nuts + '.csv'\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      print('Data file name', '\"' + datafile + '\"')\n",
        "\n",
        "    df = self.spark.read.csv(datafile, header = True, inferSchema = True)\n",
        "    return df\n",
        "\n",
        "  def loadData(self, src, nuts_level):\n",
        "    \"\"\"\n",
        "    Load data for a specific data source.\n",
        "    nuts_level may one level or a list of levels.\n",
        "    \"\"\"\n",
        "    data_path = self.data_path\n",
        "    country_code = self.country_code\n",
        "    assert src in self.data_sources\n",
        "\n",
        "    if (isinstance(nuts_level, list)):\n",
        "      src_dfs = []\n",
        "      for nuts in nuts_level:\n",
        "        df = self.loadFromCSVFile(data_path, src, nuts, country_code)\n",
        "        src_dfs.append(df)\n",
        "\n",
        "    elif (isinstance(nuts_level, str)):\n",
        "      src_dfs = self.loadFromCSVFile(data_path, src, nuts_level, country_code)\n",
        "    else:\n",
        "      src_dfs = None\n",
        "\n",
        "    return src_dfs\n",
        "\n",
        "  def loadAllData(self):\n",
        "    \"\"\"\n",
        "    NOT SUPPORTED:\n",
        "    1. Schema is not defined.\n",
        "    2. Loading data for multiple countries.\n",
        "    3. Loading data from folders.\n",
        "    Ioannis: Spark has a nice way of loading several files from a folder,\n",
        "    and associating the file name on each record, using the function\n",
        "    input_file_name. This allows to extract medatada from the path\n",
        "    into the dataframe. In your case it could be the country name, etc.\n",
        "    \"\"\"\n",
        "    data_dfs = {}\n",
        "    for src in self.data_sources:\n",
        "      nuts_level = self.nuts_level\n",
        "      if (isinstance(self.data_sources, dict)):\n",
        "        nuts_level = self.data_sources[src]\n",
        "      # REMOTE_SENSING data is at NUTS2. If nuts_level is None, leave as is.\n",
        "      # elif ((src == 'REMOTE_SENSING') and (nuts_level is not None)):\n",
        "      #   nuts_level = 'NUTS2'\n",
        "\n",
        "      if ('METEO' in src):\n",
        "        data_dfs['METEO'] = self.loadData(src, nuts_level)\n",
        "      else:\n",
        "        data_dfs[src] = self.loadData(src, nuts_level)\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      data_sources_str = ''\n",
        "      for src in data_dfs:\n",
        "        data_sources_str = data_sources_str + src + ', '\n",
        "\n",
        "      # remove the comma and space from the end\n",
        "      print('Loaded data:', data_sources_str[:-2])\n",
        "      print('\\n')\n",
        "\n",
        "    return data_dfs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "30v2QIEh2K9P"
      },
      "source": [
        "#### Data Preprocessor Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qo2OHq0x2ID1"
      },
      "outputs": [],
      "source": [
        "#%%writefile data_preprocessing.py\n",
        "from pyspark.sql import Window\n",
        "\n",
        "class CYPDataPreprocessor:\n",
        "  def __init__(self, spark, cyp_config):\n",
        "    self.spark = spark\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "\n",
        "  def extractYearDekad(self, df):\n",
        "    \"\"\"Extract year and dekad from date_col in yyyyMMdd format.\"\"\"\n",
        "    # Conversion to string type is required to make getYear(), getMonth() etc. work correctly.\n",
        "    # They use to_date() function to verify valid dates and to_date() expects the date column to be string.\n",
        "    df = df.withColumn('DATE', df['DATE'].cast(\"string\"))\n",
        "    df = df.select('*',\n",
        "                   getYear('DATE').alias('FYEAR'),\n",
        "                   getDekad('DATE').alias('DEKAD'))\n",
        "\n",
        "    # Bring FYEAR, DEKAD to the front\n",
        "    col_order = df.columns[:2] + df.columns[-2:] + df.columns[2:-2]\n",
        "    df = df.select(col_order).drop('DATE')\n",
        "    return df\n",
        "\n",
        "  def getCropSeasonInformation(self, wofost_df, season_crosses_calyear):\n",
        "    \"\"\"Crop season information based on WOFOST DVS\"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    if (('DATE' in wofost_df.columns) and ('FYEAR' not in wofost_df.columns)):\n",
        "      wofost_df = self.extractYearDekad(wofost_df)\n",
        "\n",
        "    crop_season = wofost_df.select(join_cols).distinct()\n",
        "    diff_window = Window.partitionBy(join_cols).orderBy('DEKAD')\n",
        "    cs_window = Window.partitionBy('IDREGION').orderBy('FYEAR')\n",
        "\n",
        "    wofost_df = wofost_df.withColumn('VALUE', wofost_df['DVS'])\n",
        "    wofost_df = wofost_df.withColumn('PREV', SparkF.lag(wofost_df['VALUE']).over(diff_window))\n",
        "    wofost_df = wofost_df.withColumn('DIFF', SparkF.when(SparkF.isnull(wofost_df['PREV']), 0)\\\n",
        "                                     .otherwise(wofost_df['VALUE'] - wofost_df['PREV']))\n",
        "    # calculate end of season dekad\n",
        "    dvs_nochange_filter = ((wofost_df['VALUE'] >= 200) & (wofost_df['DIFF'] == 0.0))\n",
        "    year_end_filter = (wofost_df['DEKAD'] == 36)\n",
        "    if (season_crosses_calyear):\n",
        "      value_zero_filter =  (wofost_df['VALUE'] == 0)\n",
        "    else:\n",
        "      value_zero_filter =  ((wofost_df['PREV'] >= 200) & (wofost_df['VALUE'] == 0))\n",
        "\n",
        "    end_season_filter = (dvs_nochange_filter | value_zero_filter | year_end_filter)\n",
        "    crop_season = crop_season.join(wofost_df.filter(end_season_filter).groupBy(join_cols)\\\n",
        "                                   .agg(SparkF.min('DEKAD').alias('SEASON_END_DEKAD')), join_cols)\n",
        "    wofost_df = wofost_df.drop('VALUE', 'PREV', 'DIFF')\n",
        "\n",
        "    # We take the max of SEASON_END_DEKAD for current campaign and next campaign\n",
        "    # to determine which dekads go to next campaign year.\n",
        "    max_year = crop_season.agg(SparkF.max('FYEAR')).collect()[0][0]\n",
        "    min_year = crop_season.agg(SparkF.min('FYEAR')).collect()[0][0]\n",
        "    crop_season = crop_season.withColumn('NEXT_SEASON_END', SparkF.when(crop_season['FYEAR'] == max_year,\n",
        "                                                                        crop_season['SEASON_END_DEKAD'])\\\n",
        "                                         .otherwise(SparkF.lead(crop_season['SEASON_END_DEKAD']).over(cs_window)))\n",
        "    crop_season = crop_season.withColumn('SEASON_END',\n",
        "                                         SparkF.when(crop_season['SEASON_END_DEKAD'] > crop_season['NEXT_SEASON_END'],\n",
        "                                                     crop_season['SEASON_END_DEKAD'])\\\n",
        "                                         .otherwise(crop_season['NEXT_SEASON_END']))\n",
        "    crop_season = crop_season.withColumn('PREV_SEASON_END', SparkF.when(crop_season['FYEAR'] == min_year, 0)\\\n",
        "                                         .otherwise(SparkF.lag(crop_season['SEASON_END']).over(cs_window)))\n",
        "    crop_season = crop_season.select(join_cols + ['PREV_SEASON_END', 'SEASON_END'])\n",
        "\n",
        "    return crop_season\n",
        "\n",
        "  def alignDataToCropSeason(self, df, crop_season, season_crosses_calyear):\n",
        "    \"\"\"Calculate CAMPAIGN_YEAR, CAMPAIGN_DEKAD based on crop_season\"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    max_year = crop_season.agg(SparkF.max('FYEAR')).collect()[0][0]\n",
        "    min_year = crop_season.agg(SparkF.min('FYEAR')).collect()[0][0]\n",
        "    df = df.join(crop_season, join_cols)\n",
        "\n",
        "    # Dekads > SEASON_END belong to next campaign year\n",
        "    df = df.withColumn('CAMPAIGN_YEAR',\n",
        "                       SparkF.when(df['DEKAD'] > df['SEASON_END'], df['FYEAR'] + 1)\\\n",
        "                       .otherwise(df['FYEAR']))\n",
        "    # min_year has no previous season information. We align CAMPAIGN_DEKAD to end in 36.\n",
        "    # For other years, dekads < SEASON_END are adjusted based on PREV_SEASON_END.\n",
        "    # Dekads > SEASON_END get renumbered from 1 (for next campaign).\n",
        "    df = df.withColumn('CAMPAIGN_DEKAD',\n",
        "                       SparkF.when(df['CAMPAIGN_YEAR'] == min_year, df['DEKAD'] + 36 - df['SEASON_END'])\\\n",
        "                       .otherwise(SparkF.when(df['DEKAD'] > df['SEASON_END'], df['DEKAD'] - df['SEASON_END'])\\\n",
        "                                  .otherwise(df['DEKAD'] + 36 - df['PREV_SEASON_END'])))\n",
        "\n",
        "    # Columns should be IDREGION, FYEAR, DEKAD, ..., CAMPAIGN_YEAR, CAMPAIGN_DEKAD.\n",
        "    # Bring CAMPAIGN_YEAR and CAMPAIGN_DEKAD to the front.\n",
        "    col_order = df.columns[:3] + df.columns[-2:] + df.columns[3:-2]\n",
        "    df = df.select(col_order)\n",
        "    if (season_crosses_calyear):\n",
        "      # For crop with two seasons, remove the first year. Data from the first year\n",
        "      # only contributes to the second year and we have already moved useful data\n",
        "      # to the second year (or first campaign year).\n",
        "      df = df.filter(df['CAMPAIGN_YEAR'] > min_year)\n",
        "\n",
        "    # In both cases, remove extra rows beyond max campaign year\n",
        "    df = df.filter(df['CAMPAIGN_YEAR'] <= max_year)\n",
        "    return df\n",
        "\n",
        "  def preprocessWofost(self, wofost_df, crop_season, season_crosses_calyear):\n",
        "    \"\"\"\n",
        "    Extract year and dekad from date. Use crop_season to compute\n",
        "    CAMPAIGN_YEAR and CAMPAIGN_DEKAD.\n",
        "    \"\"\"\n",
        "    drop_cols = crop_season.columns[2:]\n",
        "    if (('DATE' in wofost_df.columns) and ('FYEAR' not in wofost_df.columns)):\n",
        "      wofost_df = self.extractYearDekad(wofost_df)\n",
        "\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    wofost_df = self.alignDataToCropSeason(wofost_df, crop_season, season_crosses_calyear)\n",
        "\n",
        "    # WOFOST indicators come after IDREGION, FYEAR, DEKAD, CAMPAIGN_YEAR, CAMPAIGN_DEKAD.\n",
        "    wofost_inds = wofost_df.columns[5:]\n",
        "    # set indicators values for dekads after end of season to zero.\n",
        "    # TODO - Dilli: Find a way to avoid the for loop.\n",
        "    for ind in wofost_inds:\n",
        "      wofost_df = wofost_df.withColumn(ind,\n",
        "                                       SparkF.when(wofost_df['DEKAD'] < wofost_df['SEASON_END'],\n",
        "                                                   wofost_df[ind])\\\n",
        "                                       .otherwise(0))\n",
        "\n",
        "    wofost_df = wofost_df.drop(*drop_cols)\n",
        "    return wofost_df\n",
        "\n",
        "  def preprocessMeteo(self, meteo_df, crop_season, season_crosses_calyear):\n",
        "    \"\"\"\n",
        "    Extract year and dekad from date, calculate CWB.\n",
        "    Use crop_season to compute CAMPAIGN_YEAR and CAMPAIGN_DEKAD.\n",
        "    \"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    drop_cols = crop_season.columns[2:]\n",
        "    meteo_df = meteo_df.drop('IDCOVER')\n",
        "    meteo_df = meteo_df.withColumn('CWB',\n",
        "                                   SparkF.bround(meteo_df['PREC'] - meteo_df['ET0'], 2))\n",
        "    if (('DATE' in meteo_df.columns) and ('FYEAR' not in meteo_df.columns)):\n",
        "      meteo_df = self.extractYearDekad(meteo_df)\n",
        "\n",
        "    meteo_df = self.alignDataToCropSeason(meteo_df, crop_season, season_crosses_calyear)\n",
        "    meteo_df = meteo_df.drop(*drop_cols)\n",
        "    return meteo_df\n",
        "\n",
        "  def preprocessMeteoDaily(self, meteo_df):\n",
        "    \"\"\"\n",
        "    Convert daily meteo data to dekadal. Takes avg for all indicators\n",
        "    except TMAX (take max), TMIN (take min), PREC (take sum), ET0 (take sum), CWB (take sum).\n",
        "    \"\"\"\n",
        "    self.spark.catalog.dropTempView('meteo_daily')\n",
        "    meteo_df.createOrReplaceTempView('meteo_daily')\n",
        "    join_cols = ['IDREGION', 'CAMPAIGN_YEAR', 'CAMPAIGN_DEKAD']\n",
        "    join_df = meteo_df.select(join_cols + ['FYEAR', 'DEKAD']).distinct()\n",
        "\n",
        "    # We are ignoring VPRES, WSPD and RELH at the moment\n",
        "    # avg(VPRES) as VPRES1, avg(WSPD) as WSPD1, avg(RELH) as RELH1,\n",
        "    # TMAX| TMIN| TAVG| VPRES| WSPD| PREC| ET0| RAD| RELH| CWB\n",
        "    #\n",
        "    # It seems keeping same name after aggregation is fine. We are using a\n",
        "    # different name just to be sure nothing untoward happens.\n",
        "    query = 'select IDREGION, CAMPAIGN_YEAR, CAMPAIGN_DEKAD, '\n",
        "    query = query + ' max(TMAX) as TMAX1, min(TMIN) as TMIN1, '\n",
        "    query = query + ' bround(avg(TAVG), 2) as TAVG1, bround(sum(PREC), 2) as PREC1, '\n",
        "    query = query + ' bround(sum(ET0), 2) as ET01, bround(avg(RAD), 2) as RAD1, '\n",
        "    query = query + ' bround(sum(CWB), 2) as CWB1 '\n",
        "    query = query + ' from meteo_daily group by IDREGION, CAMPAIGN_YEAR, CAMPAIGN_DEKAD '\n",
        "    query = query + ' order by IDREGION, CAMPAIGN_YEAR, CAMPAIGN_DEKAD'\n",
        "    meteo_df = self.spark.sql(query).cache()\n",
        "\n",
        "    # rename the columns\n",
        "    selected_cols = ['TMAX', 'TMIN', 'TAVG', 'PREC', 'ET0', 'RAD', 'CWB']\n",
        "    for scol in selected_cols:\n",
        "      meteo_df = meteo_df.withColumnRenamed(scol + '1', scol)\n",
        "\n",
        "    meteo_df = meteo_df.join(join_df, join_cols)\n",
        "    # Bring FYEAR, DEKAD to the front\n",
        "    col_order = meteo_df.columns[:1] + meteo_df.columns[-2:] + meteo_df.columns[1:-2]\n",
        "    meteo_df = meteo_df.select(col_order)\n",
        "\n",
        "    return meteo_df\n",
        "\n",
        "  def remoteSensingNUTS2ToNUTS3(self, rs_df, nuts3_regions):\n",
        "    \"\"\"\n",
        "    Convert NUTS2 remote sensing data to NUTS3.\n",
        "    Remote sensing values for NUTS3 regions are inherited from parent regions.\n",
        "    NOTE this function is called before preprocessRemoteSensing.\n",
        "    preprocessRemoteSensing expects crop_season and rs_df to be at the same NUTS level.\n",
        "    \"\"\"\n",
        "    NUTS3_dict = {}\n",
        "\n",
        "    for nuts3 in nuts3_regions:\n",
        "      nuts2 = nuts3[:4]\n",
        "      try:\n",
        "        existing = NUTS3_dict[nuts2]\n",
        "      except KeyError as e:\n",
        "        existing = []\n",
        "\n",
        "      NUTS3_dict[nuts2] = existing + [nuts3]\n",
        "\n",
        "    rs_NUTS3 = rs_df.rdd.map(lambda r: (NUTS3_dict[r[0]] if r[0] in NUTS3_dict else [], r[1], r[2]))\n",
        "    rs_NUTS3_df = rs_NUTS3.toDF(['NUTS3_REG', 'DATE', 'FAPAR'])\n",
        "    rs_NUTS3_df = rs_NUTS3_df.withColumn('IDREGION', SparkF.explode('NUTS3_REG')).drop('NUTS3_REG')\n",
        "    rs_NUTS3_df = rs_NUTS3_df.select('IDREGION', 'DATE', 'FAPAR')\n",
        "\n",
        "    return rs_NUTS3_df\n",
        "\n",
        "  def preprocessRemoteSensing(self, rs_df, crop_season, season_crosses_calyear):\n",
        "    \"\"\"\n",
        "    Extract year and dekad from date.\n",
        "    Use crop_season to compute CAMPAIGN_YEAR and CAMPAIGN_DEKAD.\n",
        "    NOTE crop_season and rs_df must be at the same NUTS level.\n",
        "    \"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    drop_cols = crop_season.columns[2:]\n",
        "    if (('DATE' in rs_df.columns) and ('FYEAR' not in rs_df.columns)):\n",
        "      rs_df = self.extractYearDekad(rs_df)\n",
        "\n",
        "    rs_df = self.alignDataToCropSeason(rs_df, crop_season, season_crosses_calyear)\n",
        "    rs_df = rs_df.drop(*drop_cols)\n",
        "    return rs_df\n",
        "\n",
        "  def preprocessCentroids(self, centroids_df):\n",
        "    df_cols = centroids_df.columns\n",
        "    centroids_df = centroids_df.withColumn('CENTROID_X', SparkF.bround('CENTROID_X', 2))\n",
        "    centroids_df = centroids_df.withColumn('CENTROID_Y', SparkF.bround('CENTROID_Y', 2))\n",
        "\n",
        "    return centroids_df\n",
        "\n",
        "  def preprocessSoil(self, soil_df):\n",
        "    # SM_WC = water holding capacity\n",
        "    soil_df = soil_df.withColumn('SM_WHC', SparkF.bround(soil_df.SM_FC - soil_df.SM_WP, 2))\n",
        "    soil_df = soil_df.select(['IDREGION', 'SM_WHC'])\n",
        "\n",
        "    return soil_df\n",
        "\n",
        "  def preprocessAreaFractions(self, af_df, crop_id):\n",
        "    \"\"\"Filter area fractions data by crop id\"\"\"\n",
        "    af_df = af_df.withColumn(\"FYEAR\", af_df[\"FYEAR\"].cast(SparkT.IntegerType()))\n",
        "    af_df = af_df.filter(af_df.CROP_ID == crop_id).drop('CROP_ID')\n",
        "\n",
        "    return af_df\n",
        "\n",
        "  def preprocessYield(self, yield_df, crop_id):\n",
        "    \"\"\"\n",
        "    Yield preprocessing depends on the data format.\n",
        "    Here we cover preprocessing for France (NUTS3), Germany (NUTS3) and the Netherlands (NUTS2).\n",
        "    \"\"\"\n",
        "    # Delete trailing empty columns\n",
        "    empty_cols = [ c for c in yield_df.columns if c.startswith('_c') ]\n",
        "    for c in empty_cols:\n",
        "      yield_df = yield_df.drop(c)\n",
        "\n",
        "    # Special case for Netherlands and Germany: convert yield columns into rows\n",
        "    years = [int(c) for c in yield_df.columns if c[0].isdigit()]\n",
        "    if (len(years) > 0):\n",
        "      yield_by_year = yield_df.rdd.map(lambda x: (x[0], cropNameToID(crop_id_dict, x[0]), x[1],\n",
        "                                                  [(years[i], x[i+2]) for i in range(len(years))]))\n",
        "\n",
        "      yield_df = yield_by_year.toDF(['CROP', 'CROP_ID', 'IDREGION', 'YIELD'])\n",
        "      yield_df = yield_df.withColumn('YR_YIELD', SparkF.explode('YIELD')).drop('YIELD')\n",
        "      yield_by_year = yield_df.rdd.map(lambda x: (x[0], x[1], x[2], x[3][0], x[3][1]))\n",
        "      yield_df = yield_by_year.toDF(['CROP', 'CROP_ID', 'IDREGION', 'FYEAR', 'YIELD'])\n",
        "    else:\n",
        "      yield_by_year = yield_df.rdd.map(lambda x: (x[0], cropNameToID(crop_id_dict, x[0]), x[1], x[2], x[3]))\n",
        "      yield_df = yield_by_year.toDF(['CROP', 'CROP_ID', 'IDREGION', 'FYEAR', 'YIELD'])\n",
        "\n",
        "    yield_df = yield_df.filter(yield_df.CROP_ID == crop_id).drop('CROP', 'CROP_ID')\n",
        "    if (yield_df.count() == 0):\n",
        "      return None\n",
        "\n",
        "    yield_df = yield_df.filter(yield_df.YIELD.isNotNull())\n",
        "    yield_df = yield_df.withColumn(\"YIELD\", yield_df[\"YIELD\"].cast(SparkT.FloatType()))\n",
        "    yield_df = yield_df.filter(yield_df['YIELD'] > 0.0)\n",
        "\n",
        "    return yield_df\n",
        "\n",
        "  def preprocessYieldMCYFS(self, mcyfs_df, crop_id):\n",
        "    \"\"\"Preprocess MCYFS NUTS0 level yield predictions\"\"\"\n",
        "    # the input columns are IDREGION, CROP, PREDICTION_DATE, FYEAR, YIELD_PRED\n",
        "    mcyfs_df = mcyfs_df.withColumn('PRED_DEKAD', getDekad('PREDICTION_DATE'))\n",
        "    # the columns should now be IDREGION, CROP, PREDICTION_DATE, FYEAR, YIELD_PRED, PRED_DEKAD\n",
        "    yield_by_year = mcyfs_df.rdd.map(lambda x: (x[1], cropNameToID(crop_id_dict, x[1]),\n",
        "                                                x[0], x[3], x[2], x[4], x[5]))\n",
        "    mcyfs_df = yield_by_year.toDF(['CROP', 'CROP_ID', 'IDREGION', 'FYEAR',\n",
        "                                         'PRED_DATE', 'YIELD_PRED', 'PRED_DEKAD'])\n",
        "    mcyfs_df = mcyfs_df.filter(mcyfs_df.CROP_ID == crop_id).drop('CROP', 'CROP_ID')\n",
        "    if (mcyfs_df.count() == 0):\n",
        "      return None\n",
        "\n",
        "    mcyfs_df = mcyfs_df.withColumn(\"YIELD_PRED\", mcyfs_df[\"YIELD_PRED\"].cast(SparkT.FloatType()))\n",
        "\n",
        "    return mcyfs_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A2QR2OmysFd1"
      },
      "source": [
        "#### Run Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5ajXCUkjsIfK"
      },
      "outputs": [],
      "source": [
        "#%%writefile run_data_preprocessing.py\n",
        "def printPreprocessingInformation(df, data_source, order_cols, crop_season=None):\n",
        "  \"\"\"Print preprocessed data and additional debug information\"\"\"\n",
        "  df_regions = [reg[0] for reg in df.select('IDREGION').distinct().collect()]\n",
        "  print(data_source , 'data available for', len(df_regions), 'region(s)')\n",
        "  if (crop_season is not None):\n",
        "    print('Season end information')\n",
        "    crop_season.orderBy(['IDREGION', 'FYEAR']).show(10)\n",
        "\n",
        "  print(data_source, 'data')\n",
        "  df.orderBy(order_cols).show(10)\n",
        "\n",
        "def preprocessData(cyp_config, cyp_preprocessor, data_dfs):\n",
        "  crop_id = cyp_config.getCropID()\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  season_crosses_calyear = cyp_config.seasonCrossesCalendarYear()\n",
        "  use_centroids = cyp_config.useCentroids()\n",
        "  use_remote_sensing = cyp_config.useRemoteSensing()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  order_cols = ['IDREGION', 'CAMPAIGN_YEAR', 'CAMPAIGN_DEKAD']\n",
        "  # wofost data\n",
        "  wofost_df = data_dfs['WOFOST']\n",
        "  wofost_df = wofost_df.filter(wofost_df['CROP_ID'] == crop_id).drop('CROP_ID')\n",
        "  crop_season = cyp_preprocessor.getCropSeasonInformation(wofost_df, season_crosses_calyear)\n",
        "  wofost_df = cyp_preprocessor.preprocessWofost(wofost_df, crop_season, season_crosses_calyear)\n",
        "  wofost_regions = [reg[0] for reg in wofost_df.select('IDREGION').distinct().collect()]\n",
        "  data_dfs['WOFOST'] = wofost_df\n",
        "  if (debug_level > 1):\n",
        "    printPreprocessingInformation(wofost_df, 'WOFOST', order_cols, crop_season)\n",
        "\n",
        "  # meteo data\n",
        "  meteo_df = data_dfs['METEO']\n",
        "  meteo_df = cyp_preprocessor.preprocessMeteo(meteo_df, crop_season, season_crosses_calyear)\n",
        "  assert (meteo_df is not None)\n",
        "  data_dfs['METEO'] = meteo_df\n",
        "  if (debug_level > 1):\n",
        "    printPreprocessingInformation(meteo_df, 'METEO', order_cols)\n",
        "\n",
        "  # remote sensing data\n",
        "  rs_df = None\n",
        "  if (use_remote_sensing):\n",
        "    rs_df = data_dfs['REMOTE_SENSING']\n",
        "    rs_df = rs_df.drop('IDCOVER')\n",
        "\n",
        "    # if other data is at NUTS3, convert rs_df to NUTS3 using parent region data\n",
        "    # if (nuts_level == 'NUTS3'):\n",
        "    #   rs_df = cyp_preprocessor.remoteSensingNUTS2ToNUTS3(rs_df, wofost_regions)\n",
        "\n",
        "    rs_df = cyp_preprocessor.preprocessRemoteSensing(rs_df, crop_season, season_crosses_calyear)\n",
        "    assert (rs_df is not None)\n",
        "    data_dfs['REMOTE_SENSING'] = rs_df\n",
        "    if (debug_level > 1):\n",
        "      printPreprocessingInformation(rs_df, 'REMOTE_SENSING', order_cols)\n",
        "\n",
        "  order_cols = ['IDREGION']\n",
        "  # centroids and distance to coast\n",
        "  centroids_df = None\n",
        "  if (use_centroids):\n",
        "    centroids_df = data_dfs['CENTROIDS']\n",
        "    centroids_df = cyp_preprocessor.preprocessCentroids(centroids_df)\n",
        "    data_dfs['CENTROIDS'] = centroids_df\n",
        "    if (debug_level > 1):\n",
        "      printPreprocessingInformation(centroids_df, 'CENTROIDS', order_cols)\n",
        "\n",
        "  # soil data\n",
        "  soil_df = data_dfs['SOIL']\n",
        "  soil_df = cyp_preprocessor.preprocessSoil(soil_df)\n",
        "  data_dfs['SOIL'] = soil_df\n",
        "  if (debug_level > 1):\n",
        "    printPreprocessingInformation(soil_df, 'SOIL', order_cols)\n",
        "\n",
        "  order_cols = ['IDREGION', 'FYEAR']\n",
        "  # yield_data\n",
        "  yield_df = data_dfs['YIELD']\n",
        "  if (debug_level > 1):\n",
        "    print('Yield before preprocessing')\n",
        "    yield_df.show(10)\n",
        "\n",
        "  yield_df = cyp_preprocessor.preprocessYield(yield_df, crop_id)\n",
        "  assert (yield_df is not None)\n",
        "  data_dfs['YIELD'] = yield_df\n",
        "  if (debug_level > 1):\n",
        "    print('Yield after preprocessing')\n",
        "    yield_df.show(10)\n",
        "\n",
        "  return data_dfs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "krHPVXls8MRd"
      },
      "source": [
        "### Training and Test Split\n",
        "\n",
        "**Custom validation splits**\n",
        "\n",
        "When yield trend is used, custom sliding validation split is used."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5YPH051S8MRf"
      },
      "source": [
        "#### Training and Test Splitter Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2zBSEGkO8MRf"
      },
      "outputs": [],
      "source": [
        "#%%writefile train_test_sets.py\n",
        "import numpy as np\n",
        "\n",
        "class CYPTrainTestSplitter:\n",
        "  def __init__(self, cyp_config):\n",
        "    self.use_yield_trend = cyp_config.useYieldTrend()\n",
        "    self.test_fraction = cyp_config.getTestFraction()\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "\n",
        "  def getTestYears(self, all_years, test_fraction=None, use_yield_trend=None):\n",
        "    num_years = len(all_years)\n",
        "    test_years = []\n",
        "    if (test_fraction is None):\n",
        "      test_fraction = self.test_fraction\n",
        "\n",
        "    if (use_yield_trend is None):\n",
        "      use_yield_trend = self.use_yield_trend\n",
        "\n",
        "    if (use_yield_trend):\n",
        "      # If test_year_start 15, years with index >= 15 are added to the test set\n",
        "      test_year_start = num_years - np.floor(num_years * test_fraction).astype('int')\n",
        "      test_years = all_years[test_year_start:]\n",
        "    else:\n",
        "      # If test_year_pos = 5, every 5th year is added to test set.\n",
        "      # indices start with 0, so test_year_pos'th year has index (test_year_pos - 1)\n",
        "      test_year_pos = np.floor(1/test_fraction).astype('int')\n",
        "      test_years = all_years[test_year_pos - 1::test_year_pos]\n",
        "\n",
        "    return test_years\n",
        "\n",
        "  def trainTestSplit(self, yield_df, test_fraction=None, use_yield_trend=None):\n",
        "    all_years = sorted([yr[0] for yr in yield_df.select('FYEAR').distinct().collect()])\n",
        "    test_years = self.getTestYears(all_years, test_fraction, use_yield_trend)\n",
        "\n",
        "    return test_years\n",
        "\n",
        "  # Returns an array containings tuples (train_idxs, test_idxs) for each fold\n",
        "  # NOTE Y_train should include IDREGION, FYEAR as first two columns.\n",
        "  def customKFoldValidationSplit(self, Y_train_full, num_folds, log_fh=None):\n",
        "    \"\"\"\n",
        "    Custom K-fold Validation Splits:\n",
        "    When using yield trend, we cannot do k-fold cross-validation. The custom\n",
        "    K-Fold validation splits data in time-ordered fashion. The test data\n",
        "    always comes after the training data.\n",
        "    \"\"\"\n",
        "    all_years = sorted(np.unique(Y_train_full[:, 1]))\n",
        "    num_years = len(all_years)\n",
        "    num_test_years = 1\n",
        "    num_train_years = num_years - num_test_years * num_folds\n",
        "\n",
        "    custom_cv = []\n",
        "    custom_split_info = '\\nCustom sliding validation train, test splits'\n",
        "    custom_split_info += '\\n----------------------------------------------'\n",
        "\n",
        "    for k in range(num_folds):\n",
        "      train_years_start = k * num_test_years\n",
        "      test_years_start = train_years_start + num_train_years\n",
        "      train_years = all_years[train_years_start:test_years_start]\n",
        "      test_years = all_years[test_years_start:test_years_start + num_test_years]\n",
        "      test_indexes = np.ravel(np.nonzero(np.isin(Y_train_full[:, 1], test_years)))\n",
        "      train_indexes = np.ravel(np.nonzero(np.isin(Y_train_full[:, 1], train_years)))\n",
        "      custom_cv.append(tuple((train_indexes, test_indexes)))\n",
        "\n",
        "      train_years = [str(y) for y in train_years]\n",
        "      test_years = [str(y) for y in test_years]\n",
        "      custom_split_info += '\\nValidation set ' + str(k + 1) + ' training years: ' + ', '.join(train_years)\n",
        "      custom_split_info += '\\nValidation set ' + str(k + 1) + ' test years: ' + ', '.join(test_years)\n",
        "\n",
        "    custom_split_info += '\\n'\n",
        "    if (log_fh is not None):\n",
        "      log_fh.write(custom_split_info)\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      print(custom_split_info)\n",
        "\n",
        "    return custom_cv"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "URFR5pOxO3XZ"
      },
      "source": [
        "#### Run Training and Test Split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-TO9qzRVO3Xb"
      },
      "outputs": [],
      "source": [
        "#%%writefile run_train_test_split.py\n",
        "def printTrainTestSplits(train_df, test_df, src, order_cols):\n",
        "  \"\"\"Print Training and Test Splits\"\"\"\n",
        "  print('\\n', src, 'training data')\n",
        "  train_df.orderBy(order_cols).show(5)\n",
        "  print('\\n', src, 'test data')\n",
        "  test_df.orderBy(order_cols).show(5)\n",
        "\n",
        "# Training, Test Split\n",
        "# --------------------\n",
        "def splitTrainingTest(df, year_col, test_years):\n",
        "  \"\"\"Splitting given df into training and test dataframes.\"\"\"\n",
        "  train_df = df.filter(~df[year_col].isin(test_years))\n",
        "  test_df = df.filter(df[year_col].isin(test_years))\n",
        "\n",
        "  return [train_df, test_df]\n",
        "\n",
        "def splitDataIntoTrainingTestSets(cyp_config, preprocessed_dfs, log_fh):\n",
        "  \"\"\"\n",
        "  Split preprocessed data into training and test sets based on\n",
        "  availability of yield data.\n",
        "  \"\"\"\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  use_centroids = cyp_config.useCentroids()\n",
        "  use_remote_sensing = cyp_config.useRemoteSensing()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  yield_df = preprocessed_dfs['YIELD']\n",
        "  train_test_splitter = CYPTrainTestSplitter(cyp_config)\n",
        "  test_years = train_test_splitter.trainTestSplit(yield_df)\n",
        "  test_years_info = '\\nTest years: ' + ', '.join([str(y) for y in sorted(test_years)]) + '\\n'\n",
        "  log_fh.write(test_years_info + '\\n')\n",
        "  print(test_years_info)\n",
        "\n",
        "  # Times series data used for feature design.\n",
        "  ts_data_sources = {\n",
        "      'WOFOST' : preprocessed_dfs['WOFOST'],\n",
        "      'METEO' : preprocessed_dfs['METEO'],\n",
        "  }\n",
        "\n",
        "  if (use_remote_sensing):\n",
        "    ts_data_sources['REMOTE_SENSING'] = preprocessed_dfs['REMOTE_SENSING']\n",
        "\n",
        "  train_test_dfs = {}\n",
        "  for ts_src in ts_data_sources:\n",
        "    train_test_dfs[ts_src] = splitTrainingTest(ts_data_sources[ts_src], 'CAMPAIGN_YEAR', test_years)\n",
        "\n",
        "  # SOIL, CENTROIDS data are static.\n",
        "  train_test_dfs['SOIL'] = [preprocessed_dfs['SOIL'], preprocessed_dfs['SOIL']]\n",
        "  if (use_centroids):\n",
        "    train_test_dfs['CENTROIDS'] = [preprocessed_dfs['CENTROIDS'],\n",
        "                                   preprocessed_dfs['CENTROIDS']]\n",
        "\n",
        "  # yield data\n",
        "  train_test_dfs['YIELD'] = splitTrainingTest(yield_df, 'FYEAR', test_years)\n",
        "\n",
        "  if (debug_level > 2):\n",
        "    for src in train_test_dfs:\n",
        "      if (src in ts_data_sources):\n",
        "        order_cols = ['IDREGION', 'CAMPAIGN_YEAR', 'CAMPAIGN_DEKAD']\n",
        "      elif ((src == 'YIELD') or (src == 'AREA_FRACTIONS')):\n",
        "        order_cols = ['IDREGION', 'FYEAR']\n",
        "      else:\n",
        "        order_cols = ['IDREGION']\n",
        "\n",
        "      train_df = train_test_dfs[src][0]\n",
        "      test_df = train_test_dfs[src][1]\n",
        "      printTrainTestSplits(train_df, test_df, src, order_cols)\n",
        "\n",
        "  return train_test_dfs, test_years"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sCe_4_v--Ukb"
      },
      "source": [
        "### Data Summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gaSHuv0du9qf"
      },
      "source": [
        "#### Data Summarizer Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TD_QGQJs-Ukl"
      },
      "outputs": [],
      "source": [
        "#%%writefile data_summary.py\n",
        "from pyspark.sql import Window\n",
        "import functools\n",
        "\n",
        "class CYPDataSummarizer:\n",
        "  def __init__(self, cyp_config):\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "\n",
        "  def wofostDVSSummary(self, wofost_df, early_season_end=None):\n",
        "    \"\"\"\n",
        "    Summary of crop calendar based on DVS.\n",
        "    Early season end is relative to end of the season, hence a negative number.\n",
        "    \"\"\"\n",
        "    join_cols = ['IDREGION', 'CAMPAIGN_YEAR']\n",
        "    dvs_summary = wofost_df.select(join_cols).distinct()\n",
        "\n",
        "    # We find the start and end dekads for DVS ranges\n",
        "    my_window = Window.partitionBy(join_cols).orderBy('CAMPAIGN_DEKAD')\n",
        "\n",
        "    wofost_df = wofost_df.withColumn('VALUE', wofost_df['DVS'])\n",
        "    wofost_df = wofost_df.withColumn('PREV', SparkF.lag(wofost_df['VALUE']).over(my_window))\n",
        "    wofost_df = wofost_df.withColumn('DIFF', SparkF.when(SparkF.isnull(wofost_df['PREV']), 0)\\\n",
        "                                     .otherwise(wofost_df['VALUE'] - wofost_df['PREV']))\n",
        "    wofost_df = wofost_df.withColumn('SEASON_ALIGN', wofost_df['CAMPAIGN_DEKAD'] - wofost_df['DEKAD'])\n",
        "\n",
        "    del_cols = ['VALUE', 'PREV', 'DIFF']\n",
        "    dvs_summary = dvs_summary.join(wofost_df.filter(wofost_df['VALUE'] > 0.0).groupBy(join_cols)\\\n",
        "                                   .agg(SparkF.min('CAMPAIGN_DEKAD').alias('START_DVS')), join_cols)\n",
        "    dvs_summary = dvs_summary.join(wofost_df.filter(wofost_df['DVS'] >= 100).groupBy(join_cols)\\\n",
        "                                   .agg(SparkF.min('CAMPAIGN_DEKAD').alias('START_DVS1')), join_cols)\n",
        "    dvs_summary = dvs_summary.join(wofost_df.filter(wofost_df['DVS'] >= 200).groupBy(join_cols)\\\n",
        "                                   .agg(SparkF.min('CAMPAIGN_DEKAD').alias('START_DVS2')), join_cols)\n",
        "    dvs_summary = dvs_summary.join(wofost_df.filter(wofost_df['DVS'] >= 200).groupBy(join_cols)\\\n",
        "                                   .agg(SparkF.min('DEKAD').alias('HARVEST')), join_cols)\n",
        "    dvs_summary = dvs_summary.join(wofost_df.filter(wofost_df['DVS'] >= 200).groupBy(join_cols)\\\n",
        "                                   .agg(SparkF.max('SEASON_ALIGN').alias('SEASON_ALIGN')), join_cols)\n",
        "\n",
        "    # Calendar year end season and early season dekads for comparing with MCYFS\n",
        "    # Campaign year early season dekad to filter data during feature design\n",
        "    dvs_summary = dvs_summary.withColumn('CALENDAR_END_SEASON', dvs_summary['HARVEST'] + 1)\n",
        "    dvs_summary = dvs_summary.withColumn('CAMPAIGN_EARLY_SEASON',\n",
        "                                         dvs_summary['CALENDAR_END_SEASON'] + dvs_summary['SEASON_ALIGN'])\n",
        "    if (early_season_end is not None):\n",
        "      dvs_summary = dvs_summary.withColumn('CALENDAR_EARLY_SEASON',\n",
        "                                         dvs_summary['CALENDAR_END_SEASON'] + early_season_end)\n",
        "      dvs_summary = dvs_summary.withColumn('CAMPAIGN_EARLY_SEASON',\n",
        "                                           dvs_summary['CAMPAIGN_EARLY_SEASON'] + early_season_end)\n",
        "\n",
        "    wofost_df = wofost_df.drop(*del_cols)\n",
        "    dvs_summary = dvs_summary.drop('HARVEST', 'SEASON_ALIGN')\n",
        "\n",
        "    return dvs_summary\n",
        "\n",
        "  def indicatorsSummary(self, df, min_cols, max_cols, avg_cols):\n",
        "    \"\"\"long term min, max and avg values of selected indicators by region\"\"\"\n",
        "    avgs = []\n",
        "    if (avg_cols[1:]):\n",
        "      avgs = [SparkF.bround(SparkF.avg(x), 2).alias('avg(' + x + ')') for x in avg_cols[1:]]\n",
        "    \n",
        "    if (min_cols[:1]):\n",
        "      summary = df.select(min_cols).groupBy('IDREGION').min()\n",
        "    else:\n",
        "      summary = df.select(min_cols).groupBy('IDREGION')\n",
        "\n",
        "    if (max_cols[1:]):\n",
        "      summary = summary.join(df.select(max_cols).groupBy('IDREGION').max(), 'IDREGION')\n",
        "\n",
        "    if (avgs):\n",
        "      summary = summary.join(df.select(avg_cols).groupBy('IDREGION').agg(*avgs), 'IDREGION')\n",
        "    return summary\n",
        "\n",
        "  def yieldSummary(self, yield_df):\n",
        "    \"\"\"long term min, max and avg values of yield by region\"\"\"\n",
        "    select_cols = ['IDREGION', 'YIELD']\n",
        "    yield_summary = yield_df.select(select_cols).groupBy('IDREGION').min('YIELD')\n",
        "    yield_summary = yield_summary.join(yield_df.select(select_cols).groupBy('IDREGION')\\\n",
        "                                       .agg(SparkF.max('YIELD')), 'IDREGION')\n",
        "    yield_summary = yield_summary.join(yield_df.select(select_cols).groupBy('IDREGION')\\\n",
        "                                       .agg(SparkF.bround(SparkF.avg('YIELD'), 2)\\\n",
        "                                            .alias('avg(YIELD)')), 'IDREGION')\n",
        "    return yield_summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zym9pU3XsPrt"
      },
      "source": [
        "#### Run Data Summary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5CNwR1WAsSLz"
      },
      "outputs": [],
      "source": [
        "#%%writefile run_data_summary.py\n",
        "def printDataSummary(df, data_source):\n",
        "  \"\"\"Print summary information\"\"\"\n",
        "  if (data_source == 'WOFOST_DVS'):\n",
        "    print('Crop calender information based on WOFOST data')\n",
        "    max_year = df.select('CAMPAIGN_YEAR').agg(SparkF.max('CAMPAIGN_YEAR')).collect()[0][0]\n",
        "    df.filter(df.CAMPAIGN_YEAR == max_year).orderBy('IDREGION').show(10)\n",
        "  else:\n",
        "    print(data_source, 'indicators summary')\n",
        "    df.orderBy('IDREGION').show()\n",
        "\n",
        "def getWOFOSTSummaryCols():\n",
        "  \"\"\"WOFOST columns used for data summary\"\"\"\n",
        "  # only RSM has non-zero min values\n",
        "  min_cols = ['IDREGION', 'RSM']\n",
        "  max_cols = ['IDREGION'] + ['WLIM_YB', 'WLIM_YS', 'DVS',\n",
        "                             'WLAI', 'RSM', 'TWC', 'TWR']\n",
        "  # biomass and DVS values grow over time\n",
        "  avg_cols = ['IDREGION', 'WLAI', 'RSM', 'TWC', 'TWR']\n",
        "\n",
        "  return [min_cols, max_cols, avg_cols]\n",
        "\n",
        "def getMeteoSummaryCols():\n",
        "  \"\"\"Meteo columns used for data summary\"\"\"\n",
        "  col_names = ['TMAX', 'TMIN', 'TAVG', 'PREC', 'ET0', 'CWB', 'RAD']\n",
        "  min_cols = ['IDREGION'] + col_names\n",
        "  max_cols = ['IDREGION'] + col_names\n",
        "  avg_cols = ['IDREGION'] + col_names\n",
        "\n",
        "  return [min_cols, max_cols, avg_cols]\n",
        "\n",
        "def getRemoteSensingSummaryCols():\n",
        "  \"\"\"Remote Sensing columns used for data summary\"\"\"\n",
        "  col_names = ['FAPAR']\n",
        "  min_cols = ['IDREGION'] + col_names\n",
        "  max_cols = ['IDREGION'] + col_names\n",
        "  avg_cols = ['IDREGION'] + col_names\n",
        "\n",
        "  return [min_cols, max_cols, avg_cols]\n",
        "\n",
        "def summarizeData(cyp_config, cyp_summarizer, train_test_dfs):\n",
        "  \"\"\"\n",
        "  Summarize data. Create DVS summary to infer crop calendar.\n",
        "  Summarize selected indicators for each data source.\n",
        "  \"\"\"\n",
        "  wofost_train_df = train_test_dfs['WOFOST'][0]\n",
        "  meteo_train_df = train_test_dfs['METEO'][0]\n",
        "  yield_train_df = train_test_dfs['YIELD'][0]\n",
        "\n",
        "  use_remote_sensing = cyp_config.useRemoteSensing()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  early_season = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = None\n",
        "  if (early_season):\n",
        "    early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "\n",
        "  # DVS summary (crop calendar)\n",
        "  # NOTE this summary of crops based on wofost data should be used with caution\n",
        "  # 1. The summary is per region per year.\n",
        "  # 2. The summary is based on wofost simulations not real sowing and harvest dates\n",
        "  dvs_summary = cyp_summarizer.wofostDVSSummary(wofost_train_df, early_season_end)\n",
        "  dvs_summary = dvs_summary.drop('CALENDAR_END_SEASON', 'CALENDAR_EARLY_SEASON')\n",
        "  if (debug_level > 1):\n",
        "    printDataSummary(dvs_summary, 'WOFOST_DVS')\n",
        "\n",
        "  summary_cols = {\n",
        "      'WOFOST' : getWOFOSTSummaryCols(),\n",
        "      'METEO' : getMeteoSummaryCols(),\n",
        "  }\n",
        "\n",
        "  summary_sources_dfs = {\n",
        "      'WOFOST' : wofost_train_df,\n",
        "      'METEO' : meteo_train_df,\n",
        "  }\n",
        "\n",
        "  if (use_remote_sensing):\n",
        "    rs_train_df = train_test_dfs['REMOTE_SENSING'][0]\n",
        "    summary_cols['REMOTE_SENSING'] = getRemoteSensingSummaryCols()\n",
        "    summary_sources_dfs['REMOTE_SENSING'] = rs_train_df\n",
        "\n",
        "  summary_dfs = {}\n",
        "  for sum_src in summary_sources_dfs:\n",
        "    summary_dfs[sum_src] = cyp_summarizer.indicatorsSummary(summary_sources_dfs[sum_src],\n",
        "                                                            summary_cols[sum_src][0],\n",
        "                                                            summary_cols[sum_src][1],\n",
        "                                                            summary_cols[sum_src][2])\n",
        "\n",
        "  for src in summary_dfs:\n",
        "    if (debug_level > 2):\n",
        "      printDataSummary(summary_dfs[src], src)\n",
        "\n",
        "  yield_summary = cyp_summarizer.yieldSummary(yield_train_df)\n",
        "  if (debug_level > 2):\n",
        "    printDataSummary(yield_summary, 'YIELD')\n",
        "\n",
        "  summary_dfs['WOFOST_DVS'] = dvs_summary\n",
        "  summary_dfs['YIELD'] = yield_summary\n",
        "\n",
        "  return summary_dfs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vWwazE4MJI4v"
      },
      "source": [
        "### Crop Calendar\n",
        "\n",
        "We infer crop calendar using WOFOST DVS."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nBVBgwxSJEIn"
      },
      "outputs": [],
      "source": [
        "#%%writefile crop_calendar.py\n",
        "import numpy as np\n",
        "\n",
        "def getCropCalendarPeriods(df):\n",
        "  \"\"\"Periods for per year crop calendar\"\"\"\n",
        "  # (maximum of 4 months = 12 dekads).\n",
        "  # Subtracting 11 because both ends of the period are included.\n",
        "  # p0 : if CAMPAIGN_EARLY_SEASON > df.START_DVS\n",
        "  #        START_DVS - 11 to START_DVS\n",
        "  #      else\n",
        "  #        START_DVS - 11 to CAMPAIGN_EARLY_SEASON\n",
        "  p0_filter = SparkF.when(df.CAMPAIGN_EARLY_SEASON > df.START_DVS,\n",
        "                          (df.CAMPAIGN_DEKAD >= (df.START_DVS - 11)) &\n",
        "                          (df.CAMPAIGN_DEKAD <= df.START_DVS))\\\n",
        "                          .otherwise((df.CAMPAIGN_DEKAD >= (df.START_DVS - 11)) &\n",
        "                                     (df.CAMPAIGN_DEKAD <= df.CAMPAIGN_EARLY_SEASON))\n",
        "  # p1 : if CAMPAIGN_EARLY_SEASON > (df.START_DVS + 1)\n",
        "  #        (START_DVS - 1) to (START_DVS + 1)\n",
        "  #      else\n",
        "  #        (START_DVS - 1) to CAMPAIGN_EARLY_SEASON\n",
        "  p1_filter = SparkF.when(df.CAMPAIGN_EARLY_SEASON > (df.START_DVS + 1),\n",
        "                          (df.CAMPAIGN_DEKAD >= (df.START_DVS - 1)) &\n",
        "                          (df.CAMPAIGN_DEKAD <= (df.START_DVS + 1)))\\\n",
        "                          .otherwise((df.CAMPAIGN_DEKAD >= (df.START_DVS - 1)) &\n",
        "                                     (df.CAMPAIGN_DEKAD <= df.CAMPAIGN_EARLY_SEASON))\n",
        "  # p2 : if CAMPAIGN_EARLY_SEASON > df.START_DVS1\n",
        "  #        START_DVS to START_DVS1\n",
        "  #      else\n",
        "  #        START_DVS to CAMPAIGN_EARLY_SEASON\n",
        "  p2_filter = SparkF.when(df.CAMPAIGN_EARLY_SEASON > df.START_DVS1,\n",
        "                          (df.CAMPAIGN_DEKAD >= df.START_DVS) &\n",
        "                          (df.CAMPAIGN_DEKAD <= df.START_DVS1))\\\n",
        "                          .otherwise((df.CAMPAIGN_DEKAD >= df.START_DVS) &\n",
        "                                     (df.CAMPAIGN_DEKAD <= df.CAMPAIGN_EARLY_SEASON))\n",
        "  # p3 : if CAMPAIGN_EARLY_SEASON > (df.START_DVS1 + 1)\n",
        "  #        (START_DVS1 - 1) to (START_DVS1 + 1)\n",
        "  #      else\n",
        "  #        (START_DVS1 - 1) to CAMPAIGN_EARLY_SEASON\n",
        "  p3_filter = SparkF.when(df.CAMPAIGN_EARLY_SEASON > (df.START_DVS1 + 1),\n",
        "                          (df.CAMPAIGN_DEKAD >= (df.START_DVS1 - 1)) &\n",
        "                          (df.CAMPAIGN_DEKAD <= (df.START_DVS1 + 1)))\\\n",
        "                          .otherwise((df.CAMPAIGN_DEKAD >= (df.START_DVS1 - 1)) &\n",
        "                                     (df.CAMPAIGN_DEKAD <= df.CAMPAIGN_EARLY_SEASON))\n",
        "  # p4 : if CAMPAIGN_EARLY_SEASON > df.START_DVS2\n",
        "  #        START_DVS1 to START_DVS2\n",
        "  #      else\n",
        "  #        START_DVS1 to CAMPAIGN_EARLY_SEASON\n",
        "  p4_filter = SparkF.when(df.CAMPAIGN_EARLY_SEASON > df.START_DVS2,\n",
        "                          (df.CAMPAIGN_DEKAD >= df.START_DVS1) &\n",
        "                          (df.CAMPAIGN_DEKAD <= df.START_DVS2))\\\n",
        "                          .otherwise((df.CAMPAIGN_DEKAD >= df.START_DVS1) &\n",
        "                                     (df.CAMPAIGN_DEKAD <= df.CAMPAIGN_EARLY_SEASON))\n",
        "  # p5 : if CAMPAIGN_EARLY_SEASON > (df.START_DVS2 + 1)\n",
        "  #        (START_DVS2 - 1) to (START_DVS2 + 1)\n",
        "  #      else\n",
        "  #        (START_DVS2 - 1) to CAMPAIGN_EARLY_SEASON\n",
        "  p5_filter = SparkF.when(df.CAMPAIGN_EARLY_SEASON > (df.START_DVS2 + 1),\n",
        "                          (df.CAMPAIGN_DEKAD >= (df.START_DVS2 - 1)) &\n",
        "                          (df.CAMPAIGN_DEKAD <= (df.START_DVS2 + 1)))\\\n",
        "                          .otherwise((df.CAMPAIGN_DEKAD >= (df.START_DVS2 - 1)) &\n",
        "                                     (df.CAMPAIGN_DEKAD <= df.CAMPAIGN_EARLY_SEASON))\n",
        "\n",
        "  cc_periods = {\n",
        "      'p0' : p0_filter,\n",
        "      'p1' : p1_filter,\n",
        "      'p2' : p2_filter,\n",
        "      'p3' : p3_filter,\n",
        "      'p4' : p4_filter,\n",
        "      'p5' : p5_filter,\n",
        "  }\n",
        "\n",
        "  return cc_periods\n",
        "\n",
        "def getCountryCropCalendar(crop_cal):\n",
        "  \"\"\"Take averages to make the crop calendar per country\"\"\"\n",
        "  crop_cal = crop_cal.withColumn('COUNTRY', SparkF.substring('IDREGION', 1, 2))\n",
        "  aggrs = [ SparkF.bround(SparkF.avg(crop_cal['START_DVS'])).alias('START_DVS'),\n",
        "            SparkF.bround(SparkF.avg(crop_cal['START_DVS1'])).alias('START_DVS1'),\n",
        "            SparkF.bround(SparkF.avg(crop_cal['START_DVS2'])).alias('START_DVS2'),\n",
        "            SparkF.bround(SparkF.avg(crop_cal['CAMPAIGN_EARLY_SEASON'])).alias('CAMPAIGN_EARLY_SEASON') ]\n",
        "\n",
        "  crop_cal = crop_cal.groupBy('COUNTRY').agg(*aggrs)\n",
        "  return crop_cal\n",
        "\n",
        "def getCropCalendar(cyp_config, dvs_summary, log_fh):\n",
        "  \"\"\"Use DVS summary to infer the crop calendar\"\"\"\n",
        "  pd_dvs_summary = dvs_summary.toPandas()\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  avg_dvs_start = np.round(pd_dvs_summary['START_DVS'].mean(), 0)\n",
        "  avg_dvs1_start = np.round(pd_dvs_summary['START_DVS1'].mean(), 0)\n",
        "  avg_dvs2_start = np.round(pd_dvs_summary['START_DVS2'].mean(), 0)\n",
        "\n",
        "  # We look at 6 windows\n",
        "  # 0. Preplanting window (maximum of 4 months = 12 dekads).\n",
        "  # Subtracting 11 because both ends of the period are included.\n",
        "  p0_start = 1 if (avg_dvs_start - 11) < 1 else (avg_dvs_start - 11)\n",
        "  p0_end = avg_dvs_start\n",
        "\n",
        "  # 1. Planting window\n",
        "  p1_start = avg_dvs_start - 1\n",
        "  p1_end = avg_dvs_start + 1\n",
        "\n",
        "  # 2. Vegetative phase\n",
        "  p2_start = avg_dvs_start\n",
        "  p2_end = avg_dvs1_start\n",
        "\n",
        "  # 3. Flowering phase\n",
        "  p3_start = avg_dvs1_start - 1\n",
        "  p3_end = avg_dvs1_start + 1\n",
        "\n",
        "  # 4. Yield formation phase\n",
        "  p4_start = avg_dvs1_start\n",
        "  p4_end = avg_dvs2_start\n",
        "\n",
        "  # 5. Harvest window\n",
        "  p5_start = avg_dvs2_start - 1\n",
        "  p5_end = avg_dvs2_start + 1\n",
        "\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = 36\n",
        "  if (early_season_prediction):\n",
        "    early_season_end = np.round(pd_dvs_summary['CAMPAIGN_EARLY_SEASON'].mean(), 0)\n",
        "    p0_end = early_season_end if (p0_end > early_season_end) else p0_end\n",
        "    p1_end = early_season_end if (p1_end > early_season_end) else p1_end\n",
        "    p2_end = early_season_end if (p2_end > early_season_end) else p2_end\n",
        "    p3_end = early_season_end if (p3_end > early_season_end) else p3_end\n",
        "    p4_end = early_season_end if (p4_end > early_season_end) else p4_end\n",
        "    p5_end = early_season_end if (p5_end > early_season_end) else p5_end\n",
        "\n",
        "  crop_cal = {}\n",
        "  if (p0_end > p0_start):\n",
        "    crop_cal['p0'] = { 'desc' : 'pre-planting window', 'start' : p0_start, 'end' : p0_end }\n",
        "  if (p1_end > p1_start):\n",
        "    crop_cal['p1'] = { 'desc' : 'planting window', 'start' : p1_start, 'end' : p1_end }\n",
        "  if (p2_end > p2_start):\n",
        "    crop_cal['p2'] = { 'desc' : 'vegetative phase', 'start' : p2_start, 'end' : p2_end }\n",
        "  if (p3_end > p3_start):\n",
        "    crop_cal['p3'] = { 'desc' : 'flowering phase', 'start' : p3_start, 'end' : p3_end }\n",
        "  if (p4_end > p4_start):\n",
        "    crop_cal['p4'] = { 'desc' : 'yield formation phase', 'start' : p4_start, 'end' : p4_end }\n",
        "  if (p5_end > p5_start):\n",
        "    crop_cal['p5'] = { 'desc' : 'harvest window', 'start' : p5_start, 'end' : p5_end }\n",
        "\n",
        "  if (early_season_prediction):\n",
        "    early_season_rel_harvest = cyp_config.getEarlySeasonEndDekad()\n",
        "    early_season_info = '\\nEarly Season Prediction Dekad: ' + str(early_season_rel_harvest)\n",
        "    early_season_info += ', Campaign Dekad: ' + str(early_season_end)\n",
        "    log_fh.write(early_season_info + '\\n')\n",
        "    if (debug_level > 1):\n",
        "      print(early_season_info)\n",
        "\n",
        "  crop_cal_info = '\\nCrop Calendar'\n",
        "  crop_cal_info += '\\n-------------'\n",
        "  for p in crop_cal:\n",
        "    crop_cal_info += '\\nPeriod ' + p + ' (' + crop_cal[p]['desc'] + '): '\n",
        "    crop_cal_info += 'Campaign Dekads ' + str(crop_cal[p]['start']) + '-' + str(crop_cal[p]['end'])\n",
        "\n",
        "  log_fh.write(crop_cal_info + '\\n')\n",
        "  if (debug_level > 1):\n",
        "    print(crop_cal_info)\n",
        "\n",
        "  return crop_cal"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QokglosfnQax"
      },
      "source": [
        "### Feature Design\n",
        "\n",
        "For WOFOST, Meteo and Remote Sensing features, we aggregate indicators or count days/dekads with indicators above or below some thresholds. We use 4 thresholds: +/- 1 STD and +/- 2STD above or below the average.\n",
        "\n",
        "We determine the start and end dekads using crop calendar inferred from WOFOST DVS summary. In the case of early season prediction, end dekad is set to the prediction dekad."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MHVsi9dzv4Pz"
      },
      "source": [
        "#### Featurizer Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e94zGLjVnQa0"
      },
      "outputs": [],
      "source": [
        "#%%writefile feature_design.py\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import functools\n",
        "\n",
        "class CYPFeaturizer:\n",
        "  def __init__(self, cyp_config):\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "    self.lt_stats = {}\n",
        "\n",
        "  def extractFeatures(self, df, data_source, crop_cal,\n",
        "                      max_cols, avg_cols, extreme_cols,\n",
        "                      join_cols, fit=False):\n",
        "    \"\"\"\n",
        "    Extract aggregate and extreme features.\n",
        "    If fit=True, compute and save long-term stats.\n",
        "    \"\"\"\n",
        "    df = df.withColumn('COUNTRY', SparkF.substring('IDREGION', 1, 2))\n",
        "    crop_cal = getCountryCropCalendar(crop_cal)\n",
        "    df = df.join(SparkF.broadcast(crop_cal), 'COUNTRY')\n",
        "\n",
        "    cc_periods = getCropCalendarPeriods(df)\n",
        "    aggrs = []\n",
        "    # max aggregation\n",
        "    for p in max_cols:\n",
        "      if (max_cols[p]):\n",
        "        aggrs += [SparkF.bround(SparkF.max(SparkF.when(cc_periods[p], df[x])), 2)\\\n",
        "                  .alias('max' + x + p) for x in max_cols[p] ]\n",
        "\n",
        "    # avg aggregation\n",
        "    for p in avg_cols:\n",
        "      if (avg_cols[p]):\n",
        "        aggrs += [SparkF.bround(SparkF.avg(SparkF.when(cc_periods[p], df[x])), 2)\\\n",
        "                  .alias('avg' + x + p) for x in avg_cols[p] ]\n",
        "\n",
        "    # if not computing extreme features, we can return\n",
        "    if (not extreme_cols):\n",
        "      ft_df = df.groupBy(join_cols).agg(*aggrs)\n",
        "      return ft_df\n",
        "\n",
        "    # compute long-term stats and save them\n",
        "    if (fit):\n",
        "      stat_aggrs = []\n",
        "      for p in extreme_cols:\n",
        "        if (extreme_cols[p]):\n",
        "          stat_aggrs += [ SparkF.bround(SparkF.avg(SparkF.when(cc_periods[p], df[x])), 2)\\\n",
        "                         .alias('avg' + x + p) for x in extreme_cols[p] ]\n",
        "          stat_aggrs += [ SparkF.bround(SparkF.stddev(SparkF.when(cc_periods[p], df[x])), 2)\\\n",
        "                         .alias('std' + x + p) for x in extreme_cols[p] ]\n",
        "\n",
        "      if (stat_aggrs):\n",
        "        lt_stats = df.groupBy('COUNTRY').agg(*stat_aggrs)\n",
        "        self.lt_stats[data_source] = lt_stats\n",
        "\n",
        "    df = df.join(SparkF.broadcast(self.lt_stats[data_source]), 'COUNTRY')\n",
        "\n",
        "    # features for extreme conditions\n",
        "    for p in extreme_cols:\n",
        "      if (extreme_cols[p]):\n",
        "        # Count of days or dekads with values crossing threshold\n",
        "        for i in range(1, 3):\n",
        "          aggrs += [ SparkF.sum(SparkF.when((df[x] > (df['avg' + x + p] + i * df['std' + x + p])) &\n",
        "                                            cc_periods[p], 1))\\\n",
        "                    .alias(x + p + 'gt' + str(i) + 'STD') for x in extreme_cols[p] ]\n",
        "          aggrs += [ SparkF.sum(SparkF.when((df[x] < (df['avg' + x + p] - i * df['std' + x + p])) &\n",
        "                                            cc_periods[p], 1))\\\n",
        "                    .alias(x + p + 'lt' + str(i) + 'STD') for x in extreme_cols[p] ]\n",
        "\n",
        "    ft_df = df.groupBy(join_cols).agg(*aggrs)\n",
        "    ft_df = ft_df.na.fill(0.0)\n",
        "    return ft_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0CZS6KDg8raw"
      },
      "source": [
        "#### Yield Trend Estimator Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_2O70-hQ8rax"
      },
      "outputs": [],
      "source": [
        "#%%writefile yield_trend.py\n",
        "import numpy as np\n",
        "import functools\n",
        "from pyspark.sql import Window\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "class CYPYieldTrendEstimator:\n",
        "  def __init__(self, cyp_config):\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "    self.trend_windows = cyp_config.getTrendWindows()\n",
        "\n",
        "  def setTrendWindows(self, trend_windows):\n",
        "    \"\"\"Set trend window lengths\"\"\"\n",
        "    assert isinstance(trend_windows, list)\n",
        "    assert len(trend_windows) > 0\n",
        "    assert isinstance(trend_windows[0], int)\n",
        "    self.trend_windows = trend_windows\n",
        "\n",
        "  def getTrendWindowYields(self, df, trend_window, reg_id=None):\n",
        "    \"\"\"Extract previous years' yield values to separate columns\"\"\"\n",
        "    sel_cols = ['IDREGION', 'FYEAR', 'YIELD']\n",
        "    my_window = Window.partitionBy('IDREGION').orderBy('FYEAR')\n",
        "\n",
        "    yield_fts = df.select(sel_cols)\n",
        "    if (reg_id is not None):\n",
        "      yield_fts = yield_fts.filter(yield_fts.IDREGION == reg_id)\n",
        "\n",
        "    for i in range(trend_window):\n",
        "      yield_fts = yield_fts.withColumn('YIELD-' + str(i+1),\n",
        "                                       SparkF.lag(yield_fts.YIELD, i+1).over(my_window))\n",
        "      yield_fts = yield_fts.withColumn('YEAR-' + str(i+1),\n",
        "                                       SparkF.lag(yield_fts.FYEAR, i+1).over(my_window))\n",
        "\n",
        "    # drop columns withs null values\n",
        "    for i in range(trend_window):\n",
        "      yield_fts = yield_fts.filter(SparkF.col('YIELD-' + str(i+1)).isNotNull())\n",
        "\n",
        "    prev_yields = [ 'YIELD-' + str(i) for i in range(trend_window, 0, -1)]\n",
        "    prev_years = [ 'YEAR-' + str(i) for i in range(trend_window, 0, -1)]\n",
        "    sel_cols = ['IDREGION', 'FYEAR'] + prev_years + prev_yields\n",
        "    yield_fts = yield_fts.select(sel_cols)\n",
        "\n",
        "    return yield_fts\n",
        "\n",
        "  # Christos, Ante's suggestion\n",
        "  # - To avoid overfitting, trend estimation could use a window which skips a year in between\n",
        "  # So a window of 3 will use 6 years of data\n",
        "  def printYieldTrendRounds(self, df, reg_id, trend_windows=None):\n",
        "    \"\"\"Print the sequence of years used for yield trend estimation\"\"\"\n",
        "    reg_years = sorted([yr[0] for yr in df.filter(df['IDREGION'] == reg_id).select('FYEAR').distinct().collect()])\n",
        "    num_years = len(reg_years)\n",
        "    if (trend_windows is None):\n",
        "      trend_windows = self.trend_windows\n",
        "\n",
        "    for trend_window in trend_windows:\n",
        "      rounds = (num_years - trend_window)\n",
        "      if ((self.verbose > 2) and (trend_window == trend_windows[0])):\n",
        "        print('Trend window', trend_window)\n",
        "    \n",
        "      for rd in range(rounds):\n",
        "        test_year = reg_years[-(rd + 1)]\n",
        "        start_year = reg_years[-(rd + trend_window + 1)]\n",
        "        end_year = reg_years[-(rd + 2)]\n",
        "\n",
        "        if ((self.verbose > 2) and (trend_window == trend_windows[0])):\n",
        "          print('Round:', rd, 'Test year:', test_year,\n",
        "                'Trend Window:', start_year, '-', end_year)\n",
        "\n",
        "  def getLinearYieldTrend(self, window_years, window_yields, pred_year):\n",
        "    \"\"\"Linear yield trend prediction\"\"\"\n",
        "    coefs = np.polyfit(window_years, window_yields, 1)\n",
        "    return float(np.round(coefs[0] * pred_year + coefs[1], 2))\n",
        "\n",
        "  def getFixedWindowTrendFeatures(self, df, trend_window=None, pred_year=None):\n",
        "    \"\"\"Predict the yield trend for each IDREGION and FYEAR using a fixed window\"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    if (trend_window is None):\n",
        "      trend_window = self.trend_windows[0]\n",
        "\n",
        "    yield_ft_df = self.getTrendWindowYields(df, trend_window)\n",
        "    if (pred_year is not None):\n",
        "      yield_ft_df = yield_ft_df.filter(yield_ft_df['FYEAR'] == pred_year)\n",
        "\n",
        "    pd_yield_ft_df = yield_ft_df.toPandas()\n",
        "    region_years = pd_yield_ft_df[join_cols].values\n",
        "    prev_year_cols = ['YEAR-' + str(i) for i in range(1, trend_window + 1)]\n",
        "    prev_yield_cols = ['YIELD-' + str(i) for i in range(1, trend_window + 1)]\n",
        "    window_years = pd_yield_ft_df[prev_year_cols].values\n",
        "    window_yields = pd_yield_ft_df[prev_yield_cols].values\n",
        "\n",
        "    yield_trend = []\n",
        "    for i in range(region_years.shape[0]):\n",
        "      yield_trend.append(self.getLinearYieldTrend(window_years[i, :],\n",
        "                                                  window_yields[i, :],\n",
        "                                                  region_years[i, 1]))\n",
        "\n",
        "    pd_yield_ft_df['YIELD_TREND'] = yield_trend\n",
        "    return pd_yield_ft_df\n",
        "\n",
        "  def getFixedWindowTrend(self, df, reg_id, pred_year, trend_window=None):\n",
        "    \"\"\"\n",
        "    Return linear trend prediction for given region and year\n",
        "    using fixed trend window.\n",
        "    \"\"\"\n",
        "    if (trend_window is None):\n",
        "      trend_window = self.trend_windows[0]\n",
        "\n",
        "    reg_df = df.filter((df['IDREGION'] == reg_id) & (df['FYEAR'] <= pred_year))\n",
        "    pd_yield_ft_df = self.getFixedWindowTrendFeatures(reg_df, trend_window, pred_year)\n",
        "    if (len(pd_yield_ft_df.index) == 0):\n",
        "      print('No data to estimate yield trend')\n",
        "      return None\n",
        "\n",
        "    if (len(pd_yield_ft_df.index) == 0):\n",
        "      return None\n",
        "\n",
        "    reg_year_filter = (df['IDREGION'] == reg_id) & (df['FYEAR'] == pred_year)\n",
        "    pd_yield_ft_df['ACTUAL'] = df.filter(reg_year_filter).select('YIELD').collect()[0][0]\n",
        "    pd_yield_ft_df = pd_yield_ft_df.rename(columns={'YIELD_TREND' : 'PREDICTED'})\n",
        "\n",
        "    return pd_yield_ft_df\n",
        "\n",
        "  def getL1OutCVPredictions(self, pd_yield_ft_df, trend_window, join_cols, iter):\n",
        "    \"\"\"1 iteration of leave-one-out cross-validation\"\"\"\n",
        "    iter_year_cols = ['YEAR-' + str(i) for i in range(1, trend_window + 1) if i != iter]\n",
        "    iter_yield_cols = ['YIELD-' + str(i) for i in range(1, trend_window + 1) if i != iter]\n",
        "    window_years = pd_yield_ft_df[iter_year_cols].values\n",
        "    window_yields = pd_yield_ft_df[iter_yield_cols].values\n",
        "\n",
        "    # We are going to predict yield value for YEAR-<iter>.\n",
        "    pred_years = pd_yield_ft_df['YEAR-' + str(iter)].values\n",
        "    predicted_trend = []\n",
        "    for i in range(pred_years.shape[0]):\n",
        "      predicted_trend.append(self.getLinearYieldTrend(window_years[i, :],\n",
        "                                                      window_yields[i, :],\n",
        "                                                      pred_years[i]))\n",
        "\n",
        "    pd_iter_preds = pd_yield_ft_df[join_cols]\n",
        "    pd_iter_preds['YTRUE' + str(iter)] = pd_yield_ft_df['YIELD-' + str(iter)]\n",
        "    pd_iter_preds['YPRED' + str(iter)] = predicted_trend\n",
        "\n",
        "    if (self.verbose > 2):\n",
        "      print('Leave-one-out cross-validation: iteration', iter)\n",
        "      print(pd_iter_preds.head(5))\n",
        "\n",
        "    return pd_iter_preds\n",
        "\n",
        "  def getL1outRMSE(self, cv_actual, cv_predicted):\n",
        "    \"\"\"Compute RMSE for leave-one-out predictions\"\"\"\n",
        "    return float(np.round(np.sqrt(mean_squared_error(cv_actual, cv_predicted)), 2))\n",
        "\n",
        "  def getMinRMSEIndex(self, cv_rmses):\n",
        "    \"\"\"Index of min rmse values\"\"\"\n",
        "    return np.nanargmin(cv_rmses)\n",
        "\n",
        "  def getL1OutCVRMSE(self, df, trend_window, join_cols, pred_year=None):\n",
        "    \"\"\"Run leave-one-out cross-validation and compute RMSE\"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    pd_yield_ft_df = self.getFixedWindowTrendFeatures(df, trend_window, pred_year)\n",
        "    pd_l1out_preds = None\n",
        "    for i in range(1, trend_window + 1):\n",
        "      pd_iter_preds = self.getL1OutCVPredictions(pd_yield_ft_df, trend_window,\n",
        "                                                 join_cols, i)\n",
        "      if (pd_l1out_preds is None):\n",
        "        pd_l1out_preds = pd_iter_preds\n",
        "      else:\n",
        "        pd_l1out_preds = pd_l1out_preds.merge(pd_iter_preds, on=join_cols)\n",
        "\n",
        "    region_years = pd_l1out_preds[join_cols].values\n",
        "    ytrue_cols = ['YTRUE' + str(i) for i in range(1, trend_window + 1)]\n",
        "    ypred_cols = ['YPRED' + str(i) for i in range(1, trend_window + 1)]\n",
        "    l1out_ytrue = pd_l1out_preds[ytrue_cols].values\n",
        "    l1out_ypred = pd_l1out_preds[ypred_cols].values\n",
        "    cv_rmse = []\n",
        "    for i in range(region_years.shape[0]):\n",
        "      cv_rmse.append(self.getL1outRMSE(l1out_ytrue[i, :],\n",
        "                                       l1out_ypred[i, :]))\n",
        "\n",
        "    pd_l1out_rmse = pd_yield_ft_df[join_cols]\n",
        "    pd_l1out_rmse['YIELD_TREND' + str(trend_window)] = pd_yield_ft_df['YIELD_TREND']\n",
        "    pd_l1out_rmse['CV_RMSE' + str(trend_window)] = cv_rmse\n",
        "\n",
        "    return pd_l1out_rmse\n",
        "\n",
        "  def getOptimalTrendWindows(self, df, pred_year=None, trend_windows=None):\n",
        "    \"\"\"\n",
        "    Compute optimal yield trend values based on leave-one-out\n",
        "    cross validation errors for different trend windows.\n",
        "    \"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    if (trend_windows is None):\n",
        "      trend_windows = self.trend_windows\n",
        "\n",
        "    pd_tw_rmses = None\n",
        "    for tw in trend_windows:\n",
        "      pd_l1out_rmse = self.getL1OutCVRMSE(df, tw, join_cols, pred_year)\n",
        "      if (pd_tw_rmses is None):\n",
        "        pd_tw_rmses = pd_l1out_rmse\n",
        "      else:\n",
        "        pd_tw_rmses = pd_tw_rmses.merge(pd_l1out_rmse, on=join_cols, how='left')\n",
        "\n",
        "    if (self.verbose > 2):\n",
        "      print('Leave-one-out cross-validation: RMSE')\n",
        "      print(pd_tw_rmses.sort_values(by=join_cols).head(5))\n",
        "\n",
        "    region_years = pd_tw_rmses[join_cols].values\n",
        "    tw_rmse_cols = ['CV_RMSE' + str(tw) for tw in trend_windows]\n",
        "    tw_trend_cols = ['YIELD_TREND' + str(tw) for tw in trend_windows]\n",
        "    tw_cv_rmses = pd_tw_rmses[tw_rmse_cols].values\n",
        "    tw_yield_trend = pd_tw_rmses[tw_trend_cols].values\n",
        "\n",
        "    opt_windows = []\n",
        "    yield_trend_preds = []\n",
        "    for i in range(region_years.shape[0]):\n",
        "      min_rmse_index = self.getMinRMSEIndex(tw_cv_rmses[i, :])\n",
        "      opt_windows.append(trend_windows[min_rmse_index])\n",
        "      yield_trend_preds.append(tw_yield_trend[i, min_rmse_index])\n",
        "\n",
        "    pd_opt_win_df = pd_tw_rmses[join_cols]\n",
        "    pd_opt_win_df['OPT_TW'] = opt_windows\n",
        "    pd_opt_win_df['YIELD_TREND'] = yield_trend_preds\n",
        "    if (self.verbose > 2):\n",
        "      print('Optimal trend windows')\n",
        "      print(pd_opt_win_df.sort_values(by=join_cols).head(5))\n",
        "\n",
        "    return pd_opt_win_df\n",
        "\n",
        "  def getOptimalWindowTrendFeatures(self, df, trend_windows=None):\n",
        "    \"\"\"\n",
        "    Get previous year yield values and predicted yield trend\n",
        "    by determining optimal trend window for each region and year.\n",
        "    NOTE: We have to select the same number of features, so we\n",
        "    select previous trend_windows[0] yield values.\n",
        "    \"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    if (trend_windows is None):\n",
        "      trend_windows = self.trend_windows\n",
        "\n",
        "    pd_yield_ft_df = self.getTrendWindowYields(df, trend_windows[0]).toPandas()\n",
        "    pd_opt_win_df = self.getOptimalTrendWindows(df, trend_windows=trend_windows)\n",
        "    pd_opt_win_df = pd_opt_win_df.drop(columns=['OPT_TW'])\n",
        "    pd_yield_ft_df = pd_yield_ft_df.merge(pd_opt_win_df, on=join_cols)\n",
        "\n",
        "    return pd_yield_ft_df\n",
        "\n",
        "  def getOptimalWindowTrend(self, df, reg_id, pred_year, trend_windows=None):\n",
        "    \"\"\"\n",
        "    Compute the optimal trend window for given region and year based on\n",
        "    leave-one-out cross validation errors for different trend windows.\n",
        "    \"\"\"\n",
        "    df = df.filter(df['IDREGION'] == reg_id)\n",
        "    pd_opt_win_df = self.getOptimalTrendWindows(df, pred_year, trend_windows)\n",
        "    if (len(pd_opt_win_df.index) == 0):\n",
        "      return None\n",
        "\n",
        "    reg_year_filter = (df['IDREGION'] == reg_id) & (df['FYEAR'] == pred_year)\n",
        "    pd_opt_win_df['ACTUAL'] = df.filter(reg_year_filter).select('YIELD').collect()[0][0]\n",
        "    pd_opt_win_df = pd_opt_win_df.rename(columns={'YIELD_TREND' : 'PREDICTED'})\n",
        "\n",
        "    return pd_opt_win_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UGNO_QZCsYf2"
      },
      "source": [
        "#### Create WOFOST, Meteo and Remote Sensing Features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0NpzcktZse_8"
      },
      "outputs": [],
      "source": [
        "#%%writefile run_feature_design.py\n",
        "def wofostMaxFeatureCols():\n",
        "  \"\"\"columns or indicators using max aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  max_cols = {\n",
        "      'p0' : [],\n",
        "      'p1' : [],\n",
        "      'p2' : ['WLIM_YB', 'TWC', 'WLAI'],\n",
        "      'p3' : [],\n",
        "      'p4' : ['WLIM_YB', 'WLIM_YS', 'TWC', 'WLAI'],\n",
        "      'p5' : [],\n",
        "  }\n",
        "\n",
        "  return max_cols\n",
        "\n",
        "def wofostAvgFeatureCols():\n",
        "  \"\"\"columns or indicators using avg aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  avg_cols = {\n",
        "      'p0' : [],\n",
        "      'p1' : [],\n",
        "      'p2' : ['RSM'],\n",
        "      'p3' : [],\n",
        "      'p4' : ['RSM'],\n",
        "      'p5' : [],\n",
        "  }\n",
        "\n",
        "  return avg_cols\n",
        "\n",
        "def wofostCountFeatureCols():\n",
        "  \"\"\"columns or indicators using count aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  count_cols = {\n",
        "      'p0' : [],\n",
        "      'p1' : ['RSM'],\n",
        "      'p2' : ['RSM'],\n",
        "      'p3' : ['RSM'],\n",
        "      'p4' : ['RSM'],\n",
        "      'p5' : [],\n",
        "  }\n",
        "\n",
        "  return count_cols\n",
        "\n",
        "# Meteo Feature ideas:\n",
        "# Two dry summers caused drop in ground water level:\n",
        "#   rainfall sums going back to second of half of previous year\n",
        "# Previous year: high production, prices low, invest less in crop\n",
        "#   Focus on another crop\n",
        "def meteoMaxFeatureCols():\n",
        "  \"\"\"columns or indicators using max aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  max_cols = { 'p0' : [], 'p1' : [], 'p2' : [], 'p3' : [], 'p4' : [], 'p5' : [] }\n",
        "\n",
        "  return max_cols\n",
        "\n",
        "def meteoAvgFeatureCols():\n",
        "  \"\"\"columns or indicators using avg aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  avg_cols = {\n",
        "      'p0' : ['TAVG', 'PREC', 'CWB'],\n",
        "      'p1' : ['TAVG', 'PREC'],\n",
        "      'p2' : ['TAVG', 'CWB'],\n",
        "      'p3' : ['PREC'],\n",
        "      'p4' : ['CWB'],\n",
        "      'p5' : ['PREC'],\n",
        "  }\n",
        "\n",
        "  return avg_cols\n",
        "\n",
        "def meteoCountFeatureCols():\n",
        "  \"\"\"columns or indicators using count aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  count_cols = {\n",
        "      'p0' : [],\n",
        "      'p1' : ['TMIN', 'PREC'],\n",
        "      'p2' : [],\n",
        "      'p3' : ['PREC', 'TMAX'],\n",
        "      'p4' : [],\n",
        "      'p5' : ['PREC'],\n",
        "  }\n",
        "\n",
        "  return count_cols\n",
        "\n",
        "def rsMaxFeatureCols():\n",
        "  \"\"\"columns or indicators using max aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  max_cols = { 'p0' : [], 'p1' : [], 'p2' : [], 'p3' : [], 'p4' : [], 'p5' : [] }\n",
        "\n",
        "  return max_cols\n",
        "\n",
        "def rsAvgFeatureCols():\n",
        "  \"\"\"columns or indicators using avg aggregation\"\"\"\n",
        "  # must be in sync with crop calendar periods\n",
        "  avg_cols = {\n",
        "      'p0' : [],\n",
        "      'p1' : [],\n",
        "      'p2' : ['FAPAR'],\n",
        "      'p3' : [],\n",
        "      'p4' : ['FAPAR'],\n",
        "      'p5' : [],\n",
        "  }\n",
        "\n",
        "  return avg_cols\n",
        "\n",
        "def convertFeaturesToPandas(ft_dfs, join_cols):\n",
        "  \"\"\"Convert features to pandas and merge\"\"\"\n",
        "  train_ft_df = ft_dfs[0]\n",
        "  test_ft_df = ft_dfs[1]\n",
        "  train_ft_df = train_ft_df.withColumnRenamed('CAMPAIGN_YEAR', 'FYEAR')\n",
        "  test_ft_df = test_ft_df.withColumnRenamed('CAMPAIGN_YEAR', 'FYEAR')\n",
        "  pd_train_df = train_ft_df.toPandas()\n",
        "  pd_test_df = test_ft_df.toPandas()\n",
        "\n",
        "  return [pd_train_df, pd_test_df]\n",
        "\n",
        "def dropZeroColumns(pd_ft_dfs):\n",
        "  \"\"\"Drop columns which have all zeros in training data\"\"\"\n",
        "  pd_train_df = pd_ft_dfs[0]\n",
        "  pd_test_df = pd_ft_dfs[1]\n",
        "\n",
        "  pd_train_df = pd_train_df.loc[:, (pd_train_df != 0.0).any(axis=0)]\n",
        "  pd_train_df = pd_train_df.dropna(axis=1)\n",
        "  pd_test_df = pd_test_df[pd_train_df.columns]\n",
        "\n",
        "  return [pd_train_df, pd_test_df]\n",
        "\n",
        "def printFeatureData(pd_feature_dfs, join_cols):\n",
        "  for src in pd_feature_dfs:\n",
        "    pd_train_fts = pd_feature_dfs[src][0]\n",
        "    if (pd_train_fts is None):\n",
        "      continue\n",
        "\n",
        "    pd_test_fts = pd_feature_dfs[src][1]\n",
        "    all_cols = list(pd_train_fts.columns)\n",
        "    aggr_cols = [ c for c in all_cols if (('avg' in c) or ('max' in c))]\n",
        "    if (len(aggr_cols) > 0):\n",
        "      print('\\n', src, 'Aggregate Features: Training')\n",
        "      print(pd_train_fts[join_cols + aggr_cols].head(5))\n",
        "      print('\\n', src, 'Aggregate Features: Test')\n",
        "      print(pd_test_fts[join_cols + aggr_cols].head(5))\n",
        "\n",
        "    ext_cols = [ c for c in all_cols if (('Z+' in c) or ('Z-' in c) or\n",
        "                                         ('lt' in c) or ('gt' in c))]\n",
        "    if (len(ext_cols) > 0):\n",
        "      print('\\n', src, 'Features for Extreme Conditions: Training')\n",
        "      print(pd_train_fts[join_cols + ext_cols].head(5))\n",
        "      print('\\n', src, 'Features for Extreme Conditions: Test')\n",
        "      print(pd_test_fts[join_cols + ext_cols].head(5))\n",
        "\n",
        "def createFeatures(cyp_config, cyp_featurizer, train_test_dfs,\n",
        "                   summary_dfs, log_fh):\n",
        "  \"\"\"Create WOFOST, Meteo and Remote Sensing features\"\"\"\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  use_remote_sensing = cyp_config.useRemoteSensing()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  wofost_train_df = train_test_dfs['WOFOST'][0]\n",
        "  wofost_test_df = train_test_dfs['WOFOST'][1]\n",
        "  meteo_train_df = train_test_dfs['METEO'][0]\n",
        "  meteo_test_df = train_test_dfs['METEO'][1]\n",
        "  yield_train_df = train_test_dfs['YIELD'][0]\n",
        "\n",
        "  dvs_summary = summary_dfs['WOFOST_DVS']\n",
        "\n",
        "  # Filter out years before min yield year\n",
        "  yield_min_year = yield_train_df.agg({\"FYEAR\": \"MIN\"}).collect()[0][0]\n",
        "  print('Yield min year', yield_min_year)\n",
        "\n",
        "  wofost_train_df = wofost_train_df.filter(wofost_train_df.CAMPAIGN_YEAR >= yield_min_year)\n",
        "  meteo_train_df = meteo_train_df.filter(meteo_train_df.CAMPAIGN_YEAR >= yield_min_year)\n",
        "  dvs_summary = dvs_summary.filter(dvs_summary.CAMPAIGN_YEAR >= yield_min_year)\n",
        "\n",
        "  rs_train_df = None\n",
        "  rs_test_df = None\n",
        "  if (use_remote_sensing):\n",
        "    rs_train_df = train_test_dfs['REMOTE_SENSING'][0]\n",
        "    rs_test_df = train_test_dfs['REMOTE_SENSING'][1]\n",
        "    rs_train_df = rs_train_df.filter(rs_train_df.CAMPAIGN_YEAR >= yield_min_year)\n",
        "    rs_test_df = rs_test_df.filter(rs_test_df.CAMPAIGN_YEAR >= yield_min_year)\n",
        "\n",
        "  join_cols = ['IDREGION', 'CAMPAIGN_YEAR']\n",
        "  aggr_ft_cols = {\n",
        "      'WOFOST' : [wofostMaxFeatureCols(), wofostAvgFeatureCols()],\n",
        "      'METEO' : [meteoMaxFeatureCols(), meteoAvgFeatureCols()],\n",
        "  }\n",
        "\n",
        "  count_ft_cols = {\n",
        "      'WOFOST' : wofostCountFeatureCols(),\n",
        "      'METEO' : meteoCountFeatureCols(),\n",
        "  }\n",
        "\n",
        "  train_ft_src_dfs = {\n",
        "      'WOFOST' : wofost_train_df,\n",
        "      'METEO' : meteo_train_df,\n",
        "  }\n",
        "\n",
        "  test_ft_src_dfs = {\n",
        "      'WOFOST' : wofost_test_df,\n",
        "      'METEO' : meteo_test_df,\n",
        "  }\n",
        "\n",
        "  if (use_remote_sensing):\n",
        "    train_ft_src_dfs['REMOTE_SENSING'] = rs_train_df\n",
        "    test_ft_src_dfs['REMOTE_SENSING'] = rs_test_df\n",
        "    aggr_ft_cols['REMOTE_SENSING'] = [rsMaxFeatureCols(), rsAvgFeatureCols()]\n",
        "    count_ft_cols['REMOTE_SENSING'] = {}\n",
        "\n",
        "  crop_cal_train = dvs_summary\n",
        "  crop_cal_test = dvs_summary\n",
        "\n",
        "  train_ft_dfs = {}\n",
        "  test_ft_dfs = {}\n",
        "  for ft_src in train_ft_src_dfs:\n",
        "    train_ft_dfs[ft_src] = cyp_featurizer.extractFeatures(train_ft_src_dfs[ft_src],\n",
        "                                                          ft_src,\n",
        "                                                          crop_cal_train,\n",
        "                                                          aggr_ft_cols[ft_src][0],\n",
        "                                                          aggr_ft_cols[ft_src][1],\n",
        "                                                          count_ft_cols[ft_src],\n",
        "                                                          join_cols,\n",
        "                                                          True)\n",
        "    test_ft_dfs[ft_src] = cyp_featurizer.extractFeatures(test_ft_src_dfs[ft_src],\n",
        "                                                         ft_src,\n",
        "                                                         crop_cal_test,\n",
        "                                                         aggr_ft_cols[ft_src][0],\n",
        "                                                         aggr_ft_cols[ft_src][1],\n",
        "                                                         count_ft_cols[ft_src],\n",
        "                                                         join_cols)\n",
        "\n",
        "  pd_conversion_dict = {\n",
        "      'WOFOST' : [ train_ft_dfs['WOFOST'], test_ft_dfs['WOFOST'] ],\n",
        "      'METEO' : [ train_ft_dfs['METEO'], test_ft_dfs['METEO'] ],\n",
        "  }\n",
        "\n",
        "  if (use_remote_sensing):\n",
        "      pd_conversion_dict['REMOTE_SENSING'] = [ train_ft_dfs['REMOTE_SENSING'], test_ft_dfs['REMOTE_SENSING'] ]\n",
        "\n",
        "  pd_feature_dfs = {}\n",
        "  for ft_src in pd_conversion_dict:\n",
        "    pd_feature_dfs[ft_src] = convertFeaturesToPandas(pd_conversion_dict[ft_src], join_cols)\n",
        "\n",
        "  # Check and drop features with all zeros (possible in early season prediction).\n",
        "  for ft_src in pd_feature_dfs:\n",
        "    pd_feature_dfs[ft_src] = dropZeroColumns(pd_feature_dfs[ft_src])\n",
        "\n",
        "  if (debug_level > 1):\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    printFeatureData(pd_feature_dfs, join_cols)\n",
        "\n",
        "  return pd_feature_dfs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ERZ4JYeyss7m"
      },
      "source": [
        "#### Create Yield Trend Features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nbqouti4swJ0"
      },
      "outputs": [],
      "source": [
        "#%%writefile run_trend_feature_design.py\n",
        "def createYieldTrendFeatures(cyp_config, cyp_trend_est,\n",
        "                             yield_train_df, yield_test_df, test_years):\n",
        "  \"\"\"Create yield trend features\"\"\"\n",
        "  join_cols = ['IDREGION', 'FYEAR']\n",
        "  find_optimal = cyp_config.findOptimalTrendWindow()\n",
        "  trend_window = cyp_config.getTrendWindows()[0]\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  yield_df = yield_train_df.union(yield_test_df.select(yield_train_df.columns))\n",
        "\n",
        "  if (find_optimal):\n",
        "    pd_train_features = cyp_trend_est.getOptimalWindowTrendFeatures(yield_train_df)\n",
        "    pd_test_features = cyp_trend_est.getOptimalWindowTrendFeatures(yield_df)\n",
        "  else:\n",
        "    pd_train_features = cyp_trend_est.getFixedWindowTrendFeatures(yield_train_df)\n",
        "    pd_test_features = cyp_trend_est.getFixedWindowTrendFeatures(yield_df)\n",
        "\n",
        "  pd_test_features = pd_test_features[pd_test_features['FYEAR'].isin(test_years)]\n",
        "  prev_year_cols = ['YEAR-' + str(i) for i in range(1, trend_window + 1)]\n",
        "  pd_train_features = pd_train_features.drop(columns=prev_year_cols)\n",
        "  pd_test_features = pd_test_features.drop(columns=prev_year_cols)\n",
        "\n",
        "  if (debug_level > 1):\n",
        "    print('\\nYield Trend Features: Train')\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "    print(pd_train_features.sort_values(by=join_cols).head(5))\n",
        "    print('Total', len(pd_train_features.index), 'rows')\n",
        "    print('\\nYield Trend Features: Test')\n",
        "    print(pd_test_features.sort_values(by=join_cols).head(5))\n",
        "    print('Total', len(pd_test_features.index), 'rows')\n",
        "\n",
        "  return pd_train_features, pd_test_features"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cIDmOfx9KKuF"
      },
      "source": [
        "### Combine features and labels\n",
        "\n",
        "Combine wofost, meteo and soil with remote sensing. Combine with centroids or yield trend both if configured. Combine with yield data in the end."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KsoMBrpIKKuH"
      },
      "outputs": [],
      "source": [
        "#%%writefile combine_features.py\n",
        "def combineFeaturesLabels(cyp_config, sqlCtx,\n",
        "                          prep_train_test_dfs, pd_feature_dfs,\n",
        "                          join_cols, log_fh):\n",
        "  \"\"\"\n",
        "  Combine wofost, meteo and soil with remote sensing. Combine centroids\n",
        "  and yield trend if configured. Combine with yield data in the end.\n",
        "  If configured, save features to a CSV file.\n",
        "  \"\"\"\n",
        "  pd_soil_df = prep_train_test_dfs['SOIL'][0].toPandas()\n",
        "  pd_yield_train_df = prep_train_test_dfs['YIELD'][0].toPandas()\n",
        "  pd_yield_test_df = prep_train_test_dfs['YIELD'][1].toPandas()\n",
        "\n",
        "  # Feature dataframes have already been converted to pandas\n",
        "  pd_wofost_train_ft = pd_feature_dfs['WOFOST'][0]\n",
        "  pd_wofost_test_ft = pd_feature_dfs['WOFOST'][1]\n",
        "  pd_meteo_train_ft = pd_feature_dfs['METEO'][0]\n",
        "  pd_meteo_test_ft = pd_feature_dfs['METEO'][1]\n",
        "\n",
        "  crop = cyp_config.getCropName()\n",
        "  country = cyp_config.getCountryCode()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  use_centroids = cyp_config.useCentroids()\n",
        "  use_remote_sensing = cyp_config.useRemoteSensing()\n",
        "  save_features = cyp_config.saveFeatures()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  \n",
        "  combine_info = '\\nCombine Features and Labels'\n",
        "  combine_info += '\\n---------------------------'\n",
        "  yield_min_year = pd_yield_train_df['FYEAR'].min()\n",
        "  combine_info += '\\nYield min year ' + str(yield_min_year) + '\\n'\n",
        "\n",
        "  # start with static SOIL data\n",
        "  pd_train_df = pd_soil_df.copy(deep=True)\n",
        "  pd_test_df = pd_soil_df.copy(deep=True)\n",
        "  combine_info += '\\nData size after including SOIL data: '\n",
        "  combine_info += '\\nTrain ' + str(len(pd_train_df.index)) + ' rows.'\n",
        "  combine_info += '\\nTest ' + str(len(pd_test_df.index)) + ' rows.\\n'\n",
        "\n",
        "  if (use_centroids):\n",
        "    # combine with region centroids\n",
        "    pd_centroids_df = prep_train_test_dfs['CENTROIDS'][0].toPandas()\n",
        "    pd_train_df = pd_train_df.merge(pd_centroids_df, on=['IDREGION'])\n",
        "    pd_test_df = pd_test_df.merge(pd_centroids_df, on='IDREGION')\n",
        "    combine_info += '\\nData size after including CENTROIDS data: '\n",
        "    combine_info += '\\nTrain ' + str(len(pd_train_df.index)) + ' rows.'\n",
        "    combine_info += '\\nTest ' + str(len(pd_test_df.index)) + ' rows.\\n'\n",
        "\n",
        "  # combine with WOFOST features\n",
        "  static_cols = list(pd_train_df.columns)\n",
        "  pd_train_df = pd_train_df.merge(pd_wofost_train_ft, on=['IDREGION'])\n",
        "  pd_test_df = pd_test_df.merge(pd_wofost_test_ft, on=['IDREGION'])\n",
        "  wofost_cols = list(pd_wofost_train_ft.columns)\n",
        "  col_order = ['IDREGION', 'FYEAR'] + static_cols[1:] + wofost_cols[2:]\n",
        "  pd_train_df = pd_train_df[col_order]\n",
        "  pd_test_df = pd_test_df[col_order]\n",
        "  combine_info += '\\nData size after including WOFOST features: '\n",
        "  combine_info += '\\nTrain ' + str(len(pd_train_df.index)) + ' rows.'\n",
        "  combine_info += '\\nTest ' + str(len(pd_test_df.index)) + ' rows.\\n'\n",
        "\n",
        "  # combine with METEO features\n",
        "  pd_train_df = pd_train_df.merge(pd_meteo_train_ft, on=join_cols)\n",
        "  pd_test_df = pd_test_df.merge(pd_meteo_test_ft, on=join_cols)\n",
        "  combine_info += '\\nData size after including METEO features: '\n",
        "  combine_info += '\\nTrain ' + str(len(pd_train_df.index)) + ' rows.'\n",
        "  combine_info += '\\nTest ' + str(len(pd_test_df.index)) + ' rows.\\n'\n",
        "\n",
        "  # combine with remote sensing features\n",
        "  if (use_remote_sensing):\n",
        "    pd_rs_train_ft = pd_feature_dfs['REMOTE_SENSING'][0]\n",
        "    pd_rs_test_ft = pd_feature_dfs['REMOTE_SENSING'][1]\n",
        "\n",
        "    pd_train_df = pd_train_df.merge(pd_rs_train_ft, on=join_cols)\n",
        "    pd_test_df = pd_test_df.merge(pd_rs_test_ft, on=join_cols)\n",
        "    combine_info += '\\nData size after including REMOTE_SENSING features: '\n",
        "    combine_info += '\\nTrain ' + str(len(pd_train_df.index)) + ' rows.'\n",
        "    combine_info += '\\nTest ' + str(len(pd_test_df.index)) + ' rows.\\n'\n",
        "\n",
        "  if (use_yield_trend):\n",
        "    # combine with yield trend features\n",
        "    pd_yield_trend_train_ft = pd_feature_dfs['YIELD_TREND'][0]\n",
        "    pd_yield_trend_test_ft = pd_feature_dfs['YIELD_TREND'][1]\n",
        "    pd_train_df = pd_train_df.merge(pd_yield_trend_train_ft, on=join_cols)\n",
        "    pd_test_df = pd_test_df.merge(pd_yield_trend_test_ft, on=join_cols)\n",
        "    combine_info += '\\nData size after including yield trend features: '\n",
        "    combine_info += '\\nTrain ' + str(len(pd_train_df.index)) + ' rows.'\n",
        "    combine_info += '\\nTest ' + str(len(pd_test_df.index)) + ' rows.\\n'\n",
        "\n",
        "  # combine with yield data\n",
        "  pd_train_df = pd_train_df.merge(pd_yield_train_df, on=join_cols)\n",
        "  pd_test_df = pd_test_df.merge(pd_yield_test_df, on=join_cols)\n",
        "  pd_train_df = pd_train_df.sort_values(by=join_cols)\n",
        "  pd_test_df = pd_test_df.sort_values(by=join_cols)\n",
        "  combine_info += '\\nData size after including yield (label) data: '\n",
        "  combine_info += '\\nTrain ' + str(len(pd_train_df.index)) + ' rows.'\n",
        "  combine_info += '\\nTest ' + str(len(pd_test_df.index)) + ' rows.\\n'\n",
        "\n",
        "  log_fh.write(combine_info + '\\n')\n",
        "  if (debug_level > 1):\n",
        "    print(combine_info)\n",
        "    print('\\nAll Features and labels: Training')\n",
        "    print(pd_train_df.head(5))\n",
        "    print('\\nAll Features and labels: Test')\n",
        "    print(pd_test_df.head(5))\n",
        "\n",
        "  if (save_features):\n",
        "    early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "    early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "    feature_file_path = cyp_config.getOutputPath()\n",
        "    features_file = getFeatureFilename(crop, country, use_yield_trend,\n",
        "                                       early_season_prediction, early_season_end)\n",
        "    save_ft_path = feature_file_path + '/' + features_file\n",
        "    save_ft_info = '\\nSaving features to: ' + save_ft_path + '[train, test].csv'\n",
        "    log_fh.write(save_ft_info + '\\n')\n",
        "    if (debug_level > 1):\n",
        "      print(save_ft_info)\n",
        "\n",
        "    pd_train_df.to_csv(save_ft_path + '_train.csv', index=False, header=True)\n",
        "    pd_test_df.to_csv(save_ft_path + '_test.csv', index=False, header=True)\n",
        "\n",
        "    # NOTE: In some environments, Spark can write, but pandas cannot.\n",
        "    # In such cases, use the following code.\n",
        "    # spark_train_df = sqlCtx.createDataFrame(pd_train_df)\n",
        "    # spark_train_df.coalesce(1).write.option('header','true').mode('overwrite').csv(save_ft_path + '_train')\n",
        "    # spark_test_df = sqlCtx.createDataFrame(pd_test_df)\n",
        "    # spark_test_df.coalesce(1).write.option('header','true').mode('overwrite').csv(save_ft_path + '_test')\n",
        "\n",
        "  return pd_train_df, pd_test_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQb85mf_wgvS"
      },
      "source": [
        "### Load Saved Features and Labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R5TyGb2iwnFr"
      },
      "outputs": [],
      "source": [
        "#%%writefile load_saved_features.py\n",
        "import pandas as pd\n",
        "\n",
        "def loadSavedFeaturesLabels(cyp_config, spark):\n",
        "  \"\"\"Load saved features from a CSV file\"\"\"\n",
        "  crop = cyp_config.getCropName()\n",
        "  country = cyp_config.getCountryCode()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  feature_file_path = cyp_config.getOutputPath()\n",
        "  feature_file = getFeatureFilename(crop, country, use_yield_trend,\n",
        "                                    early_season_prediction, early_season_end)\n",
        "\n",
        "  load_ft_path = feature_file_path + '/' + feature_file\n",
        "  pd_train_df = pd.read_csv(load_ft_path + '_train.csv', header=0)\n",
        "  pd_test_df = pd.read_csv(load_ft_path + '_test.csv', header=0)\n",
        "\n",
        "  # NOTE: In some environments, Spark can read, but pandas cannot.\n",
        "  # In such cases, use the following code.\n",
        "  # spark_train_df = spark.read.csv(load_ft_path + '_train.csv', header=True, inferSchema=True)\n",
        "  # spark_test_df = spark.read.csv(load_ft_path + '_test.csv', header=True, inferSchema=True)\n",
        "  # pd_train_df = spark_train_df.toPandas()\n",
        "  # pd_test_df = spark_test_df.toPandas()\n",
        "\n",
        "  if (debug_level > 1):\n",
        "    print('\\nAll Features and labels')\n",
        "    print(pd_train_df.head(5))\n",
        "    print(pd_test_df.head(5))\n",
        "\n",
        "  return pd_train_df, pd_test_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8PjL0Nv1jEpR"
      },
      "source": [
        "### Machine Learning using scikit learn"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "drvMnHJBuBwK"
      },
      "source": [
        "#### Feature Selector Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HBcbtxArL2Ld"
      },
      "outputs": [],
      "source": [
        "#%%writefile feature_selection.py\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.feature_selection import SelectKBest\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "from sklearn.feature_selection import RFE\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.utils import parallel_backend\n",
        "\n",
        "class CYPFeatureSelector:\n",
        "  def __init__(self, cyp_config, X_train, Y_train, custom_cv, all_features):\n",
        "    self.X_train = X_train\n",
        "    self.Y_train = Y_train\n",
        "    self.custom_cv = custom_cv\n",
        "    self.all_features = all_features\n",
        "    self.scaler = cyp_config.getFeatureScaler()\n",
        "    self.cv_metric = cyp_config.getFeatureSelectionCVMetric()\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "\n",
        "  def setCustomCV(self, custom_cv):\n",
        "    \"\"\"Set custom K-Fold validation splits\"\"\"\n",
        "    self.custom_cv = custom_cv\n",
        "\n",
        "  def setFeatures(self, features):\n",
        "    \"\"\"Set the list of all features\"\"\"\n",
        "    self.all_features = all_features\n",
        "\n",
        "  # K-fold validation to find the optimal number of features\n",
        "  # and optimal hyperparameters for estimator.\n",
        "  def featureSelectionGridSearch(self, selector, est, param_grid):\n",
        "    \"\"\"Use grid search with k-fold validation to optimize the number of features\"\"\"\n",
        "    X_train_copy = np.copy(self.X_train)\n",
        "    pipeline = Pipeline([(\"scaler\", self.scaler),\n",
        "                         (\"selector\", selector),\n",
        "                         (\"estimator\", est)])\n",
        "\n",
        "    grid_search = GridSearchCV(estimator=pipeline, param_grid=param_grid,\n",
        "                               scoring=self.cv_metric, cv=self.custom_cv)\n",
        "    with parallel_backend('spark', n_jobs=-1):\n",
        "      grid_search.fit(X_train_copy, np.ravel(self.Y_train))\n",
        "\n",
        "    best_score = grid_search.best_score_\n",
        "    best_estimator = grid_search.best_estimator_\n",
        "    indices = []\n",
        "    # feature selectors should have 'get_support' function\n",
        "    if ((isinstance(selector, SelectFromModel)) or (isinstance(selector, SelectKBest)) or\n",
        "        (isinstance(selector, RFE))):\n",
        "      sel = grid_search.best_estimator_.named_steps['selector']\n",
        "      indices = sel.get_support(indices=True)\n",
        "\n",
        "    result = {\n",
        "        'indices' : indices,\n",
        "        'best_score' : best_score,\n",
        "    }\n",
        "\n",
        "    return result\n",
        "\n",
        "  # Compare different feature selectors using cross validation score.\n",
        "  # Also compare combined features with the best individual feature selector.\n",
        "  def compareFeatureSelectors(self, feature_selectors,\n",
        "                              est, est_param_grid):\n",
        "    \"\"\"Compare feature selectors based on K-fold validation scores\"\"\"\n",
        "    combined_indices = []\n",
        "    # set it to a large negative value\n",
        "    best_score = -1000\n",
        "    best_indices = []\n",
        "    best_selector = ''\n",
        "    fs_scores = {}\n",
        "    for sel_name in feature_selectors:\n",
        "      selector = feature_selectors[sel_name]['selector']\n",
        "      sel_param_grid = feature_selectors[sel_name]['param_grid']\n",
        "      param_grid = sel_param_grid.copy()\n",
        "      param_grid.update(est_param_grid)\n",
        "      result = self.featureSelectionGridSearch(selector, est, param_grid)\n",
        "      param_grid.clear()\n",
        "\n",
        "      if (self.verbose > 2):\n",
        "        print('\\nFeature selection using', sel_name)\n",
        "        print('Best cross-validation', self.cv_metric + ':',\n",
        "              np.round(result['best_score'], 3))\n",
        "\n",
        "        print('\\nSelected Features:')\n",
        "        print('-------------------')\n",
        "        printFeatures(self.all_features, result['indices'])\n",
        "\n",
        "      combined_indices = list(set(combined_indices) | set(result['indices']))\n",
        "\n",
        "      fs_scores[sel_name] = result['best_score']\n",
        "\n",
        "      if (result['best_score'] > best_score):\n",
        "        best_indices = result['indices']\n",
        "        best_score = result['best_score']\n",
        "        best_selector = sel_name\n",
        "\n",
        "    result = {\n",
        "        'best_selector' : best_selector,\n",
        "        'best_score' : best_score,\n",
        "        'best_indices' : best_indices,\n",
        "        'combined_indices' : combined_indices,\n",
        "        'scores' : fs_scores\n",
        "    }\n",
        "\n",
        "    return result\n",
        "\n",
        "  # Between the optimal sets for each estimator select the set with the higher score.\n",
        "  def selectOptimalFeatures(self, feature_selectors,\n",
        "                            est, est_name, est_param_grid, log_fh):\n",
        "    \"\"\"\n",
        "    Select optimal features by comparing individual feature selectors\n",
        "    and combined features\n",
        "    \"\"\"\n",
        "    X_train_selected = None\n",
        "    # set it to a large negative value\n",
        "    best_score = -1000\n",
        "    fs_summary = {}\n",
        "    row_count = 1\n",
        "\n",
        "    est_info = '\\nEstimator: ' + est_name\n",
        "    est_info += '\\n---------------------------'\n",
        "    log_fh.write(est_info)\n",
        "    print(est_info)\n",
        "\n",
        "    result = self.compareFeatureSelectors(feature_selectors,\n",
        "                                          est, est_param_grid)\n",
        "\n",
        "    # result includes\n",
        "    # - 'best_selector' : name of the best selector\n",
        "    # - 'best_score' : cv score of features selected with estimator\n",
        "    # - 'best_indices' : indices of features selected\n",
        "    # - 'best_estimator' : estimator with settings that gave best score\n",
        "    # - 'combined_indices' : indices of combined features\n",
        "    # - 'scores' : dict with scores of all feature selectors\n",
        "\n",
        "    for sel_name in result['scores']:\n",
        "      est_sel_row = [est_name, sel_name, np.round(result['scores'][sel_name], 2)]\n",
        "      fs_summary['row' + str(row_count)] = est_sel_row\n",
        "      row_count += 1\n",
        "\n",
        "    # calculate cross-validation score for combined features\n",
        "    X_train_selected = self.X_train[:, result['combined_indices']]\n",
        "    pipeline = Pipeline([(\"scaler\", self.scaler), (\"estimator\", est)])\n",
        "    grid_search = GridSearchCV(estimator=pipeline, param_grid=est_param_grid,\n",
        "                               scoring=self.cv_metric, cv=self.custom_cv)\n",
        "    X_train_selected_copy = np.copy(X_train_selected)\n",
        "    with parallel_backend('spark', n_jobs=-1):\n",
        "      grid_search.fit(X_train_selected_copy, np.ravel(self.Y_train))\n",
        "\n",
        "    combined_score = grid_search.best_score_\n",
        "    combo_sel_row = [est_name, 'combined', np.round(combined_score, 2)]\n",
        "\n",
        "    fs_summary['row' + str(row_count)] = combo_sel_row\n",
        "    row_count += 1\n",
        "\n",
        "    # We check if combined features give us a better score\n",
        "    # than the best feature selection method\n",
        "    if (combined_score < result['best_score']):\n",
        "      selector = result['best_selector']\n",
        "      selected_indices = result['best_indices']\n",
        "    else:\n",
        "      selected_indices = result['combined_indices']\n",
        "\n",
        "    fs_df_columns = ['estimator', 'selector', self.cv_metric]\n",
        "    fs_df = pd.DataFrame.from_dict(fs_summary, orient='index', columns=fs_df_columns)\n",
        "\n",
        "    ftsel_summary_info = '\\nFeature Selection Summary'\n",
        "    ftsel_summary_info += '\\n---------------------------'\n",
        "    ftsel_summary_info += '\\n' + fs_df.to_string(index=False) + '\\n'\n",
        "    log_fh.write(ftsel_summary_info)\n",
        "    print(ftsel_summary_info)\n",
        "\n",
        "    return selected_indices"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xWdWViKkuFM1"
      },
      "source": [
        "#### Algorithm Evaluator Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LutX8Ih9MsGx"
      },
      "outputs": [],
      "source": [
        "#%%writefile algorithm_evaluation.py\n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.linear_model import Ridge\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.utils import parallel_backend\n",
        "\n",
        "class CYPAlgorithmEvaluator:\n",
        "  def __init__(self, cyp_config, custom_cv):\n",
        "    self.scaler = cyp_config.getFeatureScaler()\n",
        "    self.estimators = cyp_config.getEstimators()\n",
        "    self.custom_cv = custom_cv\n",
        "    self.cv_metric = cyp_config.getAlgorithmTrainingCVMetric()\n",
        "    self.metrics = cyp_config.getEvaluationMetrics()\n",
        "    self.verbose = cyp_config.getDebugLevel()\n",
        "    self.use_yield_trend = cyp_config.useYieldTrend()\n",
        "    self.trend_windows = cyp_config.getTrendWindows()\n",
        "    self.predict_residuals = cyp_config.predictYieldResiduals()\n",
        "\n",
        "  def setCustomCV(self, custom_cv):\n",
        "    \"\"\"Set custom K-Fold validation splits\"\"\"\n",
        "    self.custom_cv = custom_cv\n",
        "\n",
        "  # Nash-Sutcliffe Model Efficiency\n",
        "  def nse(self, Y_true, Y_pred):\n",
        "    \"\"\"\n",
        "        Nash Sutcliffe efficiency coefficient\n",
        "        input:\n",
        "          Y_pred: predicted\n",
        "          Y_true: observed\n",
        "        output:\n",
        "          nse: Nash Sutcliffe efficient coefficient\n",
        "        \"\"\"\n",
        "    return (1 - np.sum(np.square(Y_pred - Y_true))/np.sum(np.square(Y_true - np.mean(Y_true))))\n",
        "\n",
        "  def updateAlgorithmsSummary(self, alg_summary, alg_name,\n",
        "                              train_scores, test_scores):\n",
        "    \"\"\"Update algorithms summary with scores for given algorithm\"\"\"\n",
        "    alg_row = [alg_name]\n",
        "    alg_index = len(alg_summary)\n",
        "    for met in train_scores:\n",
        "      alg_row += [train_scores[met], test_scores[met]]\n",
        "\n",
        "    alg_summary['row' + str(alg_index)] = alg_row\n",
        "\n",
        "  def createPredictionDataFrames(self, Y_train_pred, Y_test_pred, data_cols):\n",
        "    \"\"\"\"Create pandas data frames from true and predicted values\"\"\"\n",
        "    pd_train_df = pd.DataFrame(data=Y_train_pred, columns=data_cols)\n",
        "    pd_test_df = pd.DataFrame(data=Y_test_pred, columns=data_cols)\n",
        "\n",
        "    return pd_train_df, pd_test_df\n",
        "\n",
        "  def printPredictionDataFrames(self, pd_train_df, pd_test_df, log_fh):\n",
        "    \"\"\"\"Print true and predicted values from pandas data frames\"\"\"\n",
        "    train_info = '\\nYield Predictions Training Set'\n",
        "    train_info += '\\n--------------------------------'\n",
        "    train_info += '\\n' + pd_train_df.head(6).to_string(index=False)\n",
        "    log_fh.write(train_info + '\\n')\n",
        "    print(train_info)\n",
        "\n",
        "    test_info = '\\nYield Predictions Test Set'\n",
        "    test_info += '\\n--------------------------------'\n",
        "    test_info += '\\n' + pd_test_df.head(6).to_string(index=False)\n",
        "    log_fh.write(test_info + '\\n')\n",
        "    print(test_info)\n",
        "\n",
        "  def getNullMethodPredictions(self, Y_train_full, Y_test_full, log_fh):\n",
        "    \"\"\"\n",
        "    The Null method or poor man's prediction. Y_*_full includes IDREGION, FYEAR.\n",
        "    If using yield trend, Y_*_full also include YIELD_TREND.\n",
        "    The null method predicts the YIELD_TREND or the average of the training set.\n",
        "    \"\"\"\n",
        "    Y_train = Y_train_full[:, 2]\n",
        "    if (self.use_yield_trend):\n",
        "      Y_train = Y_train_full[:, 3]\n",
        "\n",
        "    min_yield = np.round(np.min(Y_train), 2)\n",
        "    max_yield = np.round(np.max(Y_train), 2)\n",
        "    avg_yield = np.round(np.mean(Y_train), 2)\n",
        "    median_yield = np.round(np.median(np.ravel(Y_train)), 2)\n",
        "\n",
        "    null_method_label = 'Null Method: '\n",
        "    if (self.use_yield_trend):\n",
        "      null_method_label += 'Predicting linear yield trend:'\n",
        "      data_cols = ['IDREGION', 'FYEAR', 'YIELD_TREND', 'YIELD']\n",
        "      pd_train_df, pd_test_df = self.createPredictionDataFrames(Y_train_full, Y_test_full,\n",
        "                                                                data_cols)\n",
        "    else:\n",
        "      Y_train_full_n = np.insert(Y_train_full, 2, avg_yield, axis=1)\n",
        "      Y_test_full_n = np.insert(Y_test_full, 2, avg_yield, axis=1)\n",
        "      null_method_label += 'Predicting average of the training set:'\n",
        "      data_cols = ['IDREGION', 'FYEAR', 'YIELD_PRED', 'YIELD']\n",
        "      pd_train_df, pd_test_df = self.createPredictionDataFrames(Y_train_full_n, Y_test_full_n,\n",
        "                                                                data_cols)\n",
        "\n",
        "    null_method_info = '\\n' + null_method_label\n",
        "    null_method_info += '\\nMin Yield: ' + str(min_yield) + ', Max Yield: ' + str(max_yield)\n",
        "    null_method_info += '\\nMedian Yield: ' + str(median_yield) + ', Mean Yield: ' + str(avg_yield)\n",
        "    log_fh.write(null_method_info + '\\n')\n",
        "    print(null_method_info)\n",
        "    self.printPredictionDataFrames(pd_train_df, pd_test_df, log_fh)\n",
        "\n",
        "    result = {\n",
        "        'train' : pd_train_df,\n",
        "        'test' : pd_test_df,\n",
        "    }\n",
        "\n",
        "    return result\n",
        "\n",
        "  def evaluateNullMethodPredictions(self, pd_train_df, pd_test_df, alg_summary):\n",
        "    \"\"\"Evaluate the predictions of the null method and add an entry to alg_summary\"\"\"\n",
        "    Y_train = pd_train_df['YIELD'].values\n",
        "    Y_test = pd_test_df['YIELD'].values\n",
        "\n",
        "    if (self.use_yield_trend):  \n",
        "      alg_name = 'trend'\n",
        "      Y_pred_train = pd_train_df['YIELD_TREND'].values\n",
        "      Y_pred_test = pd_test_df['YIELD_TREND'].values\n",
        "    else:\n",
        "      alg_name = 'average'\n",
        "      Y_pred_train = pd_train_df['YIELD_PRED'].values\n",
        "      Y_pred_test = pd_test_df['YIELD_PRED'].values\n",
        "\n",
        "    train_scores = getPredictionScores(Y_train, Y_pred_train, self.metrics)\n",
        "    test_scores = getPredictionScores(Y_test, Y_pred_test, self.metrics)\n",
        "    self.updateAlgorithmsSummary(alg_summary, alg_name, train_scores, test_scores)\n",
        "\n",
        "  def getYieldTrendML(self, X_train, Y_train, X_test):\n",
        "    \"\"\"\n",
        "    Predict yield trend using a linear model.\n",
        "    No need to scale features. They are all yield values.\n",
        "    \"\"\"\n",
        "    est = Ridge(alpha=1, random_state=42, max_iter=1000,\n",
        "                copy_X=True, fit_intercept=True)\n",
        "    est_param_grid = dict(alpha=[1e-3, 1e-2, 1e-1, 1, 10])\n",
        "\n",
        "    grid_search = GridSearchCV(estimator=est, param_grid=est_param_grid,\n",
        "                               scoring=self.cv_metric, cv=self.custom_cv)\n",
        "    X_train_copy = np.copy(X_train)\n",
        "    with parallel_backend('spark', n_jobs=-1):\n",
        "      grid_search.fit(X_train_copy, np.ravel(Y_train))\n",
        "\n",
        "    best_params = grid_search.best_params_\n",
        "    best_estimator = grid_search.best_estimator_\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      for param in est_param_grid:\n",
        "        print(param + '=', best_params[param])\n",
        "\n",
        "    Y_pred_train = np.reshape(best_estimator.predict(X_train), (X_train.shape[0], 1))\n",
        "    Y_pred_test = np.reshape(best_estimator.predict(X_test), (X_test.shape[0], 1))\n",
        "\n",
        "    return Y_pred_train, Y_pred_test\n",
        "\n",
        "  def estimateYieldTrendAndDetrend(self, X_train, Y_train, X_test, Y_test, features):\n",
        "    \"\"\"Estimate yield trend using machine learning and detrend\"\"\"\n",
        "    trend_window = self.trend_windows[0]\n",
        "    # NOTE assuming previous years' yield values are at the end\n",
        "    X_train_trend = X_train[:, -trend_window:]\n",
        "    X_test_trend = X_test[:, -trend_window:]\n",
        "\n",
        "    Y_train_trend, Y_test_trend = self.getYieldTrendML(X_train_trend, Y_train, X_test_trend)\n",
        "    # New features exclude previous years' yield and include YIELD_TREND\n",
        "    features_n = features[:-trend_window] + ['YIELD_TREND']\n",
        "    X_train_n = np.append(X_train[:, :-trend_window], Y_train_trend, axis=1)\n",
        "    X_test_n = np.append(X_test[:, :-trend_window], Y_test_trend, axis=1)\n",
        "    Y_train_res = np.reshape(Y_train, (X_train.shape[0], 1)) - Y_train_trend\n",
        "    Y_test_res = np.reshape(Y_test, (X_test.shape[0], 1)) - Y_test_trend\n",
        "\n",
        "    result =  {\n",
        "        'X_train' : X_train_n,\n",
        "        'Y_train' : Y_train_res,\n",
        "        'Y_train_trend' : Y_train_trend,\n",
        "        'X_test' : X_test_n,\n",
        "        'Y_test' : Y_test_res,\n",
        "        'Y_test_trend' : Y_test_trend,\n",
        "        'features' : features_n,\n",
        "    }\n",
        "\n",
        "    return result\n",
        "\n",
        "  def yieldPredictionsFromResiduals(self, pd_train_df, Y_train, pd_test_df, Y_test):\n",
        "    \"\"\"Predictions are residuals. Add trend back to get yield predictions.\"\"\"\n",
        "    pd_train_df['YIELD_RES'] = pd_train_df['YIELD']\n",
        "    pd_train_df['YIELD'] = Y_train\n",
        "    pd_test_df['YIELD_RES'] = pd_test_df['YIELD']\n",
        "    pd_test_df['YIELD'] = Y_test\n",
        "    \n",
        "    for alg in self.estimators:\n",
        "      pd_train_df['YIELD_RES_PRED_' + alg] = pd_train_df['YIELD_PRED_' + alg]\n",
        "      pd_train_df['YIELD_PRED_' + alg] = pd_train_df['YIELD_RES_PRED_' + alg] + pd_train_df['YIELD_TREND']\n",
        "      pd_test_df['YIELD_RES_PRED_' + alg] = pd_test_df['YIELD_PRED_' + alg]\n",
        "      pd_test_df['YIELD_PRED_' + alg] = pd_test_df['YIELD_RES_PRED_' + alg] + pd_test_df['YIELD_TREND']\n",
        "\n",
        "    sel_cols = ['IDREGION', 'FYEAR', 'YIELD_TREND', 'YIELD_RES']\n",
        "    for alg in self.estimators:\n",
        "      sel_cols += ['YIELD_RES_PRED_' + alg, 'YIELD_PRED_' + alg]\n",
        "\n",
        "    sel_cols.append('YIELD')\n",
        "    result = {\n",
        "        'train' : pd_train_df[sel_cols],\n",
        "        'test' : pd_test_df[sel_cols],\n",
        "    }\n",
        "\n",
        "    return result\n",
        "\n",
        "  def trainAndTest(self, X_train, Y_train, X_test,\n",
        "                   est, est_name, est_param_grid):\n",
        "    \"\"\"\n",
        "    Use k-fold validation to tune hyperparameters and evaluate performance.\n",
        "    \"\"\"\n",
        "    if (self.verbose > 1):\n",
        "      print('\\nEstimator', est_name)\n",
        "      print('---------------------------')\n",
        "      print(est)\n",
        "\n",
        "    pipeline = Pipeline([(\"scaler\", self.scaler), (\"estimator\", est)])\n",
        "    grid_search = GridSearchCV(estimator=pipeline, param_grid=est_param_grid,\n",
        "                               scoring=self.cv_metric, cv=self.custom_cv)\n",
        "    X_train_copy = np.copy(X_train)\n",
        "    with parallel_backend('spark', n_jobs=-1):\n",
        "      grid_search.fit(X_train_copy, np.ravel(Y_train))\n",
        "\n",
        "    best_params = grid_search.best_params_\n",
        "    best_estimator = grid_search.best_estimator_\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      for param in est_param_grid:\n",
        "        print(param + '=', best_params[param])\n",
        "\n",
        "    Y_pred_train = np.reshape(best_estimator.predict(X_train), (X_train.shape[0], 1))\n",
        "    Y_pred_test = np.reshape(best_estimator.predict(X_test), (X_test.shape[0], 1))\n",
        "\n",
        "    return Y_pred_train, Y_pred_test\n",
        "\n",
        "  def combineAlgorithmPredictions(self, pd_ml_predictions, pd_alg_predictions, alg):\n",
        "    \"\"\"Combine predictions of ML algorithms.\"\"\"\n",
        "    join_cols = ['IDREGION', 'FYEAR']\n",
        "\n",
        "    if (pd_ml_predictions is None):\n",
        "      pd_ml_predictions = pd_alg_predictions\n",
        "      pd_ml_predictions = pd_ml_predictions.rename(columns={'YIELD_PRED': 'YIELD_PRED_' + alg })\n",
        "    else:\n",
        "      pd_alg_predictions = pd_alg_predictions[join_cols + ['YIELD_PRED']]\n",
        "      pd_ml_predictions = pd_ml_predictions.merge(pd_alg_predictions, on=join_cols)\n",
        "      pd_ml_predictions = pd_ml_predictions.rename(columns={'YIELD_PRED': 'YIELD_PRED_' + alg })\n",
        "      # Put YIELD at the end\n",
        "      all_cols = list(pd_ml_predictions.columns)\n",
        "      col_order = all_cols[:-2] + ['YIELD_PRED_' + alg, 'YIELD']\n",
        "      pd_ml_predictions = pd_ml_predictions[col_order]\n",
        "\n",
        "    return pd_ml_predictions\n",
        "\n",
        "  def getMLPredictions(self, X_train, Y_train_full, X_test, Y_test_full,\n",
        "                       cyp_ftsel, ft_selectors, features, log_fh):\n",
        "    \"\"\"Train and evaluate crop yield prediction algorithms\"\"\"\n",
        "    Y_train = Y_train_full[:, -1]\n",
        "    Y_test = Y_test_full[:, -1]\n",
        "    pd_test_predictions = None\n",
        "    pd_train_predictions = None\n",
        "\n",
        "    # feature selection frequency\n",
        "    # NOTE must be in sync with crop calendar periods\n",
        "    feature_selection_counts = {\n",
        "        'static' : {},\n",
        "        'p0' : {},\n",
        "        'p1' : {},\n",
        "        'p2' : {},\n",
        "        'p3' : {},\n",
        "        'p4' : {},\n",
        "        'p5' : {},\n",
        "    }\n",
        "\n",
        "    for est_name in self.estimators:\n",
        "      # feature selection\n",
        "      est = self.estimators[est_name]['estimator']\n",
        "      param_grid = self.estimators[est_name]['fs_param_grid']\n",
        "      selected_indices = cyp_ftsel.selectOptimalFeatures(ft_selectors,\n",
        "                                                         est, est_name, param_grid,\n",
        "                                                         log_fh)\n",
        "      sel_fts_info = '\\nSelected Features:'\n",
        "      sel_fts_info += '\\n-------------------'\n",
        "      log_fh.write(sel_fts_info)\n",
        "      print(sel_fts_info)\n",
        "      printFeatures(features, selected_indices, log_fh)\n",
        "\n",
        "      # update feature selection counts\n",
        "      for idx in selected_indices:\n",
        "        ft_count = 0\n",
        "        ft = features[idx]\n",
        "        ft_period = None\n",
        "        for p in feature_selection_counts:\n",
        "          if p in ft:\n",
        "            ft_period = p\n",
        "\n",
        "        if (ft_period is None):\n",
        "          ft_period = 'static'\n",
        "\n",
        "        if (ft in feature_selection_counts[ft_period]):\n",
        "          ft_count = feature_selection_counts[ft_period][ft]\n",
        "\n",
        "        feature_selection_counts[ft_period][ft] = ft_count + 1\n",
        "\n",
        "      X_train_sel = X_train[:, selected_indices]\n",
        "      X_test_sel = X_test[:, selected_indices]\n",
        "\n",
        "      # Training and testing\n",
        "      param_grid = self.estimators[est_name]['param_grid']\n",
        "      # yield/yield residual predictions\n",
        "      Y_pred_train, Y_pred_test = self.trainAndTest(X_train_sel, Y_train, X_test_sel,\n",
        "                                                    est, est_name, param_grid)\n",
        "      data_cols = ['IDREGION', 'FYEAR']\n",
        "      if (self.use_yield_trend):\n",
        "        data_cols.append('YIELD_TREND')\n",
        "        Y_train_full_n = np.insert(Y_train_full, 3, Y_pred_train[:, 0], axis=1)\n",
        "        Y_test_full_n = np.insert(Y_test_full, 3, Y_pred_test[:, 0], axis=1)\n",
        "      else:\n",
        "        Y_train_full_n = np.insert(Y_train_full, 2, Y_pred_train[:, 0], axis=1)\n",
        "        Y_test_full_n = np.insert(Y_test_full, 2, Y_pred_test[:, 0], axis=1)\n",
        "\n",
        "      data_cols += ['YIELD_PRED', 'YIELD']\n",
        "      pd_train_df, pd_test_df = self.createPredictionDataFrames(Y_train_full_n, Y_test_full_n, data_cols)\n",
        "      pd_train_predictions = self.combineAlgorithmPredictions(pd_train_predictions, pd_train_df, est_name)\n",
        "      pd_test_predictions = self.combineAlgorithmPredictions(pd_test_predictions, pd_test_df, est_name)\n",
        "\n",
        "    ft_counts_info = '\\nFeature Selection Frequencies'\n",
        "    ft_counts_info += '\\n-------------------------------'\n",
        "    for ft_period in feature_selection_counts:\n",
        "      ft_count_str = ft_period + ': '\n",
        "      for ft in sorted(feature_selection_counts[ft_period],\n",
        "                       key=feature_selection_counts[ft_period].get, reverse=True):\n",
        "        ft_count_str += ft + '(' + str(feature_selection_counts[ft_period][ft]) + '), '\n",
        "\n",
        "      if (len(feature_selection_counts[ft_period]) > 0):\n",
        "        # drop ', ' from the end\n",
        "        ft_count_str = ft_count_str[:-2]\n",
        "\n",
        "      ft_counts_info += '\\n' + ft_count_str\n",
        "\n",
        "    ft_counts_info += '\\n'\n",
        "    log_fh.write(ft_counts_info)\n",
        "    if (self.verbose > 1):\n",
        "      print(ft_counts_info)\n",
        "\n",
        "    result = {\n",
        "        'train' : pd_train_predictions,\n",
        "        'test' : pd_test_predictions,\n",
        "    }\n",
        "\n",
        "    return result\n",
        "\n",
        "  def evaluateMLPredictions(self, pd_train_predictions, pd_test_predictions, alg_summary):\n",
        "    \"\"\"Evaluate predictions of ML algorithms and add entries to alg_summary.\"\"\"\n",
        "    Y_train = pd_train_predictions['YIELD'].values\n",
        "    Y_test = pd_test_predictions['YIELD'].values\n",
        "\n",
        "    for alg in self.estimators:\n",
        "      alg_col = 'YIELD_PRED_' + alg\n",
        "      Y_pred_train = pd_train_predictions[alg_col].values\n",
        "      Y_pred_test = pd_test_predictions[alg_col].values\n",
        "      train_scores = getPredictionScores(Y_train, Y_pred_train, self.metrics)\n",
        "      test_scores = getPredictionScores(Y_test, Y_pred_test, self.metrics)\n",
        "\n",
        "      self.updateAlgorithmsSummary(alg_summary, alg, train_scores, test_scores)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UpxrWSjsxMdx"
      },
      "source": [
        "#### Run Machine Learning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6HbC8rSxjEpT"
      },
      "outputs": [],
      "source": [
        "#%%writefile run_machine_learning.py\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from joblibspark import register_spark\n",
        "\n",
        "def warn(*args, **kwargs):\n",
        "    pass\n",
        "\n",
        "import warnings\n",
        "warnings.warn = warn\n",
        "\n",
        "def getValidationSplits(cyp_config, pd_train_df, pd_test_df, log_fh):\n",
        "  \"\"\"Split features and label into training and test sets\"\"\"\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  regions = [reg for reg in pd_train_df['IDREGION'].unique()]\n",
        "  num_regions = len(regions)\n",
        "\n",
        "  original_headers = list(pd_train_df.columns.values)\n",
        "  features = []\n",
        "  labels = []\n",
        "  if (use_yield_trend):\n",
        "    features = original_headers[2:-2]\n",
        "    labels = original_headers[:2] + original_headers[-2:]\n",
        "  else:\n",
        "    features = original_headers[2:-1]\n",
        "    labels = original_headers[:2] + original_headers[-1:]\n",
        "\n",
        "  X_train = pd_train_df[features].values\n",
        "  Y_train = pd_train_df[labels].values\n",
        "\n",
        "  train_info = '\\nTraining Data Size: ' + str(len(pd_train_df.index)) + ' rows'\n",
        "  train_info += '\\nX cols: ' + str(X_train.shape[1]) + ', Y cols: ' + str(Y_train.shape[1])\n",
        "  train_info += '\\n' + pd_train_df.head(5).to_string(index=False)\n",
        "  log_fh.write(train_info + '\\n')\n",
        "  if (debug_level > 1):\n",
        "    print(train_info)\n",
        "\n",
        "  X_test = pd_test_df[features].values\n",
        "  Y_test = pd_test_df[labels].values\n",
        "\n",
        "  test_info = '\\nTest Data Size: ' + str(len(pd_test_df.index)) + ' rows'\n",
        "  test_info += '\\nX cols: ' + str(X_test.shape[1]) + ', Y cols: ' + str(Y_test.shape[1])\n",
        "  test_info += '\\n' + pd_test_df.head(5).to_string(index=False)\n",
        "  log_fh.write(test_info + '\\n')\n",
        "  if (debug_level > 1):\n",
        "    print(test_info)\n",
        "\n",
        "  # print feature names\n",
        "  num_features = len(features)\n",
        "  indices = [idx for idx in range(num_features)]\n",
        "  feature_info = '\\nAll features'\n",
        "  feature_info += '\\n-------------'\n",
        "  log_fh.write(feature_info)\n",
        "  print(feature_info)\n",
        "  printFeatures(features, indices, log_fh)\n",
        "\n",
        "  # num_folds for k-fold cv\n",
        "  num_folds = 5\n",
        "  custom_cv = num_folds\n",
        "  if (use_yield_trend):\n",
        "    cyp_cv_splitter = CYPTrainTestSplitter(cyp_config)\n",
        "    custom_cv = cyp_cv_splitter.customKFoldValidationSplit(Y_train, num_folds, log_fh)\n",
        "\n",
        "  # L1 penalty = error (y_pred - y_obs)^2 + alpha * sum (|w_i|) = sparsity regularization\n",
        "  # L2 penalty = error (y_pred - y_obs)^2 + alpha * sqrt ( sum (w_i^2) ) = weight decay regularization\n",
        "\n",
        "  result = {\n",
        "      'X_train' : X_train,\n",
        "      'Y_train_full' : Y_train,\n",
        "      'X_test' : X_test,\n",
        "      'Y_test_full' : Y_test,\n",
        "      'custom_cv' : custom_cv,\n",
        "      'features' : features,\n",
        "  }\n",
        "\n",
        "  return result\n",
        "\n",
        "def getMachineLearningPredictions(cyp_config, pd_train_df, pd_test_df, log_fh):\n",
        "  \"\"\"Train and evaluate algorithms\"\"\"\n",
        "  metrics = cyp_config.getEvaluationMetrics()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  predict_residuals = cyp_config.predictYieldResiduals()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  # register spark parallel backend\n",
        "  register_spark()\n",
        "\n",
        "  eval_info = '\\nTraining and Evaluation'\n",
        "  eval_info += '\\n-------------------------'\n",
        "  log_fh.write(eval_info)\n",
        "  if (debug_level > 1):\n",
        "    print(eval_info)\n",
        "\n",
        "  data_splits = getValidationSplits(cyp_config, pd_train_df, pd_test_df, log_fh)\n",
        "  X_train = data_splits['X_train']\n",
        "  Y_train_full = np.copy(data_splits['Y_train_full'])\n",
        "  X_test = data_splits['X_test']\n",
        "  Y_test_full = np.copy(data_splits['Y_test_full'])\n",
        "  features = data_splits['features']\n",
        "  custom_cv = data_splits['custom_cv']\n",
        "\n",
        "  alg_summary = {}\n",
        "  cyp_algeval = CYPAlgorithmEvaluator(cyp_config, custom_cv)\n",
        "  null_preds = cyp_algeval.getNullMethodPredictions(Y_train_full, Y_test_full, log_fh)\n",
        "\n",
        "  if (use_yield_trend and predict_residuals):\n",
        "    result = cyp_algeval.estimateYieldTrendAndDetrend(X_train, Y_train_full[:, -1],\n",
        "                                                      X_test, Y_test_full[:, -1], features)\n",
        "    X_train = result['X_train']\n",
        "    Y_train_full[:, -1] = result['Y_train'][:, 0]\n",
        "    Y_train_full[:, -2] = result['Y_train_trend'][:, 0]\n",
        "    X_test = result['X_test']\n",
        "    Y_test_full[:, -1] = result['Y_test'][:,0]\n",
        "    Y_test_full[:, -2] = result['Y_test_trend'][:, 0]\n",
        "    features = result['features']\n",
        "\n",
        "  # feature selection methods\n",
        "  num_features = len(features)\n",
        "  ft_selectors = cyp_config.getFeatureSelectors(X_train, Y_train_full[:, -1],\n",
        "                                                num_features, custom_cv)\n",
        "  cyp_ftsel = CYPFeatureSelector(cyp_config, X_train, Y_train_full[:, -1], custom_cv, features)\n",
        "  ml_preds = cyp_algeval.getMLPredictions(X_train, Y_train_full, X_test, Y_test_full,\n",
        "                                          cyp_ftsel, ft_selectors, features, log_fh)\n",
        "  if (use_yield_trend and predict_residuals):\n",
        "    # NOTE Y_train_full, Y_test_full can get modified above\n",
        "    ml_preds = cyp_algeval.yieldPredictionsFromResiduals(ml_preds['train'],\n",
        "                                                         data_splits['Y_train_full'][:, -1],\n",
        "                                                         ml_preds['test'],\n",
        "                                                         data_splits['Y_test_full'][:, -1])\n",
        "\n",
        "  cyp_algeval.evaluateNullMethodPredictions(null_preds['train'], null_preds['test'], alg_summary)\n",
        "  cyp_algeval.evaluateMLPredictions(ml_preds['train'], ml_preds['test'], alg_summary)\n",
        "  cyp_algeval.printPredictionDataFrames(ml_preds['train'], ml_preds['test'], log_fh)\n",
        "\n",
        "  alg_df_columns = ['algorithm']\n",
        "  for met in metrics:\n",
        "    alg_df_columns += ['train_' + met, 'test_' + met]\n",
        "\n",
        "  alg_df = pd.DataFrame.from_dict(alg_summary, orient='index', columns=alg_df_columns)\n",
        "\n",
        "  eval_summary_info = '\\nAlgorithm Evaluation Summary'\n",
        "  eval_summary_info += '\\n-----------------------------'\n",
        "  eval_summary_info += '\\n' + alg_df.to_string(index=False) + '\\n'\n",
        "  log_fh.write(eval_summary_info)\n",
        "  print(eval_summary_info)\n",
        "\n",
        "  return ml_preds['test']\n",
        "\n",
        "def saveMLPredictions(cyp_config, sqlCtx, pd_ml_predictions):\n",
        "  \"\"\"Save ML predictions to a CSV file\"\"\"\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  crop = cyp_config.getCropName()\n",
        "  country = cyp_config.getCountryCode()\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "  ml_algs = cyp_config.getEstimators()\n",
        "\n",
        "  output_path = cyp_config.getOutputPath()\n",
        "  output_file = getPredictionFilename(crop, country, nuts_level, use_yield_trend,\n",
        "                                      early_season_prediction, early_season_end)\n",
        "\n",
        "  save_pred_path = output_path + '/' + output_file\n",
        "  if (debug_level > 1):\n",
        "    print('\\nSaving predictions to', save_pred_path + '.csv')\n",
        "    print(pd_ml_predictions.head(5))\n",
        "\n",
        "  pd_ml_predictions.to_csv(save_pred_path + '.csv', index=False, header=True)\n",
        "\n",
        "  # NOTE: In some environments, Spark can write, but pandas cannot.\n",
        "  # In such cases, use the following code.\n",
        "  # spark_predictions_df = sqlCtx.createDataFrame(pd_ml_predictions)\n",
        "  # spark_predictions_df.coalesce(1)\\\n",
        "  #                     .write.option('header','true')\\\n",
        "  #                     .mode(\"overwrite\").csv(save_pred_path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-jDs9IO_xfPu"
      },
      "source": [
        "### Load Saved Predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KOFLzsa8xjT-"
      },
      "outputs": [],
      "source": [
        "#%%writefile load_saved_predictions.py\n",
        "import pandas as pd\n",
        "\n",
        "def loadSavedPredictions(cyp_config, spark):\n",
        "  \"\"\"Load machine learning predictions from saved CSV file\"\"\"\n",
        "  crop = cyp_config.getCropName()\n",
        "  country = cyp_config.getCountryCode()\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  pred_file_path = cyp_config.getOutputPath()\n",
        "  pred_file = getPredictionFilename(crop, country, nuts_level, use_yield_trend,\n",
        "                                    early_season_prediction, early_season_end)\n",
        "  \n",
        "  pred_file += '.csv'\n",
        "  pd_ml_predictions = pd.read_csv(pred_file_path + '/' + pred_file, header=0)\n",
        "\n",
        "  # NOTE: In some environments, Spark can read, but pandas cannot.\n",
        "  # In such cases, use the following code.\n",
        "  # all_pred_df = spark.read.csv(pred_file_path + '/' + pred_file, header=True, inferSchema=True)\n",
        "  # pd_ml_predictions = all_pred_df.toPandas()\n",
        "\n",
        "  if (debug_level > 1):\n",
        "    print(pd_ml_predictions.head(5))\n",
        "\n",
        "  return pd_ml_predictions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7uvKP54QxrUK"
      },
      "source": [
        "### Compare Predictions with MCYFS"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hj_ZCXfaxuxt"
      },
      "outputs": [],
      "source": [
        "#%%writefile compare_with_mcyfs.py\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "def saveNUTS0Predictions(cyp_config, sqlCtx, nuts0_ml_predictions):\n",
        "  \"\"\"Save predictions aggregated to NUTS0\"\"\"\n",
        "  crop = cyp_config.getCropName()\n",
        "  country = cyp_config.getCountryCode()\n",
        "  nuts_level = 'NUTS0'\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  output_path = cyp_config.getOutputPath()\n",
        "  output_file = getPredictionFilename(crop, country, nuts_level, use_yield_trend,\n",
        "                                      early_season_prediction, early_season_end)\n",
        "\n",
        "  save_pred_path = output_path + '/' + output_file\n",
        "  if (debug_level > 1):\n",
        "    print('\\nNUTS0 Predictions of ML algorithms')\n",
        "    print(nuts0_ml_predictions.head(5))\n",
        "    print('\\nSaving predictions to', save_pred_path + '.csv')\n",
        "\n",
        "  nuts0_ml_predictions.to_csv(save_pred_path + '.csv', index=False, header=True)\n",
        "\n",
        "  # NOTE: In some environments, Spark can write, but pandas cannot.\n",
        "  # In such cases, use the following code.\n",
        "  # spark_predictions_df = sqlCtx.createDataFrame(nuts0_ml_predictions)\n",
        "  # spark_predictions_df.coalesce(1)\\\n",
        "  #                     .write.option('header','true')\\\n",
        "  #                     .mode(\"overwrite\").csv(save_pred_path)\n",
        "\n",
        "def getDataForMCYFSComparison(spark, cyp_config, test_years):\n",
        "  \"\"\"Load and preprocess data for MCYFS comparison\"\"\"\n",
        "  data_path = cyp_config.getDataPath()\n",
        "  crop_id = cyp_config.getCropID()\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  season_crosses_calyear = cyp_config.seasonCrossesCalendarYear()\n",
        "  early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  area_nuts = ['NUTS' + str(i) for i in range(int(nuts_level[-1]), 0, -1)]\n",
        "  data_sources = {\n",
        "      'WOFOST' : nuts_level,\n",
        "      'AREA_FRACTIONS' : area_nuts,\n",
        "      'YIELD' : 'NUTS0',\n",
        "      'YIELD_PRED_MCYFS' : 'NUTS0',\n",
        "  }\n",
        "\n",
        "  if (run_tests):\n",
        "    test_util = TestUtil(spark)\n",
        "    test_util.runAllTests()\n",
        "\n",
        "  print('##############')\n",
        "  print('# Load Data  #')\n",
        "  print('##############')\n",
        "\n",
        "  if (run_tests):\n",
        "    test_loader = TestDataLoader(spark)\n",
        "    test_loader.runAllTests()\n",
        "\n",
        "  cyp_config.setDataSources(data_sources)\n",
        "  cyp_loader = CYPDataLoader(spark, cyp_config)\n",
        "  data_dfs = cyp_loader.loadAllData()\n",
        "\n",
        "  wofost_df = data_dfs['WOFOST']\n",
        "  area_dfs = data_dfs['AREA_FRACTIONS']\n",
        "  nuts0_yield_df = data_dfs['YIELD']\n",
        "  mcyfs_yield_df = data_dfs['YIELD_PRED_MCYFS']\n",
        "\n",
        "  print('####################')\n",
        "  print('# Preprocess Data  #')\n",
        "  print('####################')\n",
        "\n",
        "  if (run_tests):\n",
        "    test_preprocessor = TestDataPreprocessor(spark)\n",
        "    test_preprocessor.runAllTests()\n",
        "\n",
        "  cyp_preprocessor = CYPDataPreprocessor(spark, cyp_config)\n",
        "  wofost_df = wofost_df.filter(wofost_df['CROP_ID'] == crop_id).drop('CROP_ID')\n",
        "  crop_season = cyp_preprocessor.getCropSeasonInformation(wofost_df, season_crosses_calyear)\n",
        "  wofost_df = cyp_preprocessor.preprocessWofost(wofost_df, crop_season, season_crosses_calyear)\n",
        "\n",
        "  for i in range(len(area_dfs)):\n",
        "    af_df = area_dfs[i]\n",
        "    af_df = cyp_preprocessor.preprocessAreaFractions(af_df, crop_id)\n",
        "    af_df = af_df.filter(af_df['FYEAR'].isin(test_years))\n",
        "    area_dfs[i] = af_df\n",
        "\n",
        "  if (debug_level > 1):\n",
        "    print('NUTS0 Yield before preprocessing')\n",
        "    nuts0_yield_df.show(10)\n",
        "\n",
        "  nuts0_yield_df = cyp_preprocessor.preprocessYield(nuts0_yield_df, crop_id)\n",
        "  nuts0_yield_df = nuts0_yield_df.filter(nuts0_yield_df['FYEAR'].isin(test_years))\n",
        "  if (debug_level > 1):\n",
        "    print('NUTS0 Yield after preprocessing')\n",
        "    nuts0_yield_df.show(10)\n",
        "\n",
        "  if (debug_level > 1):\n",
        "    print('MCYFS yield predictions before preprocessing')\n",
        "    mcyfs_yield_df.show(10)\n",
        "\n",
        "  mcyfs_yield_df = cyp_preprocessor.preprocessYieldMCYFS(mcyfs_yield_df, crop_id)\n",
        "  mcyfs_yield_df = mcyfs_yield_df.filter(mcyfs_yield_df['FYEAR'].isin(test_years))\n",
        "  if (debug_level > 1):\n",
        "    print('MCYFS yield predictions after preprocessing')\n",
        "    mcyfs_yield_df.show(10)\n",
        "\n",
        "  # Check we have yield data for crop\n",
        "  assert (nuts0_yield_df is not None)\n",
        "  assert (mcyfs_yield_df is not None)\n",
        "\n",
        "  if (run_tests):\n",
        "    test_summarizer = TestDataSummarizer(spark)\n",
        "    test_summarizer.runAllTests()\n",
        "\n",
        "  cyp_summarizer = CYPDataSummarizer(cyp_config)\n",
        "  dvs_summary = cyp_summarizer.wofostDVSSummary(wofost_df, early_season_end)\n",
        "  dvs_summary = dvs_summary.filter(dvs_summary['CAMPAIGN_YEAR'].isin(test_years))\n",
        "\n",
        "  data_dfs = {\n",
        "      'WOFOST_DVS' : dvs_summary,\n",
        "      'AREA_FRACTIONS' : area_dfs,\n",
        "      'YIELD_NUTS0' : nuts0_yield_df,\n",
        "      'YIELD_PRED_MCYFS' : mcyfs_yield_df\n",
        "  }\n",
        "\n",
        "  return data_dfs\n",
        "\n",
        "def fillMissingDataWithAverage(pd_pred_df, print_debug):\n",
        "  \"\"\"Fill missing data with regional average or zero\"\"\"\n",
        "  regions = pd_pred_df['IDREGION'].unique()\n",
        "\n",
        "  for reg_id in regions:\n",
        "    reg_filter = (pd_pred_df['IDREGION'] == reg_id)\n",
        "    pd_reg_pred_df = pd_pred_df[reg_filter]\n",
        "\n",
        "    if (len(pd_reg_pred_df[pd_reg_pred_df['YIELD_PRED'].notnull()].index) == 0):\n",
        "      if (print_debug):\n",
        "        print('No data for', reg_id)\n",
        "\n",
        "      pd_pred_df.loc[reg_filter, 'FRACTION'] = 0.0\n",
        "      pd_pred_df.loc[reg_filter, 'YIELD_PRED'] = 0.0\n",
        "    else:\n",
        "      reg_avg_yield_pred = pd_pred_df.loc[reg_filter, 'YIELD_PRED'].mean()\n",
        "      pd_pred_df.loc[reg_filter, 'YIELD_PRED'] = pd_pred_df.loc[reg_filter, 'YIELD_PRED']\\\n",
        "                                                           .fillna(reg_avg_yield_pred)  \n",
        "\n",
        "  return pd_pred_df\n",
        "\n",
        "def recalculateAreaFractions(pd_pred_df, print_debug):\n",
        "  \"\"\"Recalculate area fractions by excluding regions with missing data\"\"\"\n",
        "  join_cols = ['IDREG_PARENT', 'FYEAR']\n",
        "  pd_af_sum = pd_pred_df.groupby(join_cols).agg(FRACTION_SUM=('FRACTION', 'sum')).reset_index()\n",
        "  pd_pred_df = pd_pred_df.merge(pd_af_sum, on=join_cols, how='left')\n",
        "  pd_pred_df['FRACTION'] = pd_pred_df['FRACTION'] / pd_pred_df['FRACTION_SUM']\n",
        "  pd_pred_df = pd_pred_df.drop(columns=['FRACTION_SUM'])\n",
        "\n",
        "  return pd_pred_df\n",
        "\n",
        "def aggregatePredictionsToNUTS0(cyp_config, pd_ml_predictions,\n",
        "                                area_dfs, test_years, join_cols):\n",
        "  \"\"\"Aggregate regional predictions to national level\"\"\"\n",
        "  pd_area_dfs = []\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  crop_id = cyp_config.getCropID()\n",
        "  alg_names = list(cyp_config.getEstimators().keys())\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "\n",
        "  for af_df in area_dfs:\n",
        "    pd_af_df = af_df.toPandas()\n",
        "    pd_af_df = pd_af_df[pd_af_df['FYEAR'].isin(test_years)]\n",
        "    pd_area_dfs.append(pd_af_df)\n",
        "\n",
        "  nuts0_pred_df = None\n",
        "  for alg in alg_names:\n",
        "    sel_cols = ['IDREGION', 'FYEAR', 'YIELD_PRED_' + alg]\n",
        "    pd_alg_pred_df = pd_ml_predictions[sel_cols]\n",
        "    pd_alg_pred_df = pd_alg_pred_df.rename(columns={'YIELD_PRED_' + alg : 'YIELD_PRED'})\n",
        "\n",
        "    for idx in range(len(pd_area_dfs)):\n",
        "      pd_af_df = pd_area_dfs[idx]\n",
        "      # merge with area fractions to get all regions and years\n",
        "      pd_alg_pred_df = pd_af_df.merge(pd_alg_pred_df, on=join_cols)\n",
        "      print_debug = (debug_level > 2) and (alg == alg_names[0])\n",
        "      pd_alg_pred_df = fillMissingDataWithAverage(pd_alg_pred_df, print_debug)\n",
        "      pd_alg_pred_df['IDREG_PARENT'] = pd_alg_pred_df['IDREGION'].str[:-1]\n",
        "      pd_alg_pred_df = recalculateAreaFractions(pd_alg_pred_df, print_debug)\n",
        "      if (print_debug):\n",
        "        print('\\nAggregation to NUTS' + str(len(pd_area_dfs) - (idx + 1)))\n",
        "        print(pd_alg_pred_df[pd_alg_pred_df['FYEAR'] == test_years[0]].head(10))\n",
        "\n",
        "      pd_alg_pred_df['YPRED_WEIGHTED'] = pd_alg_pred_df['YIELD_PRED'] * pd_alg_pred_df['FRACTION']\n",
        "      pd_alg_pred_df = pd_alg_pred_df.groupby(by=['IDREG_PARENT', 'FYEAR'])\\\n",
        "                                     .agg(YPRED_WEIGHTED=('YPRED_WEIGHTED', 'sum')).reset_index()\n",
        "      pd_alg_pred_df = pd_alg_pred_df.rename(columns={'IDREG_PARENT': 'IDREGION',\n",
        "                                                      'YPRED_WEIGHTED': 'YIELD_PRED' })\n",
        "\n",
        "    pd_alg_pred_df = pd_alg_pred_df.rename(columns={ 'YIELD_PRED': 'YIELD_PRED_' + alg })\n",
        "    if (nuts0_pred_df is None):\n",
        "      nuts0_pred_df = pd_alg_pred_df\n",
        "    else:\n",
        "      nuts0_pred_df = nuts0_pred_df.merge(pd_alg_pred_df, on=join_cols)\n",
        "\n",
        "  return nuts0_pred_df\n",
        "\n",
        "def getMCYFSPrediction(pd_mcyfs_pred_df, pred_year, pred_dekad, print_debug):\n",
        "  \"\"\"Get MCYFS prediction for given year with prediction date close to pred_dekad\"\"\"\n",
        "  pd_pred_year = pd_mcyfs_pred_df[pd_mcyfs_pred_df['FYEAR'] == pred_year]\n",
        "  mcyfs_pred_dekads = pd_pred_year['PRED_DEKAD'].unique()\n",
        "  if (len(mcyfs_pred_dekads) == 0):\n",
        "    return 0.0\n",
        "\n",
        "  mcyfs_pred_dekads = sorted(mcyfs_pred_dekads)\n",
        "  mcyfs_pred_dekad = mcyfs_pred_dekads[-1]\n",
        "  if (pred_dekad < mcyfs_pred_dekad):\n",
        "    for dek in mcyfs_pred_dekads:\n",
        "      if dek >= pred_dekad:\n",
        "        mcyfs_pred_dekad = dek\n",
        "        break\n",
        "\n",
        "  pd_pred_dek = pd_pred_year[pd_pred_year['PRED_DEKAD'] == mcyfs_pred_dekad]\n",
        "  yield_pred_list = pd_pred_dek['YIELD_PRED'].values\n",
        "\n",
        "  if (print_debug):\n",
        "    print('\\nAll MCYFS dekads for', pred_year, ':', mcyfs_pred_dekads)\n",
        "    print('MCYFS prediction dekad', mcyfs_pred_dekad)\n",
        "    print('ML Baseline prediction dekad', pred_dekad)\n",
        "    print('MCYFS prediction:', yield_pred_list[0], '\\n')\n",
        "\n",
        "  return yield_pred_list[0]\n",
        "\n",
        "def getNUTS0Yield(pd_nuts0_yield_df, pred_year, print_debug):\n",
        "  \"\"\"Get the true (reported) Eurostat yield value\"\"\"\n",
        "  nuts0_yield_year = pd_nuts0_yield_df[pd_nuts0_yield_df['FYEAR'] == pred_year]\n",
        "  pred_year_yield = nuts0_yield_year['YIELD'].values\n",
        "  if (len(pred_year_yield) == 0):\n",
        "    return 0.0\n",
        "\n",
        "  if (print_debug):\n",
        "    print(pred_year, 'Eurostat yield', pred_year_yield[0])\n",
        "\n",
        "  return pred_year_yield[0]\n",
        "\n",
        "def comparePredictionsWithMCYFS(sqlCtx, cyp_config, pd_ml_predictions, log_fh):\n",
        "  \"\"\"Compare ML Baseline predictions with MCYFS predictions\"\"\"\n",
        "  # We need AREA_FRACTIONS, MCYFS yield predictions and NUTS0 Eurostat YIELD\n",
        "  # for comparison with MCYFS\n",
        "  country_code = cyp_config.getCountryCode()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  alg_names = list(cyp_config.getEstimators().keys())\n",
        "  test_years = list(pd_ml_predictions['FYEAR'].unique())\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "\n",
        "  spark = sqlCtx.sparkSession\n",
        "  data_dfs = getDataForMCYFSComparison(spark, cyp_config, test_years)\n",
        "  pd_dvs_summary = data_dfs['WOFOST_DVS'].toPandas()\n",
        "  pd_nuts0_yield_df = data_dfs['YIELD_NUTS0'].toPandas()\n",
        "  pd_mcyfs_pred_df = data_dfs['YIELD_PRED_MCYFS'].toPandas()\n",
        "  area_dfs = data_dfs['AREA_FRACTIONS']\n",
        "  join_cols = ['IDREGION', 'FYEAR']\n",
        "  test_years = pd_ml_predictions['FYEAR'].unique()\n",
        "  metrics = cyp_config.getEvaluationMetrics()\n",
        "  nuts0_pred_df = aggregatePredictionsToNUTS0(cyp_config, pd_ml_predictions,\n",
        "                                              area_dfs, test_years, join_cols)\n",
        "\n",
        "  crop_season_cols = ['IDREGION', 'CAMPAIGN_YEAR', 'CALENDAR_END_SEASON', 'CALENDAR_EARLY_SEASON']\n",
        "  pd_dvs_summary = pd_dvs_summary[crop_season_cols].rename(columns={ 'CAMPAIGN_YEAR' : 'FYEAR' })\n",
        "  pd_dvs_summary = pd_dvs_summary.groupby('FYEAR').agg(END_SEASON=('CALENDAR_END_SEASON', 'mean'),\n",
        "                                                       EARLY_SEASON=('CALENDAR_EARLY_SEASON', 'mean'))\\\n",
        "                                                       .round(0).reset_index()\n",
        "  if (debug_level > 1):\n",
        "    print(pd_dvs_summary.head(5).to_string(index=False))\n",
        "\n",
        "  alg_summary = {}\n",
        "  Y_pred_mcyfs = []\n",
        "  Y_true = []\n",
        "  nuts0_pred_df['YIELD_PRED_MCYFS'] = 0.0\n",
        "  nuts0_pred_df['YIELD'] = 0.0\n",
        "  nuts0_pred_df = nuts0_pred_df.sort_values(by=join_cols)\n",
        "  ml_pred_years = nuts0_pred_df['FYEAR'].unique()\n",
        "  mcyfs_pred_years = []\n",
        "  print_debug = (debug_level > 2)\n",
        "  if (print_debug):\n",
        "    print('\\nPredictions and true values for', country_code)\n",
        "\n",
        "  for yr in ml_pred_years:\n",
        "    pred_dekad = pd_dvs_summary[pd_dvs_summary['FYEAR'] == yr]['END_SEASON'].values[0]\n",
        "    if (early_season_prediction):\n",
        "      pred_dekad = pd_dvs_summary[pd_dvs_summary['FYEAR'] == yr]['EARLY_SEASON'].values[0]\n",
        "\n",
        "    mcyfs_pred = getMCYFSPrediction(pd_mcyfs_pred_df, yr, pred_dekad, print_debug)\n",
        "    nuts0_yield = getNUTS0Yield(pd_nuts0_yield_df, yr, print_debug)\n",
        "    if ((mcyfs_pred > 0.0) and (nuts0_yield > 0.0)):\n",
        "      nuts0_pred_df.loc[nuts0_pred_df['FYEAR'] == yr, 'YIELD'] = nuts0_yield\n",
        "      nuts0_pred_df.loc[nuts0_pred_df['FYEAR'] == yr, 'YIELD_PRED_MCYFS'] = mcyfs_pred\n",
        "      mcyfs_pred_years.append(yr)\n",
        "\n",
        "  nuts0_pred_df = nuts0_pred_df[nuts0_pred_df['FYEAR'].isin(mcyfs_pred_years)]\n",
        "  Y_true = nuts0_pred_df['YIELD'].values\n",
        "\n",
        "  if (print_debug):\n",
        "    print(nuts0_pred_df.head(5))\n",
        "\n",
        "  if (len(mcyfs_pred_years) > 0):\n",
        "    for alg in alg_names:\n",
        "      Y_pred_alg = nuts0_pred_df['YIELD_PRED_' + alg].values\n",
        "      alg_nuts0_scores = getPredictionScores(Y_true, Y_pred_alg, metrics)\n",
        "\n",
        "      alg_row = [alg]\n",
        "      for met in alg_nuts0_scores:\n",
        "        alg_row.append(alg_nuts0_scores[met])\n",
        "\n",
        "      alg_index = len(alg_summary)\n",
        "      alg_summary['row' + str(alg_index)] = alg_row\n",
        "\n",
        "    Y_pred_mcyfs = nuts0_pred_df['YIELD_PRED_MCYFS'].values\n",
        "    mcyfs_nuts0_scores = getPredictionScores(Y_true, Y_pred_mcyfs, metrics)\n",
        "    alg_row = ['MCYFS_Predictions']\n",
        "    for met in mcyfs_nuts0_scores:\n",
        "      alg_row.append(mcyfs_nuts0_scores[met])\n",
        "\n",
        "    alg_index = len(alg_summary)\n",
        "    alg_summary['row' + str(alg_index)] = alg_row\n",
        "\n",
        "    alg_df_columns = ['algorithm']\n",
        "    for met in metrics:\n",
        "      alg_df_columns += ['test_' + met]\n",
        "\n",
        "    alg_df = pd.DataFrame.from_dict(alg_summary, orient='index',\n",
        "                                    columns=alg_df_columns)\n",
        "    eval_summary_info = '\\nAlgorithm Evaluation Summary (NUTS0) for ' + country_code\n",
        "    eval_summary_info += '\\n-------------------------------------------'\n",
        "    eval_summary_info += '\\n' + alg_df.to_string(index=False) + '\\n'\n",
        "    log_fh.write(eval_summary_info)\n",
        "    print(eval_summary_info)\n",
        "\n",
        "  save_predictions = cyp_config.savePredictions()\n",
        "  if (save_predictions):\n",
        "    saveNUTS0Predictions(cyp_config, sqlCtx, nuts0_pred_df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o7FqdmXOjYUm"
      },
      "source": [
        "## Tests"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kPwEyntUYHsS"
      },
      "source": [
        "### Test Utility Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bx6fS731YHsV"
      },
      "outputs": [],
      "source": [
        "#%%writefile test_util.py\n",
        "import numpy as np\n",
        "\n",
        "class TestUtil():\n",
        "  def __init__(self, spark):\n",
        "    self.good_date = spark.createDataFrame([(1, '19940102'),\n",
        "                                          (2, '15831224')],\n",
        "                                         ['ID', 'DATE'])\n",
        "    self.bad_date = spark.createDataFrame([(1, '14341224'),\n",
        "                                           (2, '12345678'),\n",
        "                                          (3, '123-12-24')],\n",
        "                                         ['ID', 'DATE'])\n",
        "\n",
        "  def testDateFormat(self):\n",
        "    print('\\n Test Date Format')\n",
        "    self.good_date = self.good_date.withColumn('FYEAR', getYear('DATE'))\n",
        "    self.good_date.show()\n",
        "    self.bad_date = self.bad_date.withColumn('FYEAR', getYear('DATE'))\n",
        "    self.bad_date.show()\n",
        "    assert (self.bad_date.filter(self.bad_date.FYEAR.isNull()).count() == 2)\n",
        "    self.bad_date = self.bad_date.withColumn('MONTH', getMonth('DATE'))\n",
        "    self.bad_date.show()\n",
        "    assert (self.bad_date.filter(self.bad_date.MONTH.isNull()).count() == 2)\n",
        "    self.bad_date = self.bad_date.withColumn('DAY', getDay('DATE'))\n",
        "    # check the day here for first date, it's incorrect\n",
        "    # seems to be a Spark issue\n",
        "    self.bad_date.show()\n",
        "    assert (self.bad_date.filter(self.bad_date.DAY.isNull()).count() == 2)\n",
        "    self.bad_date = self.bad_date.withColumn('DEKAD', getDekad('DATE'))\n",
        "    self.bad_date.show()\n",
        "    assert (self.bad_date.filter(self.bad_date.DEKAD.isNull()).count() == 2)\n",
        "\n",
        "  def testGetYear(self):\n",
        "    print('\\n Test getYear')\n",
        "    self.good_date = self.good_date.withColumn('FYEAR', getYear('DATE'))\n",
        "    self.good_date.show()\n",
        "    year1 = self.good_date.filter(self.good_date.ID == 1).select('FYEAR').collect()[0][0]\n",
        "    assert year1 == 1994\n",
        "    year2 = self.good_date.filter(self.good_date.ID == 2).select('FYEAR').collect()[0][0]\n",
        "    assert year2 == 1583\n",
        "\n",
        "  def testGetMonth(self):\n",
        "    print('\\n Test getMonth')\n",
        "    self.good_date = self.good_date.withColumn('MONTH', getMonth('DATE'))\n",
        "    self.good_date.show()\n",
        "    month1 = self.good_date.filter(self.good_date.ID == 1).select('MONTH').collect()[0][0]\n",
        "    assert month1 == 1\n",
        "    month2 = self.good_date.filter(self.good_date.ID == 2).select('MONTH').collect()[0][0]\n",
        "    assert month2 == 12\n",
        "\n",
        "  def testGetDay(self):\n",
        "    print('\\n Test getDay')\n",
        "    self.good_date = self.good_date.withColumn('DAY', getDay('DATE'))\n",
        "    self.good_date.show()\n",
        "    day1 = self.good_date.filter(self.good_date.ID == 1).select('DAY').collect()[0][0]\n",
        "    assert day1 == 2\n",
        "    day2 = self.good_date.filter(self.good_date.ID == 2).select('DAY').collect()[0][0]\n",
        "    assert day2 == 24\n",
        "\n",
        "  def testGetDekad(self):\n",
        "    print('\\n Test getDekad')\n",
        "    self.good_date = self.good_date.withColumn('DEKAD', getDekad('DATE'))\n",
        "    self.good_date.show()\n",
        "    dekad1 = self.good_date.filter(self.good_date.ID == 1).select('DEKAD').collect()[0][0]\n",
        "    assert dekad1 == 1\n",
        "    dekad2 = self.good_date.filter(self.good_date.ID == 2).select('DEKAD').collect()[0][0]\n",
        "    assert dekad2 == 36\n",
        "\n",
        "  def testCropIDToName(self):\n",
        "    print('\\n Test cropIDToName')\n",
        "    crop_name = cropIDToName(crop_name_dict, 6)\n",
        "    print(6, ':' + crop_name)\n",
        "    assert crop_name == 'sugarbeet'\n",
        "    crop_name = cropIDToName(crop_name_dict, 8)\n",
        "    print(8, ':' + crop_name)\n",
        "    assert crop_name == 'NA'\n",
        "\n",
        "  def testCropNameToID(self):\n",
        "    print('\\n Test cropNameToID')\n",
        "    crop_id = cropNameToID(crop_id_dict, 'Potatoes')\n",
        "    print('Potatoes:', crop_id)\n",
        "    assert crop_id == 7\n",
        "    crop_id = cropNameToID(crop_id_dict, 'Soybean')\n",
        "    print('Soybean:', crop_id)\n",
        "    assert crop_id == 0\n",
        "\n",
        "  def testPrintFeatures(self):\n",
        "    print('\\n Test printFeatures')\n",
        "    features = ['feat' + str(i+1) for i in range(15)]\n",
        "    num_features = len(features)\n",
        "    num_half = np.cast['int64'](np.floor(num_features/2))\n",
        "    indices1 = [ i for i in range(num_features)]\n",
        "    indices2 = [ 2*i for i in range(num_half)]\n",
        "    indices3 = [ (2*i + 1) for i in range(num_half)]\n",
        "\n",
        "    printFeatures(features, indices1)\n",
        "    printFeatures(features, indices2)\n",
        "    printFeatures(features, indices3)\n",
        "\n",
        "  def testPlotTrend(self):\n",
        "    print('\\n Test plotTrend')\n",
        "    years = [yr for yr in range(2000, 2010)]\n",
        "    trend_values = [ (i + 1) for i in range(50, 60)]\n",
        "    actual_values = []\n",
        "    for tval in trend_values:\n",
        "      if (tval % 2) == 0:\n",
        "        actual_values.append(tval + 0.5)\n",
        "      else:\n",
        "        actual_values.append(tval - 0.5)\n",
        "\n",
        "    plotTrend(years, actual_values, trend_values, 'YIELD')\n",
        "\n",
        "  def testPlotTrueVSPredicted(self):\n",
        "    print('\\n Test plotTrueVSPredicted')\n",
        "    Y_true = [ (i + 1) for i in range(50, 60)]\n",
        "    Y_predicted = []\n",
        "    for tval in Y_true:\n",
        "      if (tval % 2) == 0:\n",
        "        Y_predicted.append(tval + 0.5)\n",
        "      else:\n",
        "        Y_predicted.append(tval - 0.5)\n",
        "\n",
        "    Y_true = np.asarray(Y_true)\n",
        "    Y_predicted = np.asarray(Y_predicted)\n",
        "\n",
        "    plotTrueVSPredicted(Y_true, Y_predicted)\n",
        "\n",
        "  def runAllTests(self):\n",
        "    print('\\nTest Utility Functions BEGIN\\n')\n",
        "    self.testDateFormat()\n",
        "    self.testGetYear()\n",
        "    self.testGetMonth()\n",
        "    self.testGetDay()\n",
        "    self.testGetDekad()\n",
        "    self.testCropIDToName()\n",
        "    self.testCropNameToID()\n",
        "    self.testPrintFeatures()\n",
        "    self.testPlotTrend()\n",
        "    self.testPlotTrueVSPredicted()\n",
        "    print('\\nTest Utility Functions END\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1iW23SQzYN09"
      },
      "source": [
        "### Test Data Loading"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gWKTClTNYN0_"
      },
      "outputs": [],
      "source": [
        "#%%writefile test_data_loading.py\n",
        "class TestDataLoader():\n",
        "  def __init__(self, spark):\n",
        "    cyp_config = CYPConfiguration()\n",
        "    self.nuts_level = cyp_config.getNUTSLevel()\n",
        "    data_sources = { 'SOIL' : self.nuts_level }\n",
        "    cyp_config.setDataSources(data_sources)\n",
        "    cyp_config.setDebugLevel(2)\n",
        "\n",
        "    self.data_loader = CYPDataLoader(spark, cyp_config)\n",
        "\n",
        "  def testDataLoad(self):\n",
        "    print('\\nTest loadData, loadAllData')\n",
        "    soil_df = self.data_loader.loadData('SOIL', self.nuts_level)\n",
        "    assert soil_df is not None\n",
        "    soil_df.show(5)\n",
        "\n",
        "    all_dfs = self.data_loader.loadAllData()\n",
        "    soil_df = all_dfs['SOIL']\n",
        "    assert soil_df is not None\n",
        "    soil_df.show(5)\n",
        "\n",
        "  def runAllTests(self):\n",
        "    print('\\nTest Data Loader BEGIN\\n')\n",
        "    self.testDataLoad()\n",
        "    print('\\nTest Data Loader END\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vil46Q08YQ9v"
      },
      "source": [
        "### Test Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zNZQd9kEYQ9w"
      },
      "outputs": [],
      "source": [
        "#%%writefile test_data_preprocessing.py\n",
        "class TestDataPreprocessor():\n",
        "  def __init__(self, spark):\n",
        "    cyp_config = CYPConfiguration()\n",
        "    cyp_config.setDebugLevel(2)\n",
        "    self.preprocessor = CYPDataPreprocessor(spark, cyp_config)\n",
        "\n",
        "    # create a small wofost data set\n",
        "    # preprocessing currently extracts the year and dekad only\n",
        "    self.wofost_df = spark.createDataFrame([(6, 'NL11', '19790110', 0.0, 0),\n",
        "                                            (6, 'NL11', '19790121', 0.0, 0),\n",
        "                                            (6, 'NL11', '19790331', 0.0, 50),\n",
        "                                            (6, 'NL11', '19790510', 0.0, 100),\n",
        "                                            (6, 'NL11', '19790821', 0.0, 150),\n",
        "                                            (6, 'NL11', '19790831', 0.0, 201),\n",
        "                                            (6, 'NL11', '19790910', 0.0, 201),\n",
        "                                            (6, 'NL11', '19800110', 0.0, 0),\n",
        "                                            (6, 'NL11', '19800121', 0.0, 0),\n",
        "                                            (6, 'NL11', '19800331', 0.0, 50),\n",
        "                                            (6, 'NL11', '19800610', 0.0, 100),\n",
        "                                            (6, 'NL11', '19800721', 0.0, 150),\n",
        "                                            (6, 'NL11', '19800831', 0.0, 200),\n",
        "                                            (6, 'NL11', '19800910', 0.0, 201),\n",
        "                                            (6, 'NL11', '19800921', 0.0, 201)],\n",
        "                                           ['CROP_ID', 'IDREGION', 'DATE', 'POT_YB', 'DVS'])\n",
        "\n",
        "    self.crop_season = None\n",
        "\n",
        "    # Create a small meteo dekadal data set\n",
        "    # Preprocessing currently extracts the year and dekad, and computes climate\n",
        "    # water balance.\n",
        "    self.meteo_dekdf = spark.createDataFrame([('NL11', '19790110', 1.2, 0.2),\n",
        "                                              ('NL11', '19790121', 2.1, 0.2),\n",
        "                                              ('NL11', '19790131', 0.2, 1.2),\n",
        "                                              ('NL11', '19790210', 0.1, 2.0),\n",
        "                                              ('NL11', '19790221', 0.0, 2.1),\n",
        "                                              ('NL11', '19790228', 1.0, 0.8),\n",
        "                                              ('NL11', '19790310', 1.1, 1.0),\n",
        "                                              ('NL11', '19800110', 1.2, 0.2),\n",
        "                                              ('NL11', '19800121', 2.1, 0.2),\n",
        "                                              ('NL11', '19800131', 0.2, 1.2),\n",
        "                                              ('NL11', '19800210', 0.1, 2.0),\n",
        "                                              ('NL11', '19800221', 0.0, 2.1),\n",
        "                                              ('NL11', '19800228', 1.0, 0.8),\n",
        "                                              ('NL11', '19800310', 1.1, 1.0)],\n",
        "                                             ['IDREGION', 'DATE', 'PREC', 'ET0'])\n",
        "\n",
        "    # Create a small meteo daily data set\n",
        "    # Preprocessing currently converts daily data to dekadal data by taking AVG\n",
        "    # for all indicators except TMAX (MAX is used instead) and TMIN (MIN is used instead).\n",
        "    query = 'select IDREGION, FYEAR, DEKAD, max(TMAX) as TMAX, min(TMIN) as TMIN, '\n",
        "    query = query + ' bround(avg(TAVG), 2) as TAVG, bround(sum(PREC), 2) as PREC, '\n",
        "    query = query + ' bround(sum(ET0), 2) as ET0, bround(avg(RAD), 2) as RAD, '\n",
        "    query = query + ' bround(sum(CWB), 2) as CWB '\n",
        "    self.meteo_daydf = spark.createDataFrame([('NL11', '19790101', 1.2, 0.2, 8.5, -1.2, 5.5, 10000.0),\n",
        "                                              ('NL11', '19790102', 2.1, 0.2, 9.1, 0.3, 6.1, 12000.0),\n",
        "                                              ('NL11', '19790103', 0.2, 1.2, 10.4, 1.2, 7.2, 14000.0),\n",
        "                                              ('NL11', '19790104', 0.1, 2.0, 8.1, -1.5, 5.2, 10000.0),\n",
        "                                              ('NL11', '19790105', 0.0, 2.1, 10.2, 1.0, 7.5, 13000.0),\n",
        "                                              ('NL11', '19790106', 1.2, 0.2, 11.2, 2.5, 8.2, 16000.0),\n",
        "                                              ('NL11', '19790112', 2.1, 0.5, 9.2, 0.5, 5.5, 12000.0),\n",
        "                                              ('NL11', '19790113', 0.2, 1.1, 10.2, 1.4, 7.1, 14000.0),\n",
        "                                              ('NL11', '19790114', 0.1, 2.0, 12.0, 3.2, 8.3, 15000.0),\n",
        "                                              ('NL11', '19790115', 0.0, 1.5, 13.1, 4.5, 9.2, 17000.0),\n",
        "                                              ('NL11', '19790122', 2.1, 0.5, 9.2, 0.5, 5.5, 12000.0),\n",
        "                                              ('NL11', '19790123', 0.2, 1.1, 10.2, 1.4, 7.1, 14000.0),\n",
        "                                              ('NL11', '19790124', 0.1, 2.0, 12.0, 3.2, 8.3, 15000.0),\n",
        "                                              ('NL11', '19790125', 0.0, 1.5, 13.1, 4.5, 9.2, 17000.0),\n",
        "                                              ('NL11', '19800101', 1.2, 0.2, 8.5, -1.2, 5.5, 10000.0),\n",
        "                                              ('NL11', '19800102', 2.1, 0.2, 9.1, 0.3, 6.1, 12000.0),\n",
        "                                              ('NL11', '19800103', 0.2, 1.2, 10.4, 1.2, 7.2, 14000.0),\n",
        "                                              ('NL11', '19800104', 0.1, 2.0, 8.1, -1.5, 5.2, 10000.0),\n",
        "                                              ('NL11', '19800105', 0.0, 2.1, 10.2, 1.0, 7.5, 13000.0),\n",
        "                                              ('NL11', '19800106', 1.2, 0.2, 11.2, 2.5, 8.2, 16000.0),\n",
        "                                              ('NL11', '19800112', 2.1, 0.5, 9.2, 0.5, 5.5, 12000.0),\n",
        "                                              ('NL11', '19800113', 0.2, 1.1, 10.2, 1.4, 7.1, 14000.0),\n",
        "                                              ('NL11', '19800114', 0.1, 2.0, 12.0, 3.2, 8.3, 15000.0),\n",
        "                                              ('NL11', '19800115', 0.0, 1.5, 13.1, 4.5, 9.2, 17000.0),\n",
        "                                              ('NL11', '19800122', 2.1, 0.5, 9.2, 0.5, 5.5, 12000.0),\n",
        "                                              ('NL11', '19800123', 0.2, 1.1, 10.2, 1.4, 7.1, 14000.0),\n",
        "                                              ('NL11', '19800124', 0.1, 2.0, 12.0, 3.2, 8.3, 15000.0),\n",
        "                                              ('NL11', '19800125', 0.0, 1.5, 13.1, 4.5, 9.2, 17000.0)],\n",
        "                                             ['IDREGION', 'DATE', 'PREC', 'ET0', 'TMAX', 'TMIN', 'TAVG', 'RAD'])\n",
        "\n",
        "    # Create a small remote sensing data set\n",
        "    # Preprocessing currently extracts the year and dekad\n",
        "    self.rs_df1 = spark.createDataFrame([('NL11', '19790321', 0.47),\n",
        "                                         ('NL11', '19790331', 0.49),\n",
        "                                         ('NL11', '19790410', 0.55),\n",
        "                                         ('NL11', '19790421', 0.49),\n",
        "                                         ('NL11', '19790430', 0.64),\n",
        "                                         ('NL11', '19800110', 0.42),\n",
        "                                         ('NL11', '19800121', 2.43),\n",
        "                                         ('NL11', '19800131', 0.41),\n",
        "                                         ('NL11', '19800210', 0.42),\n",
        "                                         ('NL11', '19800221', 0.44),\n",
        "                                         ('NL11', '19800228', 0.45),\n",
        "                                         ('NL11', '19800310', 2.43)],\n",
        "                                        ['IDREGION', 'DATE', 'FAPAR'])\n",
        "\n",
        "    self.rs_df2 = spark.createDataFrame([('FR10', '19790110', 0.42),\n",
        "                                         ('FR10', '19790121', 0.43),\n",
        "                                         ('FR10', '19790131', 0.41),\n",
        "                                         ('FR10', '19790210', 0.42),\n",
        "                                         ('FR10', '19790221', 0.44),\n",
        "                                         ('FR10', '19790228', 0.45),\n",
        "                                         ('FR10', '19790310', 0.47),\n",
        "                                         ('FR10', '19790321', 0.49),\n",
        "                                         ('FR10', '19790331', 0.55),\n",
        "                                         ('FR10', '19790410', 0.62),\n",
        "                                         ('FR10', '19790421', 0.66),\n",
        "                                         ('FR10', '19800110', 0.42),\n",
        "                                         ('FR10', '19800121', 2.43),\n",
        "                                         ('FR10', '19800131', 0.41),\n",
        "                                         ('FR10', '19800210', 0.42),\n",
        "                                         ('FR10', '19800221', 0.44),\n",
        "                                         ('FR10', '19800228', 0.45),\n",
        "                                         ('FR10', '19800310', 2.43)],\n",
        "                                        ['IDREGION', 'DATE', 'FAPAR'])\n",
        "  \n",
        "    self.crop_season_nuts3 = spark.createDataFrame([('FR101', '1979', 0, 27),\n",
        "                                                    ('FR101', '1980', 27, 28),\n",
        "                                                    ('FR102', '1979', 0, 27),\n",
        "                                                    ('FR102', '1980', 27, 29)],\n",
        "                                                   ['IDREGION', 'FYEAR', 'PREV_SEASON_END', 'SEASON_END'])\n",
        "\n",
        "    # Create small yield data sets\n",
        "    # Two formats are preprocessed: (1) year and yield are columns,\n",
        "    # (2) years are columns with yield values in rows\n",
        "    # Preprocessing currently converts (2) into 1\n",
        "    self.yield_df1 = spark.createDataFrame([('potatoes', 'FR102', '1989', 29.75),\n",
        "                                            ('potatoes', 'FR102', '1990', 25.44),\n",
        "                                            ('potatoes', 'FR103', '1989', 30.2),\n",
        "                                            ('potatoes', 'FR103', '1990', 29.9),\n",
        "                                            ('sugarbeet', 'FR102', '1989', 66.0),\n",
        "                                            ('sugarbeet', 'FR102', '1990', 55.0),\n",
        "                                            ('sugarbeet', 'FR103', '1989', 69.3),\n",
        "                                            ('sugarbeet', 'FR103', '1990', 59.1)],\n",
        "                                           ['Crop', 'IDREGION', 'FYEAR', 'YIELD'])\n",
        "\n",
        "    self.yield_df2 = spark.createDataFrame([('Total potatoes', 'NL11', 38.0, 40.5, 40.0),\n",
        "                                            ('Total potatoes', 'NL12', 49.0, 44.0, 46.8),\n",
        "                                            ('Spring barley', 'NL13', 4.6, 5.5, 6.6),\n",
        "                                            ('Spring barley', 'NL12', 5.6, 6.1, 7.0)],\n",
        "                                           ['Crop', 'IDREGION', '1994', '1995', '1996'])\n",
        "\n",
        "  def testExtractYearDekad(self):\n",
        "    print('WOFOST data after extracting year and dekad')\n",
        "    print('-------------------------------------------')\n",
        "    self.preprocessor.extractYearDekad(self.wofost_df).show(10)\n",
        "\n",
        "  def testPreprocessWofost(self):\n",
        "    print('WOFOST data after preprocessing')\n",
        "    print('--------------------------------')\n",
        "    self.wofost_df = self.wofost_df.filter(self.wofost_df['CROP_ID'] == 6).drop('CROP_ID')\n",
        "    self.crop_season = self.preprocessor.getCropSeasonInformation(self.wofost_df,\n",
        "                                                                  False)\n",
        "    self.wofost_df = self.preprocessor.preprocessWofost(self.wofost_df,\n",
        "                                                        self.crop_season,\n",
        "                                                        False)\n",
        "    self.wofost_df.show(5)\n",
        "    self.crop_season.show(5)\n",
        "\n",
        "  def testPreprocessMeteo(self):\n",
        "    print('Meteo dekadal data after preprocessing')\n",
        "    print('--------------------------------------')\n",
        "    self.meteo_dekdf = self.preprocessor.preprocessMeteo(self.meteo_dekdf,\n",
        "                                                         self.crop_season,\n",
        "                                                         False)\n",
        "    self.meteo_dekdf.show(5)\n",
        "\n",
        "  def testPreprocessMeteoDaily(self):\n",
        "    self.meteo_daydf = self.preprocessor.preprocessMeteo(self.meteo_daydf,\n",
        "                                                         self.crop_season,\n",
        "                                                         False)\n",
        "    self.meteo_daydf = self.preprocessor.preprocessMeteoDaily(self.meteo_daydf)\n",
        "    print('Meteo daily data after preprocessing')\n",
        "    print('------------------------------------')\n",
        "    self.meteo_daydf.show(5)\n",
        "\n",
        "  def testPreprocessRemoteSensing(self):\n",
        "    self.rs_df1 = self.preprocessor.preprocessRemoteSensing(self.rs_df1,\n",
        "                                                            self.crop_season,\n",
        "                                                            False)\n",
        "    print('Remote sensing data after preprocessing')\n",
        "    print('---------------------------------------')\n",
        "    self.rs_df1.show(5)\n",
        "\n",
        "  def testRemoteSensingNUTS2ToNUTS3(self):\n",
        "    print('Remote sensing data before preprocessing')\n",
        "    print('---------------------------------------')\n",
        "    self.rs_df2.show()\n",
        "    nuts3_regions = [reg[0] for reg in self.yield_df1.select('IDREGION').distinct().collect()]\n",
        "    self.rs_df2 = self.preprocessor.remoteSensingNUTS2ToNUTS3(self.rs_df2, nuts3_regions)\n",
        "    print('Remote sensing data at NUTS3')\n",
        "    print('-----------------------------')\n",
        "    self.rs_df2.show(5)\n",
        "\n",
        "    self.rs_df2 = self.preprocessor.preprocessRemoteSensing(self.rs_df2,\n",
        "                                                            self.crop_season_nuts3,\n",
        "                                                            False)\n",
        "    print('Remote sensing data after preprocessing')\n",
        "    print('---------------------------------------')\n",
        "    self.rs_df2.show(5)\n",
        "\n",
        "  def testPreprocessYield(self):\n",
        "    self.yield_df1 = self.preprocessor.preprocessYield(self.yield_df1, 7)\n",
        "    print('Yield data format 1 after preprocessing')\n",
        "    print('--------------------------------------')\n",
        "    self.yield_df1.show(5)\n",
        "\n",
        "    print('Yield data format 2 before preprocessing')\n",
        "    print('----------------------------------------')\n",
        "    self.yield_df2.show(5)\n",
        "\n",
        "    self.yield_df2 = self.preprocessor.preprocessYield(self.yield_df2, 7)\n",
        "    print('Yield data format 2 after preprocessing')\n",
        "    print('----------------------------------------')\n",
        "    self.yield_df2.show(5)\n",
        "\n",
        "  def runAllTests(self):\n",
        "    print('\\nTest Data Preprocessor BEGIN\\n')\n",
        "    self.testExtractYearDekad()\n",
        "    self.testPreprocessWofost()\n",
        "    self.testPreprocessMeteo()\n",
        "    self.testPreprocessMeteoDaily()\n",
        "    self.testPreprocessRemoteSensing()\n",
        "    self.testRemoteSensingNUTS2ToNUTS3()\n",
        "    self.testPreprocessYield()\n",
        "    print('\\nTest Data Preprocessor END\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ylzks219YUwm"
      },
      "source": [
        "### Test Data Summary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wD-xHJSiYUwn"
      },
      "outputs": [],
      "source": [
        "#%%writefile test_data_summary.py\n",
        "class TestDataSummarizer():\n",
        "  def __init__(self, spark):\n",
        "    cyp_config = CYPConfiguration()\n",
        "    cyp_config.setDebugLevel(2)\n",
        "    self.data_summarizer = CYPDataSummarizer(cyp_config)\n",
        "\n",
        "    # create a small wofost data set\n",
        "    self.wofost_df = spark.createDataFrame([('NL11', '1979', 14, 0.0, 0.0, 5.0),\n",
        "                                            ('NL11', '1979', 15, 2.0, 1.0, 10.0),\n",
        "                                            ('NL11', '1979', 16, 5.0, 4.0, 7.0),\n",
        "                                            ('NL11', '1979', 17, 15.0, 12.0, 4.0),\n",
        "                                            ('NL11', '1979', 18, 40.0, 35.0, 6.0),\n",
        "                                            ('NL11', '1979', 19, 100.0, 80.0, 5.0),\n",
        "                                            ('NL11', '1979', 20, 150.0, 120.0, 3.0),\n",
        "                                            ('NL11', '1980', 13, 0.0, 0.0, 15.0),\n",
        "                                            ('NL11', '1980', 14, 5.0, 2.0, 12.0),\n",
        "                                            ('NL11', '1980', 15, 15.0, 12.0, 10.0),\n",
        "                                            ('NL11', '1980', 16, 50.0, 40.0, 8.0),\n",
        "                                            ('NL11', '1980', 17, 100.0, 80.0, 4.0),\n",
        "                                            ('NL11', '1980', 18, 200.0, 140.0, 5.0),\n",
        "                                            ('NL11', '1980', 19, 200.0, 150.0, 12.0),\n",
        "                                            ('NL11', '1980', 20, 200.0, 150.0, 12.0)],\n",
        "                                           ['IDREGION', 'FYEAR', 'DEKAD', 'POT_YB', 'WLIM_YB', 'RSM'])\n",
        "\n",
        "    self.wofost_df2 = spark.createDataFrame([('NL11', '1979', 12, '1979', 22, 0),\n",
        "                                             ('NL11', '1979', 13, '1979', 23, 1),\n",
        "                                             ('NL11', '1979', 14, '1979', 24, 4),\n",
        "                                             ('NL11', '1979', 15, '1979', 25, 70),\n",
        "                                             ('NL11', '1979', 16, '1979', 26, 101),\n",
        "                                             ('NL11', '1979', 19, '1979', 29, 150),\n",
        "                                             ('NL11', '1979', 21, '1979', 31, 180),\n",
        "                                             ('NL11', '1979', 23, '1979', 33, 200),\n",
        "                                             ('NL11', '1979', 24, '1979', 34, 201),\n",
        "                                             ('NL11', '1979', 25, '1979', 35, 201),\n",
        "                                             ('NL11', '1980', 12, '1980', 22, 0),\n",
        "                                             ('NL11', '1980', 13, '1980', 23, 2),\n",
        "                                             ('NL11', '1980', 14, '1980', 24, 15),\n",
        "                                             ('NL11', '1980', 15, '1980', 25, 80),\n",
        "                                             ('NL11', '1980', 16, '1980', 26, 99),\n",
        "                                             ('NL11', '1980', 19, '1980', 29, 140),\n",
        "                                             ('NL11', '1980', 21, '1980', 31, 170),\n",
        "                                             ('NL11', '1980', 23, '1980', 33, 195),\n",
        "                                             ('NL11', '1980', 24, '1980', 34, 201),\n",
        "                                             ('NL11', '1980', 25, '1980', 35, 201)],\n",
        "                                           ['IDREGION', 'FYEAR', 'DEKAD', 'CAMPAIGN_YEAR', 'CAMPAIGN_DEKAD', 'DVS'])\n",
        "\n",
        "    # Create a small meteo dekadal data set\n",
        "    self.meteo_df = spark.createDataFrame([('NL11', '1979', 1, '1979', 11, 1.2, 8.5, -1.2, 5.5, 10000.0),\n",
        "                                           ('NL11', '1979', 2, '1979', 12, 2.1, 9.1, 0.3, 6.1, 12000.0),\n",
        "                                           ('NL11', '1979', 3, '1979', 13, 0.2, 10.4, 1.2, 7.2, 14000.0),\n",
        "                                           ('NL11', '1979', 4, '1979', 14, 0.1, 8.1, -1.5, 5.2, 10000.0),\n",
        "                                           ('NL11', '1979', 5, '1979', 15, 0.0, 10.2, 1.0, 7.5, 13000.0),\n",
        "                                           ('NL12', '1979', 1, '1979', 12, 1.2, 11.2, 2.5, 8.2, 16000.0),\n",
        "                                           ('NL12', '1979', 2, '1979', 13, 2.1, 9.2, 0.5, 5.5, 12000.0),\n",
        "                                           ('NL12', '1979', 3, '1979', 14, 0.2, 10.2, 1.4, 7.1, 14000.0),\n",
        "                                           ('NL12', '1979', 4, '1979', 15, 0.1, 12.0, 3.2, 8.3, 15000.0),\n",
        "                                           ('NL12', '1979', 5, '1979', 16, 0.0, 13.1, 4.5, 9.2, 17000.0)],\n",
        "                                          ['IDREGION', 'FYEAR', 'DEKAD', 'CAMPAIGN_YEAR', 'CAMPAIGN_DEKAD', 'PREC', 'TMAX', 'TMIN', 'TAVG', 'RAD'])\n",
        "\n",
        "    # Create a small remote sensing data set\n",
        "    # Preprocessing currently extracts the year and dekad\n",
        "    self.rs_df = spark.createDataFrame([('NL11', '1979', 1, 0.42),\n",
        "                                         ('NL11', '1979', 2, 0.41),\n",
        "                                         ('NL11', '1979', 3, 0.42),\n",
        "                                         ('NL11', '1980', 1, 0.44),\n",
        "                                         ('NL11', '1980', 2, 0.45),\n",
        "                                        ('NL11', '1980', 3, 0.43)],\n",
        "                                        ['IDREGION', 'FYEAR', 'DEKAD', 'FAPAR'])\n",
        "\n",
        "    # Create a small yield data set\n",
        "    self.yield_df = spark.createDataFrame([(7, 'FR102', '1989', 29.75),\n",
        "                                           (7, 'FR102', '1990', 25.44),\n",
        "                                           (7, 'FR103', '1989', 30.2),\n",
        "                                           (7, 'FR103', '1990', 29.9),\n",
        "                                           (6, 'FR102', '1989', 66.0),\n",
        "                                           (6, 'FR102', '1990', 55.0),\n",
        "                                           (6, 'FR103', '1989', 69.3),\n",
        "                                           (6, 'FR103', '1990', 59.1)],\n",
        "                                          ['CROP_ID', 'IDREGION', 'FYEAR', 'YIELD'])\n",
        "\n",
        "  def testWofostDVSSummary(self):\n",
        "    print('WOFOST Crop Calendar Summary using DVS')\n",
        "    print('-----------------------------')\n",
        "    self.data_summarizer.wofostDVSSummary(self.wofost_df2).show()\n",
        "\n",
        "  def testWofostIndicatorsSummary(self):\n",
        "    print('WOFOST indicators summary')\n",
        "    print('--------------------------')\n",
        "    min_cols = ['IDREGION']\n",
        "    max_cols = ['IDREGION', 'POT_YB', 'WLIM_YB']\n",
        "    avg_cols = ['IDREGION', 'RSM']\n",
        "    self.data_summarizer.indicatorsSummary(self.wofost_df, min_cols, max_cols, avg_cols).show()\n",
        "\n",
        "  def testMeteoIndicatorsSummary(self):\n",
        "    print('Meteo indicators summary')\n",
        "    print('-------------------------')\n",
        "    meteo_cols = self.meteo_df.columns[3:]\n",
        "    min_cols = ['IDREGION'] + meteo_cols\n",
        "    max_cols = ['IDREGION'] + meteo_cols\n",
        "    avg_cols = ['IDREGION'] + meteo_cols\n",
        "    self.data_summarizer.indicatorsSummary(self.meteo_df, min_cols, max_cols, avg_cols).show()\n",
        "\n",
        "  def testRemoteSensingSummary(self):\n",
        "    print('Remote sensing indicators summary')\n",
        "    print('----------------------------------')\n",
        "    rs_cols = ['FAPAR']\n",
        "    min_cols = ['IDREGION'] + rs_cols\n",
        "    max_cols = ['IDREGION'] + rs_cols\n",
        "    avg_cols = ['IDREGION'] + rs_cols\n",
        "    self.data_summarizer.indicatorsSummary(self.rs_df, min_cols, max_cols, avg_cols).show()\n",
        "\n",
        "  def testYieldSummary(self):\n",
        "    crop = 'potatoes'\n",
        "    print('Yield summary for', crop)\n",
        "    print('-----------------------------')\n",
        "    crop_id = cropNameToID(crop_id_dict, crop)\n",
        "    self.yield_df = self.yield_df.filter(self.yield_df.CROP_ID == crop_id)\n",
        "    self.data_summarizer.yieldSummary(self.yield_df).show()\n",
        "\n",
        "  def runAllTests(self):\n",
        "    print('\\nTest Data Summarizer BEGIN\\n')\n",
        "    self.testWofostDVSSummary()\n",
        "    self.testWofostIndicatorsSummary()\n",
        "    self.testMeteoIndicatorsSummary()\n",
        "    self.testRemoteSensingSummary()\n",
        "    self.testYieldSummary()\n",
        "    print('\\nTest Data Summarizer END\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hjepG1gAYXrc"
      },
      "source": [
        "### Test Yield Trend Estimation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rRoEhIT6YXre"
      },
      "outputs": [],
      "source": [
        "#%%writefile test_yield_trend.py\n",
        "class TestYieldTrendEstimator():\n",
        "  def __init__(self, yield_df):\n",
        "    # TODO: Create a small yield data set\n",
        "    self.yield_df = yield_df\n",
        "    cyp_config = CYPConfiguration()\n",
        "    self.verbose = 2\n",
        "    cyp_config.setDebugLevel(self.verbose)\n",
        "    self.trend_est = CYPYieldTrendEstimator(cyp_config)\n",
        "\n",
        "  def testYieldTrendTwoRegions(self):\n",
        "    print('\\nFind the optimal trend window and estimate trend for first 2 regions')\n",
        "    pd_yield_df = self.yield_df.toPandas()\n",
        "    regions = sorted(pd_yield_df['IDREGION'].unique())\n",
        "    reg1 = regions[0]\n",
        "    pd_reg1_df = pd_yield_df[pd_yield_df['IDREGION'] == reg1]\n",
        "    reg1_num_years = len(pd_reg1_df.index)\n",
        "    reg1_max_year = pd_reg1_df['FYEAR'].max()\n",
        "    reg1_min_year = pd_reg1_df['FYEAR'].min()\n",
        "\n",
        "    if (self.verbose > 2):\n",
        "      print('\\nPrint Yield Trend Rounds')\n",
        "      print('------------------------')\n",
        "\n",
        "    trend_windows = [5]\n",
        "    self.trend_est.printYieldTrendRounds(self.yield_df, reg1, trend_windows)\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      print('\\n Fixed Trend Window prediction for region 1')\n",
        "      print('---------------------------------------------------')\n",
        "    trend_window = 5\n",
        "    pd_fixed_win_df = self.trend_est.getFixedWindowTrend(self.yield_df, reg1, reg1_max_year,\n",
        "                                                         trend_window)\n",
        "    if (self.verbose > 1):\n",
        "      print(pd_fixed_win_df.head(1))\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      print('\\n Optimal Trend Window and prediction for region 1')\n",
        "      print('---------------------------------------------------')\n",
        "  \n",
        "    trend_windows = [5, 7]\n",
        "    pd_opt_win_df = self.trend_est.getOptimalWindowTrend(self.yield_df, reg1, reg1_max_year,\n",
        "                                                         trend_windows)\n",
        "    if (self.verbose > 1):\n",
        "      print(pd_opt_win_df.head(1))\n",
        "\n",
        "    reg2 = regions[1]\n",
        "    pd_reg2_df = pd_yield_df[pd_yield_df['IDREGION'] == reg2]\n",
        "    reg2_num_years = len(pd_reg2_df.index)\n",
        "    reg2_max_year = pd_reg2_df['FYEAR'].max()\n",
        "    reg2_min_year = pd_reg2_df['FYEAR'].min()\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      print('\\n Fixed Trend Window prediction for region 2')\n",
        "      print('---------------------------------------------------')\n",
        "    trend_window = 5\n",
        "    pd_fixed_win_df = self.trend_est.getFixedWindowTrend(self.yield_df, reg2, reg2_max_year,\n",
        "                                                         trend_window)\n",
        "    if (self.verbose > 1):\n",
        "      print(pd_fixed_win_df.head(1))\n",
        "\n",
        "    if (self.verbose > 1):\n",
        "      print('\\n Optimal Trend Window and prediction for region 2')\n",
        "      print('---------------------------------------------------')\n",
        "\n",
        "    pd_opt_win_df = self.trend_est.getOptimalWindowTrend(self.yield_df, reg2, reg2_max_year,\n",
        "                                                         trend_windows)\n",
        "    if (self.verbose > 1):\n",
        "      print(pd_opt_win_df.head(1))\n",
        "\n",
        "  def testYieldTrendAllRegions(self):\n",
        "    print('\\nYield trend estimation for all regions')\n",
        "\n",
        "    print('\\nOptimal Trend Windows')\n",
        "    pd_trend_df = self.trend_est.getOptimalWindowTrendFeatures(self.yield_df)\n",
        "    print(pd_trend_df.head(5))\n",
        "\n",
        "    print('\\nFixed Trend Window')\n",
        "    pd_trend_df = self.trend_est.getFixedWindowTrendFeatures(self.yield_df)\n",
        "    print(pd_trend_df.head(5))\n",
        "\n",
        "  def runAllTests(self):\n",
        "    print('\\nTest Yield Trend Estimator BEGIN\\n')\n",
        "    self.testYieldTrendTwoRegions()\n",
        "    self.testYieldTrendAllRegions()\n",
        "    print('\\nTest Yield Trend Estimator END\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ih1GgFhZe13K"
      },
      "source": [
        "### Test custom train, test split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NQNoWy51e13M"
      },
      "outputs": [],
      "source": [
        "#%%writefile test_train_test_split.py\n",
        "class TestCustomTrainTestSplit:\n",
        "  def __init__(self, yield_df):\n",
        "    cyp_config = CYPConfiguration()\n",
        "    self.verbose = 2\n",
        "    cyp_config.setDebugLevel(self.verbose)\n",
        "    self.yield_df = yield_df\n",
        "    self.trTsSplitter = CYPTrainTestSplitter(cyp_config)\n",
        "\n",
        "  def testCustomTrainTestSplit(self):\n",
        "    print('\\nTest customTrainTestSplit')\n",
        "    test_fraction = 0.2\n",
        "    regions = [reg[0] for reg in self.yield_df.select('IDREGION').distinct().collect()]\n",
        "    num_regions = len(regions)\n",
        "    test_years = self.trTsSplitter.trainTestSplit(self.yield_df, test_fraction, True)\n",
        "    all_years = [yr[0] for yr in self.yield_df.select('FYEAR').distinct().collect()]\n",
        "    yield_train_df = self.yield_df.filter(~self.yield_df['FYEAR'].isin(test_years))\n",
        "    yield_test_df = self.yield_df.filter(self.yield_df['FYEAR'].isin(test_years))\n",
        "\n",
        "    if(self.verbose > 1):\n",
        "      print('\\nCustom training, test split using yield trend')\n",
        "      print('---------------------------------------------')\n",
        "      print('Estimated size of test data', num_regions * np.floor(len(all_years) * test_fraction))\n",
        "      print('Data Size:', yield_train_df.count(), yield_test_df.count())\n",
        "      print('Test years:', test_years)\n",
        "\n",
        "    test_years = self.trTsSplitter.trainTestSplit(self.yield_df, test_fraction, False)\n",
        "\n",
        "    if(self.verbose > 1):\n",
        "      print('\\ncustom training, test split without yield trend')\n",
        "      print('------------------------------------------------')\n",
        "      print('Estimated size of test data', num_regions * np.floor(len(all_years) * test_fraction))\n",
        "      print('Data Size:', yield_train_df.count(), yield_test_df.count())\n",
        "      print('Test years:', test_years)\n",
        "\n",
        "  def testCustomKFoldValidationSplit(self):\n",
        "    print('\\nTest customKFoldValidationSplit')\n",
        "    test_fraction = 0.2\n",
        "    num_folds = 5\n",
        "    test_years = self.trTsSplitter.trainTestSplit(self.yield_df, test_fraction, 'Y')\n",
        "    yield_train_df = self.yield_df.filter(~self.yield_df['FYEAR'].isin(test_years))\n",
        "    yield_test_df = self.yield_df.filter(self.yield_df['FYEAR'].isin(test_years))\n",
        "    yield_cols = yield_train_df.columns\n",
        "    pd_yield_train_df = yield_train_df.toPandas()\n",
        "    Y_train_full = pd_yield_train_df[yield_cols].values\n",
        "\n",
        "    custom_cv = self.trTsSplitter.customKFoldValidationSplit(Y_train_full, num_folds)\n",
        "\n",
        "  def runAllTests(self):\n",
        "    print('\\nTest Custom Train, Test Splitter BEGIN\\n')\n",
        "    self.testCustomTrainTestSplit()\n",
        "    self.testCustomKFoldValidationSplit()\n",
        "    print('\\nTest Custom Train, Test Splitter END\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5NrPZSmEViO5"
      },
      "source": [
        "## Run Workflow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DWnqPIU4cM_z"
      },
      "source": [
        "### Set Configuration\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C2r5-pmuVgS1",
        "outputId": "230f3a4d-96ee-4d3e-9594-06a5a7884a77"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "##################\n",
            "# Configuration  #\n",
            "##################\n",
            "\n",
            "Current ML Baseline Configuration\n",
            "--------------------------------\n",
            "Crop name: potatoes\n",
            "Crop ID: 7\n",
            "Crop growing season crosses calendar year boundary: N\n",
            "Country code (e.g. NL): HU\n",
            "NUTS level for yield prediction: NUTS3\n",
            "Input data sources: WOFOST, METEO_DAILY, REMOTE_SENSING, SOIL, YIELD\n",
            "Estimate and use yield trend: Y\n",
            "Predict yield residuals instead of full yield: N\n",
            "Find optimal trend window: N\n",
            "List of trend window lengths (number of years): 5, 7, 10\n",
            "Use centroid coordinates and distance to coast: N\n",
            "Use remote sensing data (FAPAR): Y\n",
            "Predict yield early in the season: Y\n",
            "Early season end dekad relative to harvest: -6\n",
            "Path to all input data. Default is current directory.: .\n",
            "Path to all output files. Default is current directory.: .\n",
            "Save features to a CSV file: N\n",
            "Use features from a CSV file: N\n",
            "Save predictions to a CSV file: N\n",
            "Use predictions from a CSV file: N\n",
            "Compare predictions with MARS Crop Yield Forecasting System: N\n",
            "Debug level to control amount of debug information: 2\n",
            "\n"
          ]
        }
      ],
      "source": [
        "if (test_env == 'notebook'):\n",
        "  cyp_config = CYPConfiguration()\n",
        "\n",
        "  if (run_tests):\n",
        "    test_util = TestUtil(spark)\n",
        "    test_util.runAllTests()\n",
        "\n",
        "  my_config = {\n",
        "      'crop_name' : 'potatoes',\n",
        "      'season_crosses_calendar_year' : 'N',\n",
        "      'country_code' : 'HU',\n",
        "      'data_sources' : [ 'WOFOST', 'METEO_DAILY', 'REMOTE_SENSING', 'SOIL', 'YIELD'],\n",
        "      'data_path' : '.',\n",
        "      'output_path' : '.',\n",
        "      'nuts_level' : 'NUTS3',\n",
        "      'use_yield_trend' : 'Y',\n",
        "      'predict_yield_residuals' : 'N',\n",
        "      'trend_windows' : [5, 7, 10],\n",
        "      'use_centroids' : 'N',\n",
        "      'use_remote_sensing' : 'Y',\n",
        "      'early_season_prediction' : 'Y',\n",
        "      'early_season_end_dekad' : -6,\n",
        "      'save_features' : 'N',\n",
        "      'use_saved_features' : 'N',\n",
        "      'save_predictions' : 'N',\n",
        "      'use_saved_predictions' : 'N',\n",
        "      'compare_with_mcyfs' : 'N',\n",
        "      'debug_level' : 2,\n",
        "  }\n",
        "\n",
        "  cyp_config.updateConfiguration(my_config)\n",
        "  crop = cyp_config.getCropName()\n",
        "  country = cyp_config.getCountryCode()\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  use_saved_predictions = cyp_config.useSavedPredictions()\n",
        "  use_saved_features = cyp_config.useSavedFeatures()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "\n",
        "  print('##################')\n",
        "  print('# Configuration  #')\n",
        "  print('##################')\n",
        "  output_path = cyp_config.getOutputPath()\n",
        "  log_file = getLogFilename(crop, country, use_yield_trend,\n",
        "                            early_season_prediction, early_season_end)\n",
        "  log_fh = open(output_path + '/' + log_file, 'w+')\n",
        "  cyp_config.printConfig(log_fh)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cytBAKTO2SfR"
      },
      "source": [
        "### Load and Preprocess Data\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qpkFFtVi2SfU",
        "outputId": "5f2ccda2-1e43-493c-f1b2-11be599e9248"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "#################\n",
            "# Data Loading  #\n",
            "#################\n",
            "Data file name \"./WOFOST_NUTS3_HU.csv\"\n",
            "Data file name \"./METEO_DAILY_NUTS3_HU.csv\"\n",
            "Data file name \"./REMOTE_SENSING_NUTS3_HU.csv\"\n",
            "Data file name \"./SOIL_NUTS3_HU.csv\"\n",
            "Data file name \"./YIELD_NUTS3_HU.csv\"\n",
            "Loaded data: WOFOST, METEO, REMOTE_SENSING, SOIL, YIELD\n",
            "\n",
            "\n",
            "#######################\n",
            "# Data Preprocessing  #\n",
            "#######################\n",
            "WOFOST data available for 20 region(s)\n",
            "Season end information\n",
            "+--------+-----+---------------+----------+\n",
            "|IDREGION|FYEAR|PREV_SEASON_END|SEASON_END|\n",
            "+--------+-----+---------------+----------+\n",
            "|   HU110| 1979|              0|        28|\n",
            "|   HU110| 1980|             28|        28|\n",
            "|   HU110| 1981|             28|        26|\n",
            "|   HU110| 1982|             26|        26|\n",
            "|   HU110| 1983|             26|        27|\n",
            "|   HU110| 1984|             27|        27|\n",
            "|   HU110| 1985|             27|        27|\n",
            "|   HU110| 1986|             27|        27|\n",
            "|   HU110| 1987|             27|        27|\n",
            "|   HU110| 1988|             27|        26|\n",
            "+--------+-----+---------------+----------+\n",
            "only showing top 10 rows\n",
            "\n",
            "WOFOST data\n",
            "+--------+-----+-----+-------------+--------------+------+------+-------+-------+----+----+---+---+---+---+\n",
            "|IDREGION|FYEAR|DEKAD|CAMPAIGN_YEAR|CAMPAIGN_DEKAD|POT_YB|POT_YS|WLIM_YB|WLIM_YS|PLAI|WLAI|DVS|RSM|TWC|TWR|\n",
            "+--------+-----+-----+-------------+--------------+------+------+-------+-------+----+----+---+---+---+---+\n",
            "|   HU110| 1979|    1|         1979|             9|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    2|         1979|            10|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    3|         1979|            11|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    4|         1979|            12|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    5|         1979|            13|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    6|         1979|            14|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    7|         1979|            15|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    8|         1979|            16|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|    9|         1979|            17|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "|   HU110| 1979|   10|         1979|            18|   0.0|   0.0|    0.0|    0.0| 0.0| 0.0|  0|0.0|0.0|0.0|\n",
            "+--------+-----+-----+-------------+--------------+------+------+-------+-------+----+----+---+---+---+---+\n",
            "only showing top 10 rows\n",
            "\n",
            "METEO data available for 20 region(s)\n",
            "METEO data\n",
            "+--------+-----+-----+-------------+--------------+----+-----+----+-----+----+----+----+----+-----+-----+\n",
            "|IDREGION|FYEAR|DEKAD|CAMPAIGN_YEAR|CAMPAIGN_DEKAD|TMAX| TMIN|TAVG|VPRES|WSPD|PREC| ET0| RAD| RELH|  CWB|\n",
            "+--------+-----+-----+-------------+--------------+----+-----+----+-----+----+----+----+----+-----+-----+\n",
            "|   HU110| 1979|    1|         1979|             9| 0.1| -4.9|-2.4| 5.26| 1.4| 6.2| 0.1|3323|100.0|  6.1|\n",
            "|   HU110| 1979|    1|         1979|             9|-4.9|-10.9|-7.9| 3.08| 2.1| 4.2|0.16|3652|91.54| 4.04|\n",
            "|   HU110| 1979|    1|         1979|             9|-4.9|-10.9|-7.9| 2.49| 2.1| 0.0|0.32|3547|74.02|-0.32|\n",
            "|   HU110| 1979|    1|         1979|             9|-5.9|-12.9|-9.4| 1.93| 9.1| 0.1|0.74|3748|64.57|-0.64|\n",
            "|   HU110| 1979|    1|         1979|             9|-3.9|-10.9|-7.4| 2.63| 1.9| 0.0|0.29|3908|75.23|-0.29|\n",
            "|   HU110| 1979|    1|         1979|             9|-4.9|-11.9|-8.4| 2.57| 2.4| 0.0|0.28|3945|79.43|-0.28|\n",
            "|   HU110| 1979|    1|         1979|             9|-5.9| -9.9|-7.9| 2.28| 6.6| 0.3|0.65|2734| 67.8|-0.35|\n",
            "|   HU110| 1979|    1|         1979|             9| 9.5|  3.0| 6.2| 6.75| 5.1| 0.4|1.09|3186|71.11|-0.69|\n",
            "|   HU110| 1979|    1|         1979|             9|-3.7| -8.3|-6.0| 2.66| 4.4| 0.9|0.64|2643|68.14| 0.26|\n",
            "|   HU110| 1979|    1|         1979|             9|-5.9|-11.9|-8.9| 2.33| 4.1| 0.0|0.41|3517|74.91|-0.41|\n",
            "+--------+-----+-----+-------------+--------------+----+-----+----+-----+----+----+----+----+-----+-----+\n",
            "only showing top 10 rows\n",
            "\n",
            "REMOTE_SENSING data available for 20 region(s)\n",
            "REMOTE_SENSING data\n",
            "+--------+-----+-----+-------------+--------------+-----+\n",
            "|IDREGION|FYEAR|DEKAD|CAMPAIGN_YEAR|CAMPAIGN_DEKAD|FAPAR|\n",
            "+--------+-----+-----+-------------+--------------+-----+\n",
            "|   HU110| 1999|    1|         1999|            11| 0.17|\n",
            "|   HU110| 1999|    3|         1999|            13|0.148|\n",
            "|   HU110| 1999|    4|         1999|            14|0.131|\n",
            "|   HU110| 1999|    5|         1999|            15|0.128|\n",
            "|   HU110| 1999|    6|         1999|            16|0.132|\n",
            "|   HU110| 1999|    7|         1999|            17|0.134|\n",
            "|   HU110| 1999|    8|         1999|            18|0.139|\n",
            "|   HU110| 1999|    9|         1999|            19|0.188|\n",
            "|   HU110| 1999|   10|         1999|            20|0.282|\n",
            "|   HU110| 1999|   11|         1999|            21|0.381|\n",
            "+--------+-----+-----+-------------+--------------+-----+\n",
            "only showing top 10 rows\n",
            "\n",
            "SOIL data available for 20 region(s)\n",
            "SOIL data\n",
            "+--------+------+\n",
            "|IDREGION|SM_WHC|\n",
            "+--------+------+\n",
            "|   HU110|  0.19|\n",
            "|   HU120|  0.21|\n",
            "|   HU211|  0.24|\n",
            "|   HU212|  0.24|\n",
            "|   HU213|  0.19|\n",
            "|   HU221|   0.2|\n",
            "|   HU222|  0.21|\n",
            "|   HU223|  0.23|\n",
            "|   HU231|  0.23|\n",
            "|   HU232|  0.22|\n",
            "+--------+------+\n",
            "only showing top 10 rows\n",
            "\n",
            "Yield before preprocessing\n",
            "+-----------+--------+-----+-----+\n",
            "|       CROP|IDREGION|FYEAR|YIELD|\n",
            "+-----------+--------+-----+-----+\n",
            "|Grain maize|   HU110| 2000|  2.8|\n",
            "|Grain maize|   HU110| 2001| 2.13|\n",
            "|Grain maize|   HU110| 2002| 5.01|\n",
            "|Grain maize|   HU110| 2003| 2.91|\n",
            "|Grain maize|   HU110| 2004| 5.97|\n",
            "|Grain maize|   HU110| 2005| 6.37|\n",
            "|Grain maize|   HU110| 2006| 6.45|\n",
            "|Grain maize|   HU110| 2007| 3.36|\n",
            "|Grain maize|   HU110| 2008| 7.23|\n",
            "|Grain maize|   HU110| 2009| 5.66|\n",
            "+-----------+--------+-----+-----+\n",
            "only showing top 10 rows\n",
            "\n",
            "Yield after preprocessing\n",
            "+--------+-----+-----+\n",
            "|IDREGION|FYEAR|YIELD|\n",
            "+--------+-----+-----+\n",
            "|   HU110| 2000|22.56|\n",
            "|   HU110| 2001|24.67|\n",
            "|   HU110| 2002|22.51|\n",
            "|   HU110| 2003|26.23|\n",
            "|   HU110| 2004|25.06|\n",
            "|   HU110| 2005|20.56|\n",
            "|   HU110| 2006|22.17|\n",
            "|   HU110| 2007|15.03|\n",
            "|   HU110| 2008|23.98|\n",
            "|   HU110| 2009|25.64|\n",
            "+--------+-----+-----+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "if ((test_env == 'notebook') and\n",
        "    (not use_saved_predictions) and\n",
        "    (not use_saved_features)):\n",
        "\n",
        "  print('#################')\n",
        "  print('# Data Loading  #')\n",
        "  print('#################')\n",
        "\n",
        "  if (run_tests):\n",
        "    test_loader = TestDataLoader(spark)\n",
        "    test_loader.runAllTests()\n",
        "\n",
        "  cyp_loader = CYPDataLoader(spark, cyp_config)\n",
        "  data_dfs = cyp_loader.loadAllData()\n",
        "\n",
        "  print('#######################')\n",
        "  print('# Data Preprocessing  #')\n",
        "  print('#######################')\n",
        "\n",
        "  if (run_tests):\n",
        "    test_preprocessor = TestDataPreprocessor(spark)\n",
        "    test_preprocessor.runAllTests()\n",
        "\n",
        "  cyp_preprocessor = CYPDataPreprocessor(spark, cyp_config)\n",
        "  data_dfs = preprocessData(cyp_config, cyp_preprocessor, data_dfs)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ePa1-zrp-Vay"
      },
      "source": [
        "### Split Data into Training and Test Sets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UtLALA3gtbiQ",
        "outputId": "29b2b759-ae16-4c02-b031-0f316e6a7327"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "###########################\n",
            "# Training and Test Split #\n",
            "###########################\n",
            "\n",
            "Test years: 2010, 2011, 2012, 2013, 2014, 2015\n",
            "\n"
          ]
        }
      ],
      "source": [
        "if ((test_env == 'notebook') and\n",
        "    (not use_saved_predictions) and\n",
        "    (not use_saved_features)):\n",
        "\n",
        "  print('###########################')\n",
        "  print('# Training and Test Split #')\n",
        "  print('###########################')\n",
        "\n",
        "  if (run_tests):\n",
        "    yield_df = data_dfs['YIELD']\n",
        "    test_custom = TestCustomTrainTestSplit(yield_df)\n",
        "    test_custom.runAllTests()\n",
        "\n",
        "  prep_train_test_dfs, test_years = splitDataIntoTrainingTestSets(cyp_config, data_dfs, log_fh)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b7w13GLYExv1"
      },
      "source": [
        "### Summarize Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HxYQ_BnPExv3",
        "outputId": "481448af-7a5a-4037-9c9b-f666fe9fd0be"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "#################\n",
            "# Data Summary  #\n",
            "#################\n",
            "Crop calender information based on WOFOST data\n",
            "+--------+-------------+---------+----------+----------+---------------------+\n",
            "|IDREGION|CAMPAIGN_YEAR|START_DVS|START_DVS1|START_DVS2|CAMPAIGN_EARLY_SEASON|\n",
            "+--------+-------------+---------+----------+----------+---------------------+\n",
            "|   HU110|         2018|       22|        25|        34|                   29|\n",
            "|   HU120|         2018|       21|        25|        34|                   29|\n",
            "|   HU211|         2018|       21|        25|        34|                   29|\n",
            "|   HU212|         2018|       22|        25|        34|                   29|\n",
            "|   HU213|         2018|       20|        24|        33|                   28|\n",
            "|   HU221|         2018|       21|        25|        34|                   29|\n",
            "|   HU222|         2018|       22|        26|        35|                   30|\n",
            "|   HU223|         2018|       22|        26|        35|                   30|\n",
            "|   HU231|         2018|       22|        25|        34|                   29|\n",
            "|   HU232|         2018|       23|        26|        34|                   29|\n",
            "+--------+-------------+---------+----------+----------+---------------------+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "if ((test_env == 'notebook') and\n",
        "    (not use_saved_predictions) and\n",
        "    (not use_saved_features)):\n",
        "\n",
        "  print('#################')\n",
        "  print('# Data Summary  #')\n",
        "  print('#################')\n",
        "\n",
        "  if (run_tests):\n",
        "    test_summarizer = TestDataSummarizer(spark)\n",
        "    test_summarizer.runAllTests()\n",
        "\n",
        "  cyp_summarizer = CYPDataSummarizer(cyp_config)\n",
        "  summary_dfs = summarizeData(cyp_config, cyp_summarizer, prep_train_test_dfs)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LvVoVMRgcejF"
      },
      "source": [
        "### Create Features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ucxoQlMXwKEg",
        "outputId": "9244c1d0-d469-49ab-92b9-de7eedbf20f0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "###################\n",
            "# Feature Design  #\n",
            "###################\n",
            "Yield min year 1996\n",
            "\n",
            " WOFOST Aggregate Features: Training\n",
            "  IDREGION  FYEAR  maxWLIM_YBp2  ...  maxWLAIp4  avgRSMp2  avgRSMp4\n",
            "0    HU110   1996       4788.35  ...       8.40     88.62     27.25\n",
            "1    HU110   1997       3045.29  ...       8.32     79.22     36.94\n",
            "2    HU110   1998       4210.65  ...       8.05     73.16     27.79\n",
            "3    HU110   1999       4602.54  ...       8.32     77.03     54.90\n",
            "4    HU110   2000       3674.77  ...       6.19     56.60     10.42\n",
            "\n",
            "[5 rows x 11 columns]\n",
            "\n",
            " WOFOST Aggregate Features: Test\n",
            "  IDREGION  FYEAR  maxWLIM_YBp2  ...  maxWLAIp4  avgRSMp2  avgRSMp4\n",
            "0    HU110   2010       4323.26  ...       8.76     96.69     78.94\n",
            "1    HU110   2011       4119.10  ...       7.40     68.52     22.31\n",
            "2    HU110   2012       2474.15  ...       7.06     59.04     24.25\n",
            "3    HU110   2013       3115.44  ...       9.29     81.56     40.05\n",
            "4    HU110   2014       4589.48  ...       8.23     82.81     27.18\n",
            "\n",
            "[5 rows x 11 columns]\n",
            "\n",
            " WOFOST Features for Extreme Conditions: Training\n",
            "  IDREGION  FYEAR  RSMp1gt1STD  ...  RSMp4gt1STD  RSMp4lt1STD  RSMp4gt2STD\n",
            "0    HU110   1996            0  ...            1            1            0\n",
            "1    HU110   1997            0  ...            1            0            0\n",
            "2    HU110   1998            0  ...            0            0            0\n",
            "3    HU110   1999            0  ...            2            0            1\n",
            "4    HU110   2000            0  ...            0            4            0\n",
            "\n",
            "[5 rows x 17 columns]\n",
            "\n",
            " WOFOST Features for Extreme Conditions: Test\n",
            "  IDREGION  FYEAR  RSMp1gt1STD  ...  RSMp4gt1STD  RSMp4lt1STD  RSMp4gt2STD\n",
            "0    HU110   2010            0  ...            4            0            4\n",
            "1    HU110   2011            0  ...            0            1            0\n",
            "2    HU110   2012            0  ...            0            0            0\n",
            "3    HU110   2013            0  ...            2            0            0\n",
            "4    HU110   2014            0  ...            1            1            0\n",
            "\n",
            "[5 rows x 17 columns]\n",
            "\n",
            " METEO Aggregate Features: Training\n",
            "  IDREGION  FYEAR  avgTAVGp0  ...  avgCWBp2  avgPRECp3  avgCWBp4\n",
            "0    HU321   1996       0.70  ...     -1.90       1.10     -3.55\n",
            "1    HU222   1999       6.08  ...     -1.66       2.56     -1.42\n",
            "2    HU233   1998       5.96  ...     -1.64       2.14     -1.86\n",
            "3    HU312   1998       6.07  ...     -1.32       1.14     -2.86\n",
            "4    HU212   2004       5.68  ...     -0.84       3.51     -2.93\n",
            "\n",
            "[5 rows x 11 columns]\n",
            "\n",
            " METEO Aggregate Features: Test\n",
            "  IDREGION  FYEAR  avgTAVGp0  ...  avgCWBp2  avgPRECp3  avgCWBp4\n",
            "0    HU212   2015       5.57  ...     -2.04       2.15     -4.39\n",
            "1    HU332   2011       1.97  ...     -2.46       1.35     -3.75\n",
            "2    HU313   2015       4.88  ...     -2.40       1.85     -4.25\n",
            "3    HU120   2012       3.24  ...     -2.78       1.70     -3.81\n",
            "4    HU323   2014       5.42  ...     -1.52       2.02     -3.01\n",
            "\n",
            "[5 rows x 11 columns]\n",
            "\n",
            " METEO Features for Extreme Conditions: Training\n",
            "  IDREGION  FYEAR  TMINp1gt1STD  ...  PRECp3gt2STD  TMAXp3gt2STD  TMAXp3lt2STD\n",
            "0    HU321   1996             5  ...             2             1             0\n",
            "1    HU222   1999             7  ...             4             0             1\n",
            "2    HU233   1998             2  ...             3             0             2\n",
            "3    HU312   1998             4  ...             0             0             2\n",
            "4    HU212   2004             8  ...             3             0             0\n",
            "\n",
            "[5 rows x 14 columns]\n",
            "\n",
            " METEO Features for Extreme Conditions: Test\n",
            "  IDREGION  FYEAR  TMINp1gt1STD  ...  PRECp3gt2STD  TMAXp3gt2STD  TMAXp3lt2STD\n",
            "0    HU212   2015             8  ...             3             0             3\n",
            "1    HU332   2011             1  ...             2             0             4\n",
            "2    HU313   2015             4  ...             1             0             0\n",
            "3    HU120   2012             1  ...             1             0             2\n",
            "4    HU323   2014             1  ...             2             0             2\n",
            "\n",
            "[5 rows x 14 columns]\n",
            "\n",
            " REMOTE_SENSING Aggregate Features: Training\n",
            "  IDREGION  FYEAR  avgFAPARp2  avgFAPARp4\n",
            "0    HU110   1999        0.51        0.61\n",
            "1    HU110   2000        0.48        0.45\n",
            "2    HU110   2001        0.45        0.52\n",
            "3    HU110   2002        0.46        0.51\n",
            "4    HU110   2003        0.38        0.45\n",
            "\n",
            " REMOTE_SENSING Aggregate Features: Test\n",
            "  IDREGION  FYEAR  avgFAPARp2  avgFAPARp4\n",
            "0    HU110   2010        0.57        0.57\n",
            "1    HU110   2011        0.55        0.53\n",
            "2    HU110   2012        0.41        0.49\n",
            "3    HU110   2013        0.49        0.57\n",
            "4    HU110   2014        0.67        0.49\n",
            "\n",
            "Yield Trend Features: Train\n",
            "  IDREGION  FYEAR    YIELD-5  ...    YIELD-2    YIELD-1  YIELD_TREND\n",
            "0    HU110   2005  22.559999  ...  26.230000  25.059999        26.17\n",
            "1    HU110   2006  24.670000  ...  25.059999  20.559999        22.10\n",
            "2    HU110   2007  22.510000  ...  20.559999  22.170000        21.40\n",
            "3    HU110   2008  26.230000  ...  22.170000  15.030000        14.22\n",
            "4    HU110   2009  25.059999  ...  15.030000  23.980000        19.05\n",
            "\n",
            "[5 rows x 8 columns]\n",
            "Total 172 rows\n",
            "\n",
            "Yield Trend Features: Test\n",
            "  IDREGION  FYEAR    YIELD-5  ...    YIELD-2    YIELD-1  YIELD_TREND\n",
            "5    HU110   2010  20.559999  ...  23.980000  25.639999        25.07\n",
            "6    HU110   2011  22.170000  ...  25.639999  14.770000        19.06\n",
            "7    HU110   2012  15.030000  ...  14.770000  25.750000        24.70\n",
            "8    HU110   2013  23.980000  ...  25.750000  20.850000        20.35\n",
            "9    HU110   2014  25.639999  ...  20.850000  18.330000        18.51\n",
            "\n",
            "[5 rows x 8 columns]\n",
            "Total 120 rows\n"
          ]
        }
      ],
      "source": [
        "if ((test_env == 'notebook') and\n",
        "    (not use_saved_predictions) and\n",
        "    (not use_saved_features)):\n",
        "\n",
        "  print('###################')\n",
        "  print('# Feature Design  #')\n",
        "  print('###################')\n",
        "\n",
        "  # WOFOST, Meteo and Remote Sensing Features\n",
        "  cyp_featurizer = CYPFeaturizer(cyp_config)\n",
        "  pd_feature_dfs = createFeatures(cyp_config, cyp_featurizer,\n",
        "                                  prep_train_test_dfs, summary_dfs, log_fh)\n",
        "\n",
        "  # yield trend features\n",
        "  if (use_yield_trend):\n",
        "    yield_train_df = prep_train_test_dfs['YIELD'][0]\n",
        "    yield_test_df = prep_train_test_dfs['YIELD'][1]\n",
        "\n",
        "    if (run_tests):\n",
        "      test_yield_trend = TestYieldTrendEstimator(yield_train_df)\n",
        "      test_yield_trend.runAllTests()\n",
        "\n",
        "    cyp_trend_est = CYPYieldTrendEstimator(cyp_config)\n",
        "    pd_yield_train_ft, pd_yield_test_ft = createYieldTrendFeatures(cyp_config, cyp_trend_est,\n",
        "                                                                   yield_train_df, yield_test_df,\n",
        "                                                                   test_years)\n",
        "    pd_feature_dfs['YIELD_TREND'] = [pd_yield_train_ft, pd_yield_test_ft]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EOy8NcAtfnhu"
      },
      "source": [
        "### Combine Features and Labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1brZTniBfpaO",
        "outputId": "4a252710-7922-4648-92dd-d0b9d6099956"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Combine Features and Labels\n",
            "---------------------------\n",
            "Yield min year 1996\n",
            "\n",
            "Data size after including SOIL data: \n",
            "Train 20 rows.\n",
            "Test 20 rows.\n",
            "\n",
            "Data size after including WOFOST features: \n",
            "Train 340 rows.\n",
            "Test 120 rows.\n",
            "\n",
            "Data size after including METEO features: \n",
            "Train 340 rows.\n",
            "Test 120 rows.\n",
            "\n",
            "Data size after including REMOTE_SENSING features: \n",
            "Train 280 rows.\n",
            "Test 120 rows.\n",
            "\n",
            "Data size after including yield trend features: \n",
            "Train 172 rows.\n",
            "Test 120 rows.\n",
            "\n",
            "Data size after including yield (label) data: \n",
            "Train 172 rows.\n",
            "Test 120 rows.\n",
            "\n",
            "\n",
            "All Features and labels: Training\n",
            "   IDREGION  FYEAR  SM_WHC  ...    YIELD-1  YIELD_TREND      YIELD\n",
            "32    HU110   2005    0.19  ...  25.059999        26.17  20.559999\n",
            "33    HU110   2006    0.19  ...  20.559999        22.10  22.170000\n",
            "34    HU110   2007    0.19  ...  22.170000        21.40  15.030000\n",
            "35    HU110   2008    0.19  ...  15.030000        14.22  23.980000\n",
            "36    HU110   2009    0.19  ...  23.980000        19.05  25.639999\n",
            "\n",
            "[5 rows x 57 columns]\n",
            "\n",
            "All Features and labels: Test\n",
            "   IDREGION  FYEAR  SM_WHC  ...    YIELD-1  YIELD_TREND      YIELD\n",
            "24    HU110   2010    0.19  ...  25.639999        25.07  14.770000\n",
            "25    HU110   2011    0.19  ...  14.770000        19.06  25.750000\n",
            "26    HU110   2012    0.19  ...  25.750000        24.70  20.850000\n",
            "27    HU110   2013    0.19  ...  20.850000        20.35  18.330000\n",
            "28    HU110   2014    0.19  ...  18.330000        18.51  22.809999\n",
            "\n",
            "[5 rows x 57 columns]\n"
          ]
        }
      ],
      "source": [
        "if ((test_env == 'notebook') and\n",
        "    (not use_saved_predictions) and\n",
        "    (not use_saved_features)):\n",
        "\n",
        "  join_cols = ['IDREGION', 'FYEAR']\n",
        "  pd_train_df, pd_test_df = combineFeaturesLabels(cyp_config, sqlContext,\n",
        "                                                  prep_train_test_dfs, pd_feature_dfs,\n",
        "                                                  join_cols, log_fh)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9lzdenOWYxYr"
      },
      "source": [
        "### Apply Machine Learning using scikit learn\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d1AKH4dbtR1d",
        "outputId": "c1c5adcd-5f66-462a-a356-dc883cafcc4f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "###################################\n",
            "# Machine Learning using sklearn  #\n",
            "###################################\n",
            "\n",
            "Training and Evaluation\n",
            "-------------------------\n",
            "\n",
            "Training Data Size: 172 rows\n",
            "X cols: 53, Y cols: 4\n",
            "IDREGION  FYEAR  SM_WHC  maxWLIM_YBp2  maxTWCp2  maxWLAIp2  maxWLIM_YBp4  maxWLIM_YSp4  maxTWCp4  maxWLAIp4  avgRSMp2  avgRSMp4  RSMp1gt1STD  RSMp1lt1STD  RSMp1gt2STD  RSMp1lt2STD  RSMp2gt1STD  RSMp2lt1STD  RSMp2gt2STD  RSMp2lt2STD  RSMp3gt1STD  RSMp3lt1STD  RSMp3gt2STD  RSMp3lt2STD  RSMp4gt1STD  RSMp4lt1STD  RSMp4gt2STD  avgTAVGp0  avgPRECp0  avgCWBp0  avgTAVGp1  avgPRECp1  avgTAVGp2  avgCWBp2  avgPRECp3  avgCWBp4  TMINp1gt1STD  PRECp1gt1STD  TMINp1lt1STD  TMINp1gt2STD  PRECp1gt2STD  TMINp1lt2STD  PRECp3gt1STD  TMAXp3gt1STD  TMAXp3lt1STD  PRECp3gt2STD  TMAXp3gt2STD  TMAXp3lt2STD  avgFAPARp2  avgFAPARp4    YIELD-5    YIELD-4    YIELD-3    YIELD-2    YIELD-1  YIELD_TREND      YIELD\n",
            "   HU110   2005    0.19       4177.50      9.66       7.20       8457.46       2291.72     19.95       7.46     78.76     36.82            0            0            0            0            1            1            0            0            2            0            1            0            1            0            0       3.11       1.25      0.00      11.88       2.30      15.07     -1.54       2.90     -2.74             3             6             5             0             4             0             5             4             9             2             0             3        0.50        0.55  22.559999  24.670000  22.510000  26.230000  25.059999        26.17  20.559999\n",
            "   HU110   2006    0.19       4606.44      8.95       8.27       9992.00       2447.24     23.08       9.52     77.61     47.15            0            0            0            0            1            0            0            0            2            0            0            0            2            0            2       2.52       1.29      0.18      13.19       1.05      15.39     -1.32       3.23     -1.66             6             1             3             0             1             0             6             0            13             4             0             2        0.56        0.59  24.670000  22.510000  26.230000  25.059999  20.559999        22.10  22.170000\n",
            "   HU110   2007    0.19       3877.32     10.11       6.39       5402.39        998.94     16.56       6.39     57.36     17.13            0            3            0            0            0            2            0            0            0            0            0            0            0            2            0       7.01       0.83     -0.73      13.54       0.14      17.03     -3.31       1.41     -4.03             2             0             2             0             0             0             4            14             2             1             0             0        0.51        0.43  22.510000  26.230000  25.059999  20.559999  22.170000        21.40  15.030000\n",
            "   HU110   2008    0.19       2609.31      5.29       5.00       7196.50       1572.41     17.65       7.46     82.28     37.07            0            0            0            0            2            0            0            0            1            0            0            0            1            0            0       3.68       0.77     -0.42       9.75       1.68      13.87     -1.85       1.35     -2.56             3             5            10             0             2             1             2             3             5             2             0             0        0.57        0.58  26.230000  25.059999  20.559999  22.170000  15.030000        14.22  23.980000\n",
            "   HU110   2009    0.19       2592.73      6.29       4.96       6351.60       2258.58     14.45       5.36     59.81     23.61            0            0            0            0            1            2            0            0            0            0            0            0            0            0            0       3.38       1.26      0.12      12.66       0.53      16.25     -3.92       0.72     -2.72             2             1             5             0             1             4             1             3             8             1             0             3        0.48        0.48  25.059999  20.559999  22.170000  15.030000  23.980000        19.05  25.639999\n",
            "\n",
            "Test Data Size: 120 rows\n",
            "X cols: 53, Y cols: 4\n",
            "IDREGION  FYEAR  SM_WHC  maxWLIM_YBp2  maxTWCp2  maxWLAIp2  maxWLIM_YBp4  maxWLIM_YSp4  maxTWCp4  maxWLAIp4  avgRSMp2  avgRSMp4  RSMp1gt1STD  RSMp1lt1STD  RSMp1gt2STD  RSMp1lt2STD  RSMp2gt1STD  RSMp2lt1STD  RSMp2gt2STD  RSMp2lt2STD  RSMp3gt1STD  RSMp3lt1STD  RSMp3gt2STD  RSMp3lt2STD  RSMp4gt1STD  RSMp4lt1STD  RSMp4gt2STD  avgTAVGp0  avgPRECp0  avgCWBp0  avgTAVGp1  avgPRECp1  avgTAVGp2  avgCWBp2  avgPRECp3  avgCWBp4  TMINp1gt1STD  PRECp1gt1STD  TMINp1lt1STD  TMINp1gt2STD  PRECp1gt2STD  TMINp1lt2STD  PRECp3gt1STD  TMAXp3gt1STD  TMAXp3lt1STD  PRECp3gt2STD  TMAXp3gt2STD  TMAXp3lt2STD  avgFAPARp2  avgFAPARp4    YIELD-5    YIELD-4    YIELD-3    YIELD-2    YIELD-1  YIELD_TREND      YIELD\n",
            "   HU110   2010    0.19       4323.26      7.46       8.10      11459.93       4225.52     24.26       8.76     96.69     78.94            0            1            0            0            3            0            1            0            3            0            2            0            4            0            4       3.47       1.70      0.52      11.86       1.77      14.80      1.25       6.28     -0.35             1             5             0             0             3             0            10             3            11            10             0             6        0.57        0.57  20.559999  22.170000  15.030000  23.980000  25.639999        25.07  14.770000\n",
            "   HU110   2011    0.19       4119.10      9.68       7.40       6855.11       1611.25     17.44       7.40     68.52     22.31            0            0            0            0            1            1            0            0            0            0            0            0            0            1            0       3.66       0.58     -0.76      13.24       0.41      15.33     -2.85       1.88     -3.38             3             0             1             0             0             0             4             4             2             1             0             0        0.55        0.53  22.170000  15.030000  23.980000  25.639999  14.770000        19.06  25.750000\n",
            "   HU110   2012    0.19       2474.15      5.50       5.36       6009.11       1021.44     15.57       7.06     59.04     24.25            0            3            0            0            0            1            0            0            0            0            0            0            0            0            0       3.36       0.80     -0.56      11.25       0.55      14.26     -2.74       1.25     -3.28             1             2             7             0             0             2             3             4             7             1             0             1        0.41        0.49  15.030000  23.980000  25.639999  14.770000  25.750000        24.70  20.850000\n",
            "   HU110   2013    0.19       3115.44      6.46       6.59       8709.04       1591.33     18.42       9.29     81.56     40.05            0            0            0            0            2            0            0            0            1            0            0            0            2            0            0       2.20       2.41      1.68       6.84       2.95      14.86     -1.81       2.44     -2.27             0             8            13             0             4             6             4             1            11             3             0             1        0.49        0.57  23.980000  25.639999  14.770000  25.750000  20.850000        20.35  18.330000\n",
            "   HU110   2014    0.19       4589.48      9.47       8.23       7141.74       1349.68     17.81       8.23     82.81     27.18            0            0            0            0            1            0            0            0            1            1            1            0            1            1            0       7.02       0.78     -0.68      13.46       1.03      15.13     -1.02       2.46     -3.54             8             1             1             0             1             0             2             6            10             2             1             2        0.67        0.49  25.639999  14.770000  25.750000  20.850000  18.330000        18.51  22.809999\n",
            "\n",
            "All features\n",
            "-------------\n",
            "\n",
            "1: SM_WHC, 2: maxWLIM_YBp2, 3: maxTWCp2, 4: maxWLAIp2, 5: maxWLIM_YBp4\n",
            "6: maxWLIM_YSp4, 7: maxTWCp4, 8: maxWLAIp4, 9: avgRSMp2, 10: avgRSMp4\n",
            "11: RSMp1gt1STD, 12: RSMp1lt1STD, 13: RSMp1gt2STD, 14: RSMp1lt2STD, 15: RSMp2gt1STD\n",
            "16: RSMp2lt1STD, 17: RSMp2gt2STD, 18: RSMp2lt2STD, 19: RSMp3gt1STD, 20: RSMp3lt1STD\n",
            "21: RSMp3gt2STD, 22: RSMp3lt2STD, 23: RSMp4gt1STD, 24: RSMp4lt1STD, 25: RSMp4gt2STD\n",
            "26: avgTAVGp0, 27: avgPRECp0, 28: avgCWBp0, 29: avgTAVGp1, 30: avgPRECp1\n",
            "31: avgTAVGp2, 32: avgCWBp2, 33: avgPRECp3, 34: avgCWBp4, 35: TMINp1gt1STD\n",
            "36: PRECp1gt1STD, 37: TMINp1lt1STD, 38: TMINp1gt2STD, 39: PRECp1gt2STD, 40: TMINp1lt2STD\n",
            "41: PRECp3gt1STD, 42: TMAXp3gt1STD, 43: TMAXp3lt1STD, 44: PRECp3gt2STD, 45: TMAXp3gt2STD\n",
            "46: TMAXp3lt2STD, 47: avgFAPARp2, 48: avgFAPARp4, 49: YIELD-5, 50: YIELD-4\n",
            "51: YIELD-3, 52: YIELD-2, 53: YIELD-1\n",
            "\n",
            "\n",
            "Custom sliding validation train, test splits\n",
            "----------------------------------------------\n",
            "Validation set 1 training years: 2001, 2002, 2003, 2004\n",
            "Validation set 1 test years: 2005\n",
            "Validation set 2 training years: 2002, 2003, 2004, 2005\n",
            "Validation set 2 test years: 2006\n",
            "Validation set 3 training years: 2003, 2004, 2005, 2006\n",
            "Validation set 3 test years: 2007\n",
            "Validation set 4 training years: 2004, 2005, 2006, 2007\n",
            "Validation set 4 test years: 2008\n",
            "Validation set 5 training years: 2005, 2006, 2007, 2008\n",
            "Validation set 5 test years: 2009\n",
            "\n",
            "\n",
            "Null Method: Predicting linear yield trend:\n",
            "Min Yield: 6.64, Max Yield: 37.82\n",
            "Median Yield: 20.31, Mean Yield: 20.21\n",
            "\n",
            "Yield Predictions Training Set\n",
            "--------------------------------\n",
            "IDREGION FYEAR YIELD_TREND  YIELD\n",
            "   HU110  2005       26.17  20.56\n",
            "   HU110  2006        22.1  22.17\n",
            "   HU110  2007        21.4  15.03\n",
            "   HU110  2008       14.22  23.98\n",
            "   HU110  2009       19.05  25.64\n",
            "   HU120  2005       19.65  18.06\n",
            "\n",
            "Yield Predictions Test Set\n",
            "--------------------------------\n",
            "IDREGION FYEAR YIELD_TREND  YIELD\n",
            "   HU110  2010       25.07  14.77\n",
            "   HU110  2011       19.06  25.75\n",
            "   HU110  2012        24.7  20.85\n",
            "   HU110  2013       20.35  18.33\n",
            "   HU110  2014       18.51  22.81\n",
            "   HU110  2015        23.1  28.29\n",
            "\n",
            "Estimator: Ridge\n",
            "---------------------------\n",
            "\n",
            "Feature Selection Summary\n",
            "---------------------------\n",
            "estimator       selector  neg_mean_squared_error\n",
            "    Ridge  random_forest                  -24.46\n",
            "    Ridge      RFE_Lasso                  -45.72\n",
            "    Ridge       combined                  -24.31\n",
            "\n",
            "\n",
            "Selected Features:\n",
            "-------------------\n",
            "\n",
            "5: maxWLIM_YBp4, 6: maxWLIM_YSp4, 7: maxTWCp4, 10: avgRSMp4, 24: RSMp4lt1STD\n",
            "28: avgCWBp0, 30: avgPRECp1, 31: avgTAVGp2, 32: avgCWBp2, 34: avgCWBp4\n",
            "35: TMINp1gt1STD, 36: PRECp1gt1STD, 42: TMAXp3gt1STD, 43: TMAXp3lt1STD, 48: avgFAPARp4\n",
            "49: YIELD-5, 50: YIELD-4, 51: YIELD-3, 52: YIELD-2, 53: YIELD-1\n",
            "\n",
            "\n",
            "\n",
            "Estimator Ridge\n",
            "---------------------------\n",
            "Ridge(alpha=1, max_iter=1000, random_state=42)\n",
            "estimator__alpha= 10\n",
            "\n",
            "Estimator: KNN\n",
            "---------------------------\n",
            "\n",
            "Feature Selection Summary\n",
            "---------------------------\n",
            "estimator       selector  neg_mean_squared_error\n",
            "      KNN  random_forest                  -20.29\n",
            "      KNN      RFE_Lasso                  -26.28\n",
            "      KNN       combined                  -21.94\n",
            "\n",
            "\n",
            "Selected Features:\n",
            "-------------------\n",
            "\n",
            "1: SM_WHC, 5: maxWLIM_YBp4, 6: maxWLIM_YSp4, 7: maxTWCp4, 10: avgRSMp4\n",
            "27: avgPRECp0, 28: avgCWBp0, 30: avgPRECp1, 31: avgTAVGp2, 32: avgCWBp2\n",
            "34: avgCWBp4, 42: TMAXp3gt1STD, 43: TMAXp3lt1STD, 47: avgFAPARp2, 48: avgFAPARp4\n",
            "49: YIELD-5, 50: YIELD-4, 51: YIELD-3, 52: YIELD-2, 53: YIELD-1\n",
            "\n",
            "\n",
            "\n",
            "Estimator KNN\n",
            "---------------------------\n",
            "KNeighborsRegressor(weights='distance')\n",
            "estimator__n_neighbors= 9\n",
            "\n",
            "Estimator: SVR\n",
            "---------------------------\n",
            "\n",
            "Feature Selection Summary\n",
            "---------------------------\n",
            "estimator       selector  neg_mean_squared_error\n",
            "      SVR  random_forest                  -23.52\n",
            "      SVR      RFE_Lasso                  -28.07\n",
            "      SVR       combined                  -22.97\n",
            "\n",
            "\n",
            "Selected Features:\n",
            "-------------------\n",
            "\n",
            "1: SM_WHC, 5: maxWLIM_YBp4, 6: maxWLIM_YSp4, 7: maxTWCp4, 10: avgRSMp4\n",
            "14: RSMp1lt2STD, 15: RSMp2gt1STD, 16: RSMp2lt1STD, 18: RSMp2lt2STD, 23: RSMp4gt1STD\n",
            "24: RSMp4lt1STD, 25: RSMp4gt2STD, 27: avgPRECp0, 28: avgCWBp0, 30: avgPRECp1\n",
            "31: avgTAVGp2, 32: avgCWBp2, 33: avgPRECp3, 34: avgCWBp4, 35: TMINp1gt1STD\n",
            "36: PRECp1gt1STD, 40: TMINp1lt2STD, 42: TMAXp3gt1STD, 43: TMAXp3lt1STD, 44: PRECp3gt2STD\n",
            "45: TMAXp3gt2STD, 46: TMAXp3lt2STD, 47: avgFAPARp2, 48: avgFAPARp4, 49: YIELD-5\n",
            "50: YIELD-4, 51: YIELD-3, 52: YIELD-2, 53: YIELD-1\n",
            "\n",
            "\n",
            "Estimator SVR\n",
            "---------------------------\n",
            "SVR()\n",
            "estimator__C= 10.0\n",
            "estimator__epsilon= 0.5\n",
            "\n",
            "Estimator: GBDT\n",
            "---------------------------\n",
            "\n",
            "Feature Selection Summary\n",
            "---------------------------\n",
            "estimator       selector  neg_mean_squared_error\n",
            "     GBDT  random_forest                  -29.32\n",
            "     GBDT      RFE_Lasso                  -29.68\n",
            "     GBDT       combined                  -29.19\n",
            "\n",
            "\n",
            "Selected Features:\n",
            "-------------------\n",
            "\n",
            "1: SM_WHC, 5: maxWLIM_YBp4, 6: maxWLIM_YSp4, 7: maxTWCp4, 10: avgRSMp4\n",
            "24: RSMp4lt1STD, 27: avgPRECp0, 28: avgCWBp0, 30: avgPRECp1, 31: avgTAVGp2\n",
            "32: avgCWBp2, 34: avgCWBp4, 35: TMINp1gt1STD, 36: PRECp1gt1STD, 42: TMAXp3gt1STD\n",
            "43: TMAXp3lt1STD, 47: avgFAPARp2, 48: avgFAPARp4, 49: YIELD-5, 50: YIELD-4\n",
            "51: YIELD-3, 52: YIELD-2, 53: YIELD-1\n",
            "\n",
            "\n",
            "Estimator GBDT\n",
            "---------------------------\n",
            "GradientBoostingRegressor(learning_rate=0.01, loss='lad', min_samples_leaf=5,\n",
            "                          random_state=42, subsample=0.8)\n",
            "estimator__max_depth= 10\n",
            "estimator__n_estimators= 500\n",
            "\n",
            "Feature Selection Frequencies\n",
            "-------------------------------\n",
            "static: YIELD-5(4), YIELD-4(4), YIELD-3(4), YIELD-2(4), YIELD-1(4), SM_WHC(3)\n",
            "p0: avgCWBp0(4), avgPRECp0(3)\n",
            "p1: avgPRECp1(4), TMINp1gt1STD(3), PRECp1gt1STD(3), RSMp1lt2STD(1), TMINp1lt2STD(1)\n",
            "p2: avgTAVGp2(4), avgCWBp2(4), avgFAPARp2(3), RSMp2gt1STD(1), RSMp2lt1STD(1), RSMp2lt2STD(1)\n",
            "p3: TMAXp3gt1STD(4), TMAXp3lt1STD(4), avgPRECp3(1), PRECp3gt2STD(1), TMAXp3gt2STD(1), TMAXp3lt2STD(1)\n",
            "p4: maxWLIM_YBp4(4), maxWLIM_YSp4(4), maxTWCp4(4), avgRSMp4(4), avgCWBp4(4), avgFAPARp4(4), RSMp4lt1STD(3), RSMp4gt1STD(1), RSMp4gt2STD(1)\n",
            "p5: \n",
            "\n",
            "\n",
            "Yield Predictions Training Set\n",
            "--------------------------------\n",
            "IDREGION FYEAR YIELD_TREND YIELD_PRED_Ridge YIELD_PRED_KNN YIELD_PRED_SVR YIELD_PRED_GBDT  YIELD\n",
            "   HU110  2005       26.17          27.6972          20.56        28.5976         21.8934  20.56\n",
            "   HU110  2006        22.1          25.5551          22.17        22.6702         22.1984  22.17\n",
            "   HU110  2007        21.4          16.4229          15.03        15.6268         15.9651  15.03\n",
            "   HU110  2008       14.22          23.8296          23.98         24.154         23.9838  23.98\n",
            "   HU110  2009       19.05          20.5709          25.64         23.092         21.5905  25.64\n",
            "   HU120  2005       19.65          22.1776          18.06          18.56         18.2004  18.06\n",
            "\n",
            "Yield Predictions Test Set\n",
            "--------------------------------\n",
            "IDREGION FYEAR YIELD_TREND YIELD_PRED_Ridge YIELD_PRED_KNN YIELD_PRED_SVR YIELD_PRED_GBDT  YIELD\n",
            "   HU110  2010       25.07          26.1117        24.9143        21.4269         24.3583  14.77\n",
            "   HU110  2011       19.06          18.7321        19.2057        15.0286         16.5418  25.75\n",
            "   HU110  2012        24.7          21.0205        18.2469        21.4353         18.6543  20.85\n",
            "   HU110  2013       20.35          27.4015        24.4684        23.2222         23.3411  18.33\n",
            "   HU110  2014       18.51          20.7417             20        19.2474         19.4794  22.81\n",
            "   HU110  2015        23.1          16.4589        16.6393        15.9032         16.9212  28.29\n",
            "\n",
            "Algorithm Evaluation Summary\n",
            "-----------------------------\n",
            "algorithm  train_MAE  test_MAE  train_MAPE  test_MAPE  train_RMSE  test_RMSE  train_R2  test_R2\n",
            "    trend      24.01     20.89       26.58      24.43       30.29      25.87     -0.05     0.50\n",
            "    Ridge      14.25     23.33       16.51      29.03       17.81      29.21      0.64     0.37\n",
            "      KNN       0.00     25.80        0.00      32.46        0.00      30.82      1.00     0.29\n",
            "      SVR       6.42     25.60        7.60      32.11        9.90      31.40      0.89     0.27\n",
            "     GBDT       4.72     23.71        5.50      30.28        8.91      29.01      0.91     0.38\n",
            "\n"
          ]
        }
      ],
      "source": [
        "if ((test_env == 'notebook') and\n",
        "    (not use_saved_predictions)):\n",
        "\n",
        "  if (use_saved_features):\n",
        "    pd_train_df, pd_test_df = loadSavedFeaturesLabels(cyp_config, spark)\n",
        "\n",
        "  print('\\n###################################')\n",
        "  print('# Machine Learning using sklearn  #')\n",
        "  print('###################################')\n",
        "\n",
        "  pd_ml_predictions = getMachineLearningPredictions(cyp_config, pd_train_df, pd_test_df, log_fh)\n",
        "  save_predictions = cyp_config.savePredictions()\n",
        "  if (save_predictions):\n",
        "    saveMLPredictions(cyp_config, sqlContext, pd_ml_predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ivn9xbXlPs4d"
      },
      "source": [
        "### Compare Predictions with JRC Predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pyzPXCxEtpC1"
      },
      "outputs": [],
      "source": [
        "if (test_env == 'notebook'):\n",
        "  if (use_saved_predictions):\n",
        "    pd_ml_predictions = loadSavedPredictions(cyp_config, spark)\n",
        "\n",
        "  compareWithMCYFS = cyp_config.compareWithMCYFS()\n",
        "  if (compareWithMCYFS):\n",
        "    comparePredictionsWithMCYFS(sqlContext, cyp_config, pd_ml_predictions, log_fh)\n",
        "\n",
        "  log_fh.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KZ1Ya_s0bVbZ"
      },
      "source": [
        "### Python Script Main\n",
        "\n",
        "To be used in environment supporting command-line arguments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "Wo3wAY6AWdId",
        "outputId": "0755a758-a789-49ee-f632-849cbbc97c5c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Writing main.py\n"
          ]
        }
      ],
      "source": [
        "#%%writefile main.py\n",
        "import sys\n",
        "import argparse\n",
        "\n",
        "def main():\n",
        "  print('##################')\n",
        "  print('# Configuration  #')\n",
        "  print('##################')\n",
        "\n",
        "  parser = argparse.ArgumentParser(prog='mlbaseline.py')\n",
        "\n",
        "  # Some command-line argument names are slightly different\n",
        "  # from configuration option names for brevity.\n",
        "  args_dict = {\n",
        "      '--crop' : { 'type' : str,\n",
        "                   'default' : 'potatoes',\n",
        "                   'help' : 'crop name (default: potatoes)',\n",
        "                 },\n",
        "      '--crosses-calendar-year' : { 'type' : str,\n",
        "                                    'default' : 'N',\n",
        "                                    'choices' : ['Y', 'N'],\n",
        "                                    'help' : 'crop growing season crosses calendar year boundary (default: N)',\n",
        "                                  },\n",
        "      '--country' : { 'type' : str,\n",
        "                      'default' : 'NL',\n",
        "                      'choices' : ['BG', 'DE', 'ES', 'FR', 'HU', 'IT', 'NL', 'PL', 'RO'],\n",
        "                      'help' : 'country code (default: NL)',\n",
        "                    },\n",
        "      '--nuts-level' : { 'type' : str,\n",
        "                         'default' : 'NUTS2',\n",
        "                         'choices' : ['NUTS2', 'NUTS3'],\n",
        "                         'help' : 'country code (default: NL)',\n",
        "                       },\n",
        "      '--data-path' : { 'type' : str,\n",
        "                        'default' : '.',\n",
        "                        'help' : 'path to data files (default: .)',\n",
        "                       },\n",
        "      '--output-path' : { 'type' : str,\n",
        "                          'default' : '.',\n",
        "                          'help' : 'path to output files (default: .)',\n",
        "                        },\n",
        "      '--yield-trend' : { 'type' : str,\n",
        "                          'default' : 'N',\n",
        "                          'choices' : ['Y', 'N'],\n",
        "                          'help' : 'estimate and use yield trend (default: N)',\n",
        "                        },\n",
        "      '--optimal-trend-window' : { 'type' : str,\n",
        "                                   'default' : 'N',\n",
        "                                   'choices' : ['Y', 'N'],\n",
        "                                   'help' : 'find optimal trend window for each year (default: N)',\n",
        "                                 },\n",
        "      '--predict-residuals' : { 'type' : str,\n",
        "                                'default' : 'N',\n",
        "                                'choices' : ['Y', 'N'],\n",
        "                                'help' : 'predict yield residuals instead of full yield (default: N)',\n",
        "                              },\n",
        "      '--early-season' : { 'type' : str,\n",
        "                           'default' : 'N',\n",
        "                           'choices' : ['Y', 'N'],\n",
        "                           'help' : 'early season prediction (default: N)',\n",
        "                         },\n",
        "      '--early-season-end' : { 'type' : int,\n",
        "                               'default' : 15,\n",
        "                               'help' : 'early season end dekad (default: 15)',\n",
        "                             },\n",
        "      '--centroids' : { 'type' : str,\n",
        "                        'default' : 'N',\n",
        "                        'choices' : ['Y', 'N'],\n",
        "                        'help' : 'use centroid coordinates and distance to coast (default: N)',\n",
        "                      },\n",
        "      '--remote-sensing' : { 'type' : str,\n",
        "                             'default' : 'Y',\n",
        "                             'choices' : ['Y', 'N'],\n",
        "                             'help' : 'use remote sensing data (default: Y)',\n",
        "                           },\n",
        "      '--save-features' : { 'type' : str,\n",
        "                            'default' : 'N',\n",
        "                            'choices' : ['Y', 'N'],\n",
        "                            'help' : 'save features to a CSV file (default: N)',\n",
        "                          },\n",
        "      '--use-saved-features' : { 'type' : str,\n",
        "                                 'default' : 'N',\n",
        "                                 'choices' : ['Y', 'N'],\n",
        "                                 'help' : 'use features from a CSV file (default: N). Set ',\n",
        "                               },\n",
        "      '--save-predictions' : { 'type' : str,\n",
        "                               'default' : 'Y',\n",
        "                               'choices' : ['Y', 'N'],\n",
        "                               'help' : 'save predictions to a CSV file (default: Y)',\n",
        "                             },\n",
        "      '--use-saved-predictions' : { 'type' : str,\n",
        "                                    'default' : 'N',\n",
        "                                    'choices' : ['Y', 'N'],\n",
        "                                    'help' : 'use predictions from a CSV file (default: N)',\n",
        "                                  },\n",
        "      '--compare-with-mcyfs' : { 'type' : str,\n",
        "                                 'default' : 'N',\n",
        "                                 'choices' : ['Y', 'N'],\n",
        "                                 'help' : 'compare predictions with MCYFS (default: N)',\n",
        "                               },\n",
        "      '--debug-level' : { 'type' : int,\n",
        "                          'default' : 0,\n",
        "                          'choices' : range(4),\n",
        "                          'help' : 'amount of debug information to print (default: 0)',\n",
        "                        },\n",
        "  }\n",
        "\n",
        "  for arg in args_dict:\n",
        "    arg_config = args_dict[arg]\n",
        "    # add cases if other argument settings are used\n",
        "    if ('choices' in arg_config):\n",
        "      parser.add_argument(arg, type=arg_config['type'], default=arg_config['default'],\n",
        "                          choices=arg_config['choices'], help=arg_config['help'])\n",
        "    else:\n",
        "      parser.add_argument(arg, type=arg_config['type'], default=arg_config['default'],\n",
        "                          help=arg_config['help'])\n",
        "\n",
        "  if (run_tests):\n",
        "    test_util = TestUtil(spark)\n",
        "    test_util.runAllTests()\n",
        "\n",
        "  args = parser.parse_args()\n",
        "  cyp_config = CYPConfiguration()\n",
        "\n",
        "  # must be in sync with args_dict used to parse args\n",
        "  config_update = {\n",
        "      'crop_name' : args.crop,\n",
        "      'season_crosses_calendar_year' : args.crosses_calendar_year,\n",
        "      'country_code' : args.country,\n",
        "      'nuts_level' : args.nuts_level,\n",
        "      'data_path' : args.data_path,\n",
        "      'output_path' : args.output_path,\n",
        "      'use_yield_trend' : args.yield_trend,\n",
        "      'find_optimal_trend_window' : args.optimal_trend_window,\n",
        "      'predict_yield_residuals' : args.predict_residuals,\n",
        "      'use_centroids' : args.centroids,\n",
        "      'use_remote_sensing' : args.remote_sensing,\n",
        "      'early_season_prediction' : args.early_season,\n",
        "      'early_season_end_dekad' : args.early_season_end,\n",
        "      'save_features' : args.save_features,\n",
        "      'use_saved_features' : args.use_saved_features,\n",
        "      'save_predictions' : args.save_predictions,\n",
        "      'use_saved_predictions' : args.use_saved_predictions,\n",
        "      'compare_with_mcyfs' : args.compare_with_mcyfs,\n",
        "      'debug_level' : args.debug_level,\n",
        "  }\n",
        "\n",
        "  cyp_config.updateConfiguration(config_update)\n",
        "  crop = cyp_config.getCropName()\n",
        "  country = cyp_config.getCountryCode()\n",
        "  nuts_level = cyp_config.getNUTSLevel()\n",
        "  debug_level = cyp_config.getDebugLevel()\n",
        "  use_saved_predictions = cyp_config.useSavedPredictions()\n",
        "  use_saved_features = cyp_config.useSavedFeatures()\n",
        "  use_yield_trend = cyp_config.useYieldTrend()\n",
        "  early_season_prediction = cyp_config.earlySeasonPrediction()\n",
        "  early_season_end = cyp_config.getEarlySeasonEndDekad()\n",
        "\n",
        "  output_path = cyp_config.getOutputPath()\n",
        "  log_file = getLogFilename(crop, country, use_yield_trend,\n",
        "                            early_season_prediction, early_season_end)\n",
        "  log_fh = open(output_path + '/' + log_file, 'w+')\n",
        "  cyp_config.printConfig(log_fh)\n",
        "\n",
        "  if (not use_saved_predictions):\n",
        "    if (not use_saved_features):\n",
        "      print('#################')\n",
        "      print('# Data Loading  #')\n",
        "      print('#################')\n",
        "\n",
        "      if (run_tests):\n",
        "        test_loader = TestDataLoader(spark)\n",
        "        test_loader.runAllTests()\n",
        "\n",
        "      cyp_loader = CYPDataLoader(spark, cyp_config)\n",
        "      data_dfs = cyp_loader.loadAllData()\n",
        "\n",
        "      print('#######################')\n",
        "      print('# Data Preprocessing  #')\n",
        "      print('#######################')\n",
        "\n",
        "      if (run_tests):\n",
        "        test_preprocessor = TestDataPreprocessor(spark)\n",
        "        test_preprocessor.runAllTests()\n",
        "\n",
        "      cyp_preprocessor = CYPDataPreprocessor(spark, cyp_config)\n",
        "      data_dfs = preprocessData(cyp_config, cyp_preprocessor, data_dfs)\n",
        "\n",
        "      print('###########################')\n",
        "      print('# Training and Test Split #')\n",
        "      print('###########################')\n",
        "\n",
        "      if (run_tests):\n",
        "        yield_df = data_dfs['YIELD']\n",
        "        test_custom = TestCustomTrainTestSplit(yield_df)\n",
        "        test_custom.runAllTests()\n",
        "\n",
        "      prep_train_test_dfs, test_years = splitDataIntoTrainingTestSets(cyp_config, data_dfs, log_fh)\n",
        "\n",
        "      print('#################')\n",
        "      print('# Data Summary  #')\n",
        "      print('#################')\n",
        "\n",
        "      if (run_tests):\n",
        "        test_summarizer = TestDataSummarizer(spark)\n",
        "        test_summarizer.runAllTests()\n",
        "\n",
        "      cyp_summarizer = CYPDataSummarizer(cyp_config)\n",
        "      summary_dfs = summarizeData(cyp_config, cyp_summarizer, prep_train_test_dfs)\n",
        "\n",
        "      print('###################')\n",
        "      print('# Feature Design  #')\n",
        "      print('###################')\n",
        "\n",
        "      # WOFOST, Meteo and Remote Sensing Features\n",
        "      cyp_featurizer = CYPFeaturizer(cyp_config)\n",
        "      pd_feature_dfs = createFeatures(cyp_config, cyp_featurizer,\n",
        "                                      prep_train_test_dfs, summary_dfs, log_fh)\n",
        "\n",
        "      # yield trend features\n",
        "      if (use_yield_trend):\n",
        "        yield_train_df = prep_train_test_dfs['YIELD'][0]\n",
        "        yield_test_df = prep_train_test_dfs['YIELD'][1]\n",
        "\n",
        "        if (run_tests):\n",
        "          test_yield_trend = TestYieldTrendEstimator(yield_train_df)\n",
        "          test_yield_trend.runAllTests()\n",
        "\n",
        "        cyp_trend_est = CYPYieldTrendEstimator(cyp_config)\n",
        "        pd_yield_train_ft, pd_yield_test_ft = createYieldTrendFeatures(cyp_config, cyp_trend_est,\n",
        "                                                                       yield_train_df, yield_test_df,\n",
        "                                                                       test_years)\n",
        "        pd_feature_dfs['YIELD_TREND'] = [pd_yield_train_ft, pd_yield_test_ft]\n",
        "\n",
        "      # combine features\n",
        "      join_cols = ['IDREGION', 'FYEAR']\n",
        "      pd_train_df, pd_test_df = combineFeaturesLabels(cyp_config, sqlContext,\n",
        "                                                      prep_train_test_dfs, pd_feature_dfs,\n",
        "                                                      join_cols, log_fh)\n",
        "\n",
        "    # use saved features\n",
        "    else:\n",
        "      pd_train_df, pd_test_df = loadSavedFeaturesLabels(cyp_config, spark)\n",
        "\n",
        "    print('###################################')\n",
        "    print('# Machine Learning using sklearn  #')\n",
        "    print('###################################')\n",
        "\n",
        "    pd_ml_predictions = getMachineLearningPredictions(cyp_config, pd_train_df, pd_test_df, log_fh)\n",
        "    save_predictions = cyp_config.savePredictions()\n",
        "    if (save_predictions):\n",
        "      saveMLPredictions(cyp_config, sqlContext, pd_ml_predictions)\n",
        "\n",
        "  # use saved predictions\n",
        "  else:\n",
        "    pd_ml_predictions = loadSavedPredictions(cyp_config, spark)\n",
        "\n",
        "  # compare with MCYFS\n",
        "  compareWithMCYFS = cyp_config.compareWithMCYFS()\n",
        "  if (compareWithMCYFS):\n",
        "    comparePredictionsWithMCYFS(sqlContext, cyp_config, pd_ml_predictions, log_fh)\n",
        "\n",
        "  log_fh.close()\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    main()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "KZ1Ya_s0bVbZ"
      ],
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}