submit clean nn_cv_ensemble.ipynb

{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport seaborn as sns\nimport datetime\nimport tqdm\nimport re\nimport time\nfrom string import punctuation\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\n\nimport torch, torch.nn as nn\nimport torch.nn.functional as F\n\nsns.set_style('darkgrid')\nmpl.rcParams['figure.figsize'] = [15,10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from kaggle.competitions import nflrush","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom string import punctuation\nimport datetime\nimport re\nimport torch\nimport math\nimport tqdm\n\n\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024 ** 2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(\n                    np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(\n                    np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(\n                    np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(\n                    np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(\n                    np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(\n                    np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024 ** 2\n    if verbose:\n        print(f'Mem. usage decreased to {end_mem} Mb ({100 * (start_mem - end_mem) / start_mem}% reduction)')\n    return df\n\n\ndef crps(y_true, y_pred):\n    return np.mean(np.square(y_true - y_pred), axis=1)\n\n\ndef yard_to_cdf(yard):\n    yard = np.round(yard).astype(int)\n    indices = yard + 99\n    cdfs = np.zeros((yard.shape[0], 199))\n    for i in range(len(cdfs)):\n        cdfs[i, indices[i]:] = 1\n    return cdfs\n\n\ndef cdf_to_yard(cdf):\n    yard_index = (cdf == 1).argmax(axis=1)\n    yard = yard_index - 99\n    return yard\n\n\ndef cdf_to_yard_torch(cdf):\n    yard_index = torch.sum((torch.as_tensor(cdf) <= 0), dim=1)\n    yard = yard_index - 99\n    return yard\n\n\ndef crps_torch(y_true, y_pred):\n    y_true = torch.as_tensor(y_true)\n    y_pred = torch.as_tensor(y_pred)\n    return torch.mean((y_true - y_pred).pow(2), dim=1)\n\n\ndef crps_loss(y_true, y_pred_pdf):\n    y_pred_cdf = torch.cumsum(torch.as_tensor(y_pred_pdf), dim=1)\n    return crps_torch(y_true, y_pred_cdf).mean()\n\n\ndef crps_loss_cdf(y_true, y_pred_cdf):\n    return crps_torch(y_true, y_pred_cdf).mean()\n\n\ndef clean_StadiumType(txt):\n    if pd.isna(txt):\n        return np.nan\n    txt = txt.lower()\n    txt = ''.join([c for c in txt if c not in punctuation])\n    txt = re.sub(' +', ' ', txt)\n    txt = txt.strip()\n    txt = txt.replace('outside', 'outdoor')\n    txt = txt.replace('outdor', 'outdoor')\n    txt = txt.replace('outddors', 'outdoor')\n    txt = txt.replace('outdoors', 'outdoor')\n    txt = txt.replace('oudoor', 'outdoor')\n    txt = txt.replace('indoors', 'indoor')\n    txt = txt.replace('ourdoor', 'outdoor')\n    txt = txt.replace('retractable', 'rtr.')\n    return txt\n\n\ndef transform_StadiumType(txt):\n    if pd.isna(txt):\n        return np.nan\n    if 'outdoor' in txt or 'open' in txt:\n        return 1\n    if 'indoor' in txt or 'closed' in txt:\n        return 0\n    return np.nan\n\n\ndef str_to_seconds(txt):\n    txt = txt.split(':')\n    ans = int(txt[0]) * 60 + int(txt[1]) + int(txt[2]) / 60\n    return ans\n\n\ndef str_to_float(txt):\n    try:\n        return float(txt)\n    except Exception as e:\n        return np.NaN\n\n\ndef map_weather(txt):\n    ans = 1\n    if pd.isna(txt):\n        return 0\n    if 'partly' in txt:\n        ans *= 0.5\n    if 'climate controlled' in txt or 'indoor' in txt:\n        return ans * 3\n    if 'sunny' in txt or 'sun' in txt:\n        return ans * 2\n    if 'clear' in txt:\n        return ans\n    if 'cloudy' in txt:\n        return -ans\n    if 'rain' in txt or 'rainy' in txt:\n        return -2 * ans\n    if 'snow' in txt:\n        return -3 * ans\n    return 0\n\n\ndef standartize_orientations(df):\n    \"\"\"\n        https://www.kaggle.com/cpmpml/initial-wrangling-voronoi-areas-in-python\n\n        Make sure the offensive team is always moving left to right.\n    \"\"\"\n    df['ToLeft'] = df.PlayDirection == \"left\"\n    df['TeamOnOffense'] = \"home\"\n    df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = \"away\"\n    df['IsOnOffense'] = df.Team == df.TeamOnOffense  # Is player on offense?\n    df['HomeOnOffense'] = (df['TeamOnOffense'] == 'home').astype(int)\n\n    df['YardLine_std'] = 100 - df.YardLine\n    df.loc[df.FieldPosition.fillna('') == df.PossessionTeam,\n           'YardLine_std'\n    ] = df.loc[df.FieldPosition.fillna('') == df.PossessionTeam,\n               'YardLine']\n    df['X_std'] = df.X\n    df.loc[df.ToLeft, 'X_std'] = 120 - df.loc[df.ToLeft, 'X']\n    df['Y_std'] = df.Y\n    df.loc[df.ToLeft, 'Y_std'] = 160 / 3 - df.loc[df.ToLeft, 'Y']\n\n    df['Dir_rad'] = np.mod(90 - df.Dir, 360) * math.pi / 180.0\n    df['Dir_std'] = df.Dir_rad\n    df.loc[df.ToLeft, 'Dir_std'] = np.mod(np.pi + df.loc[df.ToLeft, 'Dir_rad'],\n                                          2 * np.pi)\n\n    df['Orientation_rad'] = np.mod(df.Orientation, 360) * math.pi / 180.0\n\n    df.loc[df.Season >= 2018, 'Orientation_rad'\n    ] = np.mod(df.loc[df.Season >= 2018, 'Orientation'] - 90,\n               360) * math.pi / 180.0\n\n    df['Orientation_rad'] = np.mod(df.Orientation, 360) * math.pi / 180.0\n    df.loc[df.Season >= 2018, 'Orientation_rad'\n    ] = np.mod(df.loc[df.Season >= 2018, 'Orientation'] - 90,\n               360) * math.pi / 180.0\n    df['Orientation_std'] = df.Orientation_rad\n    df.loc[df.ToLeft, 'Orientation_std'] = np.mod(\n        math.pi + df.loc[df.ToLeft, 'Orientation_rad'], 2 * math.pi)\n\n    replace_cols = ['YardLine', 'X', 'Y', 'Dir', 'Orientation']\n    for col in replace_cols:\n        df[col] = df[col + '_std']\n        df.drop([col + '_std'], axis=1, inplace=True)\n\n    drop_cols = ['Dir_rad', 'Orientation_rad']\n    for col in drop_cols:\n        df.drop([col], axis=1, inplace=True)\n\n    return df\n\n\ndef map_turf(df):\n    Turf = {'Field Turf': 'Artificial', 'A-Turf Titan': 'Artificial',\n            'Grass': 'Natural', 'UBU Sports Speed S5-M': 'Artificial',\n            'Artificial': 'Artificial', 'DD GrassMaster': 'Artificial',\n            'Natural Grass': 'Natural',\n            'UBU Speed Series-S5-M': 'Artificial', 'FieldTurf': 'Artificial',\n            'FieldTurf 360': 'Artificial', 'Natural grass': 'Natural',\n            'grass': 'Natural',\n            'Natural': 'Natural', 'Artifical': 'Artificial',\n            'FieldTurf360': 'Artificial', 'Naturall Grass': 'Natural',\n            'Field turf': 'Artificial',\n            'SISGrass': 'Artificial', 'Twenty-Four/Seven Turf': 'Artificial',\n            'natural grass': 'Natural'}\n    \n    turf_type = df['Turf'].map(Turf)\n    df['TurfIsNatural'] = (turf_type == 'Natural')\n    df = df.drop(['Turf'], axis=1)\n    return df\n\ndef clean_abbrs(df):\n    # CAREFUL. What if a new team appears?\n    map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}\n    for abb in df['PossessionTeam'].unique():\n        map_abbr[abb] = abb\n\n    def safe_map(val):\n        if map_abbr.get(val):\n            return map_abbr[val]\n        else:\n            return val\n\n    df['PossessionTeam'] = df['PossessionTeam'].apply(safe_map)\n    df['HomeTeamAbbr'] = df['HomeTeamAbbr'].apply(safe_map)\n    df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].apply(safe_map)\n    df['FieldPosition'] = df['FieldPosition'].apply(safe_map)\n    return df\n\ndef clean_weather(df):\n    df['WindSpeed'] = df['WindSpeed'].apply(\n        lambda x: x.lower().replace('mph', '').strip() if not pd.isna(x) else x)\n    df['WindSpeed'] = df['WindSpeed'].apply(lambda x: (int(x.split('-')[0]) + int(\n        x.split('-')[1])) / 2 if not pd.isna(x) and '-' in x else x)\n    df['WindSpeed'] = df['WindSpeed'].apply(\n        lambda x: (int(x.split()[0]) + int(x.split()[-1])) / 2 if not pd.isna(\n            x) and type(x) != float and 'gusts up to' in x else x)\n    df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)\n\n    df = df.drop(['WindDirection'], axis=1)\n\n    df['GameWeather'] = df['GameWeather'].str.lower()\n    indoor = \"indoor\"\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: indoor if not pd.isna(x) and indoor in x else x)\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: x.replace('coudy', 'cloudy').replace('clouidy',\n                                                       'cloudy').replace('party',\n                                                                         'partly') if not pd.isna(\n            x) else x)\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: x.replace('clear and sunny', 'sunny and clear') if not pd.isna(\n            x) else x)\n    df['GameWeather'] = df['GameWeather'].apply(\n        lambda x: x.replace('skies', '').replace(\"mostly\",\n                                                 \"\").strip() if not pd.isna(\n            x) else x)\n    df['GameWeather'] = df['GameWeather'].apply(map_weather)\n    return df\n\n# Encoding\n\ndef encode_formations(df):\n    # Formation columns\n    df = pd.concat([df.drop(['OffenseFormation'], axis=1),\n                    pd.get_dummies(df['OffenseFormation'],\n                                   prefix='OffenseFormation')], axis=1)\n    # Filling missing dummy columns at test stage\n    expected_columns = ['OffenseFormation_ACE',\n                        'OffenseFormation_EMPTY',\n                        'OffenseFormation_JUMBO',\n                        'OffenseFormation_PISTOL',\n                        'OffenseFormation_SHOTGUN',\n                        'OffenseFormation_SINGLEBACK',\n                        'OffenseFormation_WILDCAT',\n                        'OffenseFormation_I_FORM']\n    for col in expected_columns:\n        if not col in df.columns:\n            df[col] = 0\n    return df\n\ndef encode_personell(df):\n    # DefensePersonnel\n    counts = []\n    for i, val in df['DefensePersonnel'].str.split(',').iteritems():\n        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0,\n               'QB': 0}\n        if val is np.NaN:\n            counts.append({})\n            continue\n        for item in val:\n            name, number = item.strip().split(' ')[::-1]\n            row[name] = int(number)\n        counts.append(row)\n    defense_presonell_df = pd.DataFrame(counts)\n    defense_presonell_df.columns = ['defense_' + x for x in\n                                    defense_presonell_df.columns]\n    defense_presonell_df = defense_presonell_df.fillna(0).astype(int)\n    defense_presonell_df.index = df.index\n    df = pd.concat([df.drop(['DefensePersonnel'], axis=1), defense_presonell_df],\n                   axis=1)\n\n    # OffensePersonnel\n    counts = []\n    for i, val in df['OffensePersonnel'].str.split(',').iteritems():\n        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0,\n               'QB': 0}\n        if val is np.NaN:\n            counts.append({})\n            continue\n        for item in val:\n            name, number = item.strip().split(' ')[::-1]\n            row[name] = int(number)\n        counts.append(row)\n    offense_personnel_df = pd.DataFrame(counts)\n    offense_personnel_df.columns = ['offense_' + x for x in\n                                    offense_personnel_df.columns]\n    offense_personnel_df = offense_personnel_df.fillna(0).astype(int)\n    offense_personnel_df.index = df.index\n    df = pd.concat([df.drop(['OffensePersonnel'], axis=1), offense_personnel_df],\n                   axis=1)\n    return df\n\ndef engineer_features(df):\n    df['DefendersInTheBox_vs_Distance'] = (\n        df['DefendersInTheBox'] / df['Distance'])\n\n    df = add_phys_features(df)\n    return df\n\ndef preprocess_features(df):\n    \"\"\"Accepts df like train data, returns cleaned, standartized and enriched df\"\"\"\n\n    df = clean_abbrs(df)\n\n    df = standartize_orientations(df)\n\n    df = map_turf(df)\n\n    df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)\n    df['StadiumTypeShort'] = df['StadiumType'].apply(transform_StadiumType)\n    df = df.drop(['StadiumType'], axis=1)\n\n    # df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']\n    # df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']\n    # df['HomePossesion'] = (df['PossessionTeam'] == df['HomeTeamAbbr'])\n\n    df['GameClock'] = df['GameClock'].apply(str_to_seconds)\n\n    df['PlayerHeight'] = df['PlayerHeight'].apply(\n        lambda x: 12 * int(x.split('-')[0]) + int(x.split('-')[1]))\n    df['PlayerBMI'] = 703 * (df['PlayerWeight'] / (df['PlayerHeight']) ** 2)\n\n    df['TimeHandoff'] = pd.to_datetime(df['TimeHandoff'], utc=True)\n    df['TimeSnap'] = pd.to_datetime(df['TimeSnap'], utc=True)\n    df['TimeDelta'] = (df['TimeHandoff'] - df['TimeSnap']).apply(\n        lambda x: x.total_seconds())\n    df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(\n        lambda x: datetime.datetime.strptime(x, \"%m/%d/%Y\"))\n    df['PlayerBirthDate'] = pd.to_datetime(df['PlayerBirthDate'], utc=True)\n\n    seconds_in_year = 60 * 60 * 24 * 365.25\n    df['PlayerAge'] = (df['TimeHandoff'] - df['PlayerBirthDate']).apply(\n        lambda x: x.total_seconds()) / seconds_in_year\n\n    df = clean_weather(df)\n\n    df['IsRusher'] = df['NflId'] == df['NflIdRusher']\n    df['IsQB'] = df['Position'] == 'QB'\n\n    df = sort_df(df)\n\n    df = reduce_mem_usage(df)\n\n    return df\n\ndef sort_df(df):\n    df.sort_values(by=['PlayId', 'IsOnOffense', 'IsRusher', 'IsQB'], inplace=True)\n    return df\n\ndef make_y(df):\n    y = np.zeros(shape=(df['Yards'][::22].shape[0], 199))\n    for i, yard in enumerate(df['Yards'][::22]):\n        y[i, yard + 99:] = np.ones(shape=(1, 100 - yard))\n    return y\n\ndef compute_play_phys_features(play_id, play_df):\n    assert play_df.shape[0] == 22\n    play_index = play_df.index\n    offense_index = play_df.IsOnOffense\n    defense_index = ~play_df.IsOnOffense\n    offense_df = play_df[offense_index]\n    defense_df = play_df[defense_index]\n\n    offense_centroid_x = offense_df.X.mean()\n    offense_centroid_y = offense_df.Y.mean()\n    offense_x_std = offense_df.X.std()\n    offense_y_std = offense_df.Y.std()\n    offense_mean_force = offense_df.Force.mean()\n\n    defense_centroid_x = defense_df.X.mean()\n    defense_x_std = defense_df.X.std()\n    defense_centroid_y = defense_df.Y.mean()\n    defense_y_std = defense_df.Y.std()\n    defense_mean_force = defense_df.Force.mean()\n\n    rusher_row = offense_df[offense_df.IsRusher].iloc[0]\n\n    qb_row = offense_df[offense_df.Position == 'QB']\n    if not qb_row.empty:\n        qb_row = qb_row.iloc[0]\n\n    offense_centroid_pos = np.array([offense_centroid_x, offense_centroid_y])\n    defense_centroid_pos = np.array([defense_centroid_x, defense_centroid_y])\n    rusher_pos = rusher_row[['X', 'Y']].values[0]\n    qb_pos = qb_row[['X', 'Y']].values[0] if not qb_row.empty else None\n\n    rusher_dist_to_qb = np.linalg.norm(\n        rusher_pos - qb_pos) if not qb_row.empty else None\n    rusher_dist_to_offense_centroid = np.linalg.norm(\n        rusher_pos - offense_centroid_pos)\n    rusher_dist_to_defence_centroid = np.linalg.norm(\n        rusher_pos - defense_centroid_pos)\n\n    # Defender to rusher distances\n    defense_distances_to_runner = []\n    for row in defense_df.itertuples():\n        pos = np.array([row.X, row.Y])\n        defense_distances_to_runner.append(np.linalg.norm(rusher_pos - pos))\n    dist_to_rusher = defense_distances_to_runner = np.array(\n        defense_distances_to_runner)\n\n    time_to_rusher = defense_df['S'] / dist_to_rusher\n\n    defender_dist_to_runner_min = defense_distances_to_runner.min()\n\n    defender_time_to_runner_min = time_to_rusher.min()\n\n    # closest defenders\n    closest_defender = \\\n        defense_df[dist_to_rusher == defender_dist_to_runner_min].iloc[0]\n    closest_bytime_defender = \\\n        defense_df[time_to_rusher == defender_time_to_runner_min].iloc[0]\n\n    closest_defender_force_div_rusher_force = (\n        closest_defender.Force / rusher_row.Force)\n    closest_bytime_defender_force_div_rusher_force = (\n        closest_bytime_defender.Force / rusher_row.Force)\n\n    closest_bytime_defender_speed_div_rusher_speed = closest_bytime_defender.S / rusher_row.S\n    closest_bytime_defender_acceleration_div_rusher_acceleration = closest_bytime_defender.A / rusher_row.A\n\n    # Add play features\n    new_play_features = {\n        'phys_offense_x_std': offense_x_std,\n        'phys_offense_y_std': offense_y_std,\n        'phys_offense_mean_force': offense_mean_force,\n        'phys_defense_x_std': defense_x_std,\n        'phys_defense_y_std': defense_y_std,\n        'phys_defense_mean_force': defense_mean_force,\n\n        'phys_rusher_dist_to_qb': rusher_dist_to_qb,\n        'phys_rusher_dist_to_offense_centroid': rusher_dist_to_offense_centroid,\n        'phys_rusher_dist_to_defence_centroid': rusher_dist_to_defence_centroid,\n\n        'phys_defender_dist_to_runner_min': defender_dist_to_runner_min,\n\n        'phys_defender_time_to_runner_min': defender_time_to_runner_min,\n\n        'phys_closest_defender_force_div_rusher_force': closest_defender_force_div_rusher_force,\n        'phys_closest_bytime_defender_force_div_rusher_force': closest_bytime_defender_force_div_rusher_force,\n\n        'phys_closest_bytime_defender_speed_div_rusher_speed': closest_bytime_defender_speed_div_rusher_speed,\n        'phys_closest_bytime_defender_acceleration_div_rusher_acceleration': closest_bytime_defender_acceleration_div_rusher_acceleration,\n    }\n    return new_play_features\n\ndef compute_phys_features(df, verbose=True):\n    playid_index = df.PlayId.unique()\n    features_rows= []\n    \n    iterator = playid_index\n    if verbose:\n        iterator = tqdm.tqdm(playid_index)\n        \n    for play_id in iterator:\n        play_df = df[df.PlayId == play_id]\n        features_rows.append(compute_play_phys_features(play_id, play_df))\n\n    features_df = pd.DataFrame(features_rows, index=playid_index)\n    return features_df\n\ndef make_x(df, encode=True, fillna=True, verbose=True):\n    \"\"\"Input: source data, preprocesed.\n       Output: X dataframe\n    \"\"\"\n    source_play_id = df['PlayId']\n    df = df.copy()\n    df.index = source_play_id\n\n    # Encoding\n    if encode:\n        df = encode_formations(df)\n        df = encode_personell(df)\n\n    # Feature engineering\n\n    df['DefendersInTheBox_vs_Distance'] = (\n        df['DefendersInTheBox'] / df['Distance'])\n\n    df['Force'] = df['A'] * df['PlayerWeight']\n\n    if verbose:\n        print('Computing phys features')\n\n    phys_features = compute_phys_features(df, verbose=verbose)\n\n    # Assemble X\n\n    # Drop unnececary rows, keep only rusher\n\n    df = df[df.IsRusher].copy()\n\n    # Drop unnececary columns\n\n    cols_to_drop = [\n        'Yards', 'PlayDirection', 'TeamOnOffense', 'NflId', 'NflIdRusher',\n        'TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'FieldPosition',\n        'DisplayName', 'PossessionTeam', 'PlayerCollegeName', 'Position',\n        'HomeTeamAbbr', 'VisitorTeamAbbr', 'Stadium', 'Location', 'GameId',\n        'PlayId', 'Team', 'IsRusher', 'IsQB', 'IsOnOffense', 'Toleft',\n        'HomeOnOffense', 'Temperature', 'Humidity', 'WindSpeed', 'GameWeather',\n        'WindDirection', 'ToLeft', 'X', 'Y', 'Dis', 'PlayerWeight', 'PlayerHeight'\n    ]\n    cols_to_drop = list(set(cols_to_drop).intersection(set(list(df.columns))))\n    df.drop(cols_to_drop, axis=1, inplace=True)\n\n    if verbose:\n        print('Dropped cols:', cols_to_drop)\n\n    # Assemble X\n    \n    # Get player columns\n    cols_player = ['S',\n                   'A',\n                   'Orientation',\n                   'Dir',\n                   'PlayerBMI',\n                   'PlayerAge',\n                   'Force']\n                \n    X_df = df.copy()\n\n    # Add features\n\n    X_df = pd.concat([X_df, phys_features], axis=1)\n\n    # Pospreprocesing\n\n    X_df = X_df.replace([np.inf, -np.inf], np.nan)\n\n    if fillna:\n        X_df.fillna(-999, inplace=True)\n\n    X_df = reduce_mem_usage(X_df)\n\n    assert X_df.shape[0] == source_play_id.drop_duplicates().count()\n    return X_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})\n# df_train = pd.read_csv('data/train.csv', dtype={'WindSpeed': 'object'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_preprocessed = preprocess_features(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = make_x(df_train_preprocessed, fillna=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = make_y(df_train_preprocessed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape, y_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"N_FEATURES = X_train.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def iterate_minibatches(X, y, batchsize):\n    X = torch.as_tensor(X).double().to(device=DEVICE)\n    y = torch.as_tensor(y).double().to(device=DEVICE)\n    indices = np.random.permutation(np.arange(len(X)))\n    for start in range(0, len(indices), batchsize):\n        ix = indices[start: start + batchsize]\n        yield X[ix], y[ix]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'cpu'\nBATCH_SIZE = 100\nLEARNING_RATE = 0.0001\nNUM_EPOCHS = 200\nEARLY_STOP_AFTER = 20\nWEIGHT_DECAY = 0.0002\nSCHEDULER_PATIENCE = 4\nL1_LOSS_WEIGHT = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NeuralNet(nn.Module):\n    def __init__(self, n_blocks=3):\n        super().__init__()\n        linear_blocks = [\n            nn.Sequential(\n                nn.Linear(N_FEATURES, N_FEATURES*2 ),\n                nn.ReLU(),\n            )\n        ]\n\n        for i in range(n_blocks):\n            linear_blocks.append(\n                nn.Sequential(\n                    nn.Linear(N_FEATURES*2, N_FEATURES*2),\n                    nn.ReLU(),\n                    nn.Linear(N_FEATURES*2, N_FEATURES*2),\n                    nn.ReLU(),\n                    nn.Dropout(0.1),\n                    nn.BatchNorm1d(N_FEATURES*2),\n                ))\n            \n        self.seq = nn.Sequential(*linear_blocks)\n\n        self.head_classifier = nn.Sequential(\n            nn.Linear(N_FEATURES*2, 199),\n            nn.Softmax()\n        )\n        self.head_regressor = nn.Sequential(\n            nn.Linear(N_FEATURES*2, 1),\n        )\n\n    def forward(self, x_batch):\n        latent_out = self.seq.forward(x_batch)\n        return self.head_classifier(latent_out), self.head_regressor(latent_out)\n\n    def loss(self, x_batch, y_batch):\n        y_batch = torch.as_tensor(y_batch).double().to(device=DEVICE)\n        y_pdf, y_yard = self(x_batch)\n        loss_crps = crps_loss(y_batch, y_pdf)\n        \n        y_yard = y_yard.flatten().double()\n        y_batch_yard = cdf_to_yard_torch(y_batch).double()\n        loss_mae = torch.mean(torch.abs(y_batch_yard-y_yard))\n        return loss_crps + loss_mae*L1_LOSS_WEIGHT\n\n    def predict_pdf(self, x_batch):\n        y_pdf, y_yard = self(x_batch)\n        return y_pdf\n    \n    def predict_cdf(self, x_batch):\n        y_pred_pdf = self.predict_pdf(x_batch)\n        y_pred_cdf = torch.cumsum(y_pred_pdf, dim=1)\n        y_pred_cdf = torch.clamp(y_pred_cdf, 0, 1)\n        return y_pred_cdf\n\nmodel = NeuralNet()\nmodel = model.double().to(device=DEVICE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, opt, scheduler, X_train, y_train, X_val, y_val, early_stop_patience, batch_size, n_epochs, print_every=5):\n    train_loss = []\n    train_loss_stds = []\n    val_accuracy = []\n    val_accuracy_stds = []\n\n    best_val_acc = None\n    best_model = None\n    patience = 0\n\n    for epoch in range(n_epochs):\n        try:\n            opt.zero_grad()\n\n            start_time = time.time()\n            model.train(True)\n            epoch_loss = []\n\n            for X_batch, y_batch in iter(iterate_minibatches(X_train, y_train, batch_size)):\n                loss = model.loss(X_batch, y_batch)\n                epoch_loss.append(float(loss.item()))\n                loss.backward()\n                opt.step()\n                opt.zero_grad()\n\n            train_loss.append(np.mean(epoch_loss))\n            train_loss_stds.append(np.std(epoch_loss))\n            model.train(False)\n\n            epoch_val_acc = []\n            for X_batch, y_batch in iter(iterate_minibatches(X_val, y_val, batch_size)): \n                y_val_pred_cdf = model.predict_cdf(X_batch).to(device=DEVICE)\n                val_loss = crps_loss_cdf(y_batch, y_val_pred_cdf)\n                epoch_val_acc.append(val_loss.item())\n            val_acc = np.mean(epoch_val_acc)\n            val_accuracy.append(val_acc)\n            val_accuracy_stds.append(np.std(epoch_val_acc))\n\n\n            scheduler.step(val_acc)\n\n            if epoch % print_every == 0:\n                print(\"Epoch {} of {} took {:.3f}s\".format(\n                    epoch + 1, n_epochs, time.time() - start_time))\n                print(\"  training loss: \\t{:.6f}\".format(train_loss[-1]))\n                print(\"  validation score: \\t\\t\\t{:.6f}\".format(val_accuracy[-1]))\n\n            if best_val_acc is None or val_acc < best_val_acc:\n                best_val_acc = val_acc\n                patience = 0\n                torch.save(model.state_dict(), 'best_model')\n            else:\n                patience += 1\n                print(f'Validation score has not improved for {patience} epochs.')\n            if patience >= early_stop_patience:\n                print('Early stopping.')\n                break\n        except KeyboardInterrupt:\n            break\n    model.load_state_dict(torch.load('best_model'))\n    model.train(False)\n\n    history = {\n        'train_loss': train_loss,\n        'train_loss_stds': train_loss_stds,\n        'val_accuracy': val_accuracy,\n        'val_accuracy_stds': val_accuracy_stds,\n    }\n    \n    return model, history\n\ndef train_nn_pipeline(model, X, y, early_stop_after=EARLY_STOP_AFTER, \n                      batch_size=BATCH_SIZE,\n                      n_epochs=NUM_EPOCHS):\n    scaler = MinMaxScaler()\n    X = scaler.fit_transform(X)\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)\n    \n    opt = torch.optim.Adamax(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, mode='min', patience=SCHEDULER_PATIENCE, verbose=True, threshold=1e-6, eps=1e-9)\n\n    model, train_history = train(model, opt, scheduler, X_train, y_train, X_val, y_val, early_stop_after, batch_size, n_epochs)\n    \n    return model, scaler, train_history\n\ndef nn_pipeline_predict(model, scaler, X):\n    X = torch.as_tensor(scaler.transform(X)).double().to(device=DEVICE)\n    y_pred = model.predict_cdf(X)\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CV"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import RepeatedKFold\n\ncv = RepeatedKFold(n_splits=5, n_repeats=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_cv(X, y, cv, model_class, train_function, n_epochs=NUM_EPOCHS):\n    models, scalers, losses = [], [], []\n    i = 0\n    X = X.values\n    for train_index, test_index in cv.split(X):\n        print(f'Training model {i}')\n        try:\n            X_train, X_test = X[train_index], X[test_index]\n            y_train, y_test = y[train_index], y[test_index]\n            model = model_class().double().to(device=DEVICE)\n            model, scaler, history = train_nn_pipeline(model, X_train, y_train, n_epochs=n_epochs)\n            y_pred_cdf_test = nn_pipeline_predict(model, scaler, X_test)\n            y_test = torch.as_tensor(y_test).to(device=DEVICE)\n            test_loss = crps_torch(y_test, y_pred_cdf_test).mean()\n            losses.append(test_loss.cpu().data.item())\n            models.append(model)\n            scalers.append(scaler)\n            i+=1\n        except KeyboardInterrupt:\n            break\n    return models, scalers, losses\n\ndef cv_predict(models, scalers, X):\n    X = X.values\n    predicted_cdfs = []\n    for model, scaler in zip(models, scalers):\n        X_prep = torch.as_tensor(scaler.transform(X)).double().to(device=DEVICE)\n        y_pred_cdf = nn_pipeline_predict(model, scaler, X).cpu().detach().numpy()\n        predicted_cdfs.append(y_pred_cdf)\n    return np.mean(predicted_cdfs, axis=0) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models, scalers, losses = run_cv(X_train, y_train, cv, NeuralNet, train_nn_pipeline)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SUBMIT"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = nflrush.make_env()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"def get_predictions(df_test, y_test, models, scalers):\n    X = make_x(preprocess_features(df_test), fillna=True, verbose=False)\n    y_pred = cv_predict(models, scalers, X)\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for df_test, y_cdf_test in tqdm.tqdm(iter_test):\n    y_pred = get_predictions(df_test, y_cdf_test, models, scalers)\n    env.predict(pd.DataFrame(data=y_pred,columns=y_cdf_test.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"env.write_submission_file()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}