[Fig01,02,10,11]-phase-diagram-T32.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fig 01, 02, 10, 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_import import *\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "from plotting import *\n",
    "device = 'cuda:0'\n",
    "from tqdm import tqdm\n",
    "\n",
    "def load_result_table_best(project=\"feeds/v4_dvsT\"):\n",
    "    # Project is specified by <entity/project-name>\n",
    "    runs = api.runs(project)\n",
    "\n",
    "    summary_list = []\n",
    "    for run in tqdm(runs): \n",
    "        # .summary contains the output keys/values for metrics like accuracy.\n",
    "        #  We call ._json_dict to omit large files \n",
    "        try:\n",
    "            history = run.history(keys=['val_acc'])\n",
    "            best = history['val_acc'].max()\n",
    "        except:\n",
    "            best = 0\n",
    "        \n",
    "        \n",
    "        res = {**run.summary._json_dict,\n",
    "            **{k: v for k,v in run.config.items()\n",
    "            },'name':run.name,'entity': run.entity, 'project': run.project, 'best_val_acc': best}\n",
    "        #config_cols = list(run.config.keys())\n",
    "        summary_list.append(res)\n",
    "\n",
    "    df = pd.DataFrame(summary_list)\n",
    "    return df.set_index('name') #, config_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This will take for ever because the data loading is inefficient, but we only do it once..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_orig = load_result_table_best('feeds/phase_diagram_T32').reset_index()\n",
    "df_orig = df_orig[df_orig['epoch']==499]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure_dir = FIGURE_DIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allowed_p = [1,2,4,8,16,32,64,128,3, 6, 11, 23, 45, 91]\n",
    "allowed_model_dim = [1,2,4,8,16,32,64,128,3, 6, 11, 23, 45, 91]\n",
    "\n",
    "df_orig = df_orig[df_orig['p'].isin(allowed_p) & df_orig['model_dim'].isin(allowed_model_dim)]\n",
    "df_orig.model_dim.unique(), df_orig.p.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_orig.groupby(CONFIG_COLS).mean(numeric_only=True).reset_index()\n",
    "df_std = df_orig.groupby(CONFIG_COLS).std(numeric_only=True).reset_index()\n",
    "df_max = df_orig.groupby(CONFIG_COLS).max(numeric_only=True).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def bound1(T,L):\n",
    "    # for lin, lin_sftm\n",
    "    return math.ceil(T*(2*L-3)**2/(T-1+(2*L-3)**2))\n",
    "\n",
    "def bound2(T,L):\n",
    "    # for dot, bos, p=1\n",
    "    return bound1(T,L) + 1\n",
    "\n",
    "def bound3(T,L):\n",
    "    # for dot, bos, p=T\n",
    "    return math.ceil(T*(2*L-3)/(T-1+(2*L-3))) #+ 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = 32\n",
    "L = 10\n",
    "k = df[(df['T'] == T) & (df.seq_len == L) ]\n",
    "k_std = df_std[(df_std['T'] == T) & (df_std.seq_len == L) ]\n",
    "k_max = df_max[(df_std['T'] == T) & (df_std.seq_len == L) ]\n",
    "\n",
    "fig, axes = plt.subplots(2,3,figsize=(12,6),sharex=True,sharey=True)\n",
    "\n",
    "axes_assignment = {\n",
    "        'linear': (1,2),\n",
    "        'linear+sftm': (0,2),\n",
    "        'dot': (1,1),\n",
    "        'dot+sftm': (0,1),\n",
    "        'dotBOS' : (1,0),\n",
    "        'dotBOS+sftm': (0,0),\n",
    "}\n",
    "\n",
    "axes[axes_assignment['linear+sftm']].set_title('[lin]\\nlinear mixing',fontsize=9)\n",
    "axes[axes_assignment['dot+sftm']].set_title('[dot]\\ndot-product attention',fontsize=9)\n",
    "axes[axes_assignment['dotBOS+sftm']].set_title('[bos]\\ndot-product attention & BOS token',fontsize=9)\n",
    "\n",
    "axes[1,0].set_ylabel('no softmax',fontsize=9)\n",
    "axes[0,0].set_ylabel('with softmax',fontsize=9)\n",
    "axes[1,1].set_xlabel('hidden layer size $p$',fontsize=12)\n",
    "\n",
    "\n",
    "for name, model_config in MODELS.items():\n",
    "    ax = axes[axes_assignment[name]]\n",
    "    a = k[(k.attention_input == model_config['attention_input'])\n",
    "            & (k.no_softmax == model_config['no_softmax']) \n",
    "            & (k.dataset_type == model_config['dataset'])]\n",
    "    a_std = k_std[(k_std.attention_input == model_config['attention_input'])\n",
    "            & (k_std.no_softmax == model_config['no_softmax']) \n",
    "            & (k_std.dataset_type == model_config['dataset'])]\n",
    "    a_max = k_max[(k_max.attention_input == model_config['attention_input'])\n",
    "                & (k_max.no_softmax == model_config['no_softmax']) \n",
    "                & (k_max.dataset_type == model_config['dataset'])]\n",
    "    \n",
    "    if len(a) == 0:\n",
    "        continue\n",
    "\n",
    "    aother_mean = pd.pivot_table(a, index='p', columns='model_dim', values='val_acc')\n",
    "    aother_max = pd.pivot_table(a_max, index='p', columns='model_dim', values='best_val_acc')\n",
    "    \n",
    "    \n",
    "    smap = ax.pcolor(aother_mean.index+1, aother_mean.columns+1, aother_mean.values.T, cmap='Spectral', vmin=0, vmax=100)\n",
    "    import matplotlib.patches as patches\n",
    "    x = aother_max.index\n",
    "    y = aother_max.columns\n",
    "    Z = aother_max.values.T\n",
    "    for i in range(len(x)-1):\n",
    "        for j in range(len(y)-1):\n",
    "                if Z[j, i] >= 100.0:\n",
    "                        # Get the coordinates of the lower-left corner of the cell\n",
    "                        #rect = patches.Rectangle((x[i], y[j]), x[i+1]-x[i], y[j+1]-y[j],\n",
    "                        #                        linewidth=2, edgecolor='red', facecolor='none')\n",
    "                        #ax.add_patch(rect)\n",
    "                        ax.scatter(x[i+1]+1,y[j+1]+1,s=20,marker='*',c='white')\n",
    "                elif Z[j, i] >= 99.0:\n",
    "                        # Get the coordinates of the lower-left corner of the cell\n",
    "                        #rect = patches.Rectangle((x[i], y[j]), x[i+1]-x[i], y[j+1]-y[j],\n",
    "                        #                        linewidth=2, edgecolor='red', facecolor='none')\n",
    "                        #ax.add_patch(rect)\n",
    "                        ax.scatter(x[i+1]+1,y[j+1]+1,s=10,marker='.',c='white')\n",
    "                      \n",
    "    \n",
    "    #smap = ax.pcolor(aother2.index, aother2.columns, aother2.values.T, cmap='binary_r', vmin=0, vmax=1,alpha=0.5,edgecolors='black',linewidth=1)\n",
    "    #smap = ax.scatter(a.p,a.model_dim,  c=a.val_acc,label=name, marker='o', lw=1,vmin=0,vmax=100,cmap='Spectral',s=200)\n",
    "    \n",
    "    ax.axvline(T,c='white')\n",
    "    ax.axhline(T,c='white')\n",
    "    #ax.axhline(np.log(T)+1,c='white')\n",
    "    ax.set_xscale('log')\n",
    "    ax.set_yscale('log')\n",
    "    \n",
    "    if 'linear' in name:\n",
    "        ax2 = ax.twinx()\n",
    "        ax2.set_yscale('log')\n",
    "        ax2.set_ylim(ax.get_ylim())\n",
    "        ax2.set_yticks([T])\n",
    "        ax2.set_yticklabels(['T'])\n",
    "fig.text(0.06, 0.5, 'embedding dimension $d$', va='center', rotation='vertical',fontsize=12)\n",
    "#smap = plt.cm.ScalarMappable(cmap='Spectral', vmin=0, vmax=100)\n",
    "fig.colorbar(smap,ax=axes,location='right',shrink=0.5,label='mean accuracy [%]')\n",
    "plt.savefig(figure_dir / 'memorizing_counts-phase-diagram-T32-mean-best-marked.png',bbox_inches='tight')\n",
    "\n",
    "\"\"\"\n",
    "axes[axes_assignment['linear+sftm']].axhline(bound1(T,L),c='white',ls='--',label='bound 1')\n",
    "axes[axes_assignment['linear']].axhline(bound1(T,L),c='white',ls='--',label='bound 1')\n",
    "axes[axes_assignment['dot']].axhline(bound3(T,L),c='white',ls='--',label='bound 1')\n",
    "axes[axes_assignment['dot+sftm']].axhline(bound3(T,L),c='white',ls='--',label='bound 1')\n",
    "axes[axes_assignment['dotBOS']].axhline(bound3(T,L),c='white',ls='--',label='bound 1')\n",
    "axes[axes_assignment['dotBOS']].axhline(bound2(T,L),c='white',ls='--',label='bound 1')\n",
    "axes[axes_assignment['dotBOS+sftm']].axhline(2,c='white',ls='--',label='bound 1')\n",
    "\"\"\"\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = 32\n",
    "L = 10\n",
    "k = df[(df['T'] == T) & (df.seq_len == L) ]\n",
    "k_std = df_std[(df_std['T'] == T) & (df_std.seq_len == L) ]\n",
    "\n",
    "fig, axes = plt.subplots(2,3,figsize=(12,6),sharex=True,sharey=True)\n",
    "\n",
    "axes_assignment = {\n",
    "        'linear': (1,2),\n",
    "        'linear+sftm': (0,2),\n",
    "        'dot': (1,1),\n",
    "        'dot+sftm': (0,1),\n",
    "        'dotBOS' : (1,0),\n",
    "        'dotBOS+sftm': (0,0),\n",
    "}\n",
    "\n",
    "axes[axes_assignment['linear+sftm']].set_title('[lin]\\nlinear mixing',fontsize=9)\n",
    "axes[axes_assignment['dot+sftm']].set_title('[dot]\\ndot-product attention',fontsize=9)\n",
    "axes[axes_assignment['dotBOS+sftm']].set_title('[bos]\\ndot-product attention & BOS token',fontsize=9)\n",
    "\n",
    "axes[1,0].set_ylabel('no softmax',fontsize=9)\n",
    "axes[0,0].set_ylabel('with softmax',fontsize=9)\n",
    "axes[1,1].set_xlabel('hidden layer size $p$',fontsize=12)\n",
    "\n",
    "\n",
    "for name, model_config in MODELS.items():\n",
    "    ax = axes[axes_assignment[name]]\n",
    "    a = k[(k.attention_input == model_config['attention_input'])\n",
    "            & (k.no_softmax == model_config['no_softmax']) \n",
    "            & (k.dataset_type == model_config['dataset'])]\n",
    "    a_std = k_std[(k_std.attention_input == model_config['attention_input'])\n",
    "            & (k_std.no_softmax == model_config['no_softmax']) \n",
    "            & (k_std.dataset_type == model_config['dataset'])]\n",
    "    if len(a) == 0:\n",
    "        continue\n",
    "\n",
    "    aother = pd.pivot_table(a, index='p', columns='model_dim', values='val_acc')\n",
    "    smap = ax.pcolor(aother.index+1, aother.columns+1, aother.values.T, cmap='Spectral', vmin=0, vmax=100)\n",
    "    #smap = ax.scatter(a.p,a.model_dim,  c=a.val_acc,label=name, marker='o', lw=1,vmin=0,vmax=100,cmap='Spectral',s=200)\n",
    "    \n",
    "    ax.axvline(T,c='white')\n",
    "    ax.axhline(T,c='white')\n",
    "    ax.set_xscale('log')\n",
    "    ax.set_yscale('log')\n",
    "    \n",
    "    if 'linear' in name:\n",
    "        ax2 = ax.twinx()\n",
    "        ax2.set_yscale('log')\n",
    "        ax2.set_ylim(ax.get_ylim())\n",
    "        ax2.set_yticks([T])\n",
    "        ax2.set_yticklabels(['T'])\n",
    "fig.text(0.06, 0.5, 'embedding dimension $d$', va='center', rotation='vertical',fontsize=12)\n",
    "#smap = plt.cm.ScalarMappable(cmap='Spectral', vmin=0, vmax=100)\n",
    "fig.colorbar(smap,ax=axes,location='right',shrink=0.5,label='mean accuracy [%]')\n",
    "plt.savefig(figure_dir / 'memorizing_counts-phase-diagram-T32-avg.png',bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = 32\n",
    "L = 10\n",
    "k = df[(df['T'] == T) & (df.seq_len == L) ]\n",
    "k_std = df_std[(df_std['T'] == T) & (df_std.seq_len == L) ]\n",
    "k_max = df_max[(df_std['T'] == T) & (df_std.seq_len == L) ]\n",
    "\n",
    "fig, axes = plt.subplots(2,3,figsize=(12,6),sharex=True,sharey=True)\n",
    "\n",
    "axes_assignment = {\n",
    "        'linear': (1,2),\n",
    "        'linear+sftm': (0,2),\n",
    "        'dot': (1,1),\n",
    "        'dot+sftm': (0,1),\n",
    "        'dotBOS' : (1,0),\n",
    "        'dotBOS+sftm': (0,0),\n",
    "}\n",
    "\n",
    "axes[axes_assignment['linear+sftm']].set_title('[lin]\\nlinear mixing',fontsize=9)\n",
    "axes[axes_assignment['dot+sftm']].set_title('[dot]\\ndot-product attention',fontsize=9)\n",
    "axes[axes_assignment['dotBOS+sftm']].set_title('[bos]\\ndot-product attention & BOS token',fontsize=9)\n",
    "\n",
    "axes[1,0].set_ylabel('no softmax',fontsize=9)\n",
    "axes[0,0].set_ylabel('with softmax',fontsize=9)\n",
    "axes[1,1].set_xlabel('hidden layer size $p$',fontsize=12)\n",
    "\n",
    "\n",
    "for name, model_config in MODELS.items():\n",
    "    ax = axes[axes_assignment[name]]\n",
    "    a = k[(k.attention_input == model_config['attention_input'])\n",
    "            & (k.no_softmax == model_config['no_softmax']) \n",
    "            & (k.dataset_type == model_config['dataset'])]\n",
    "    a_std = k_std[(k_std.attention_input == model_config['attention_input'])\n",
    "            & (k_std.no_softmax == model_config['no_softmax']) \n",
    "            & (k_std.dataset_type == model_config['dataset'])]\n",
    "    a_max = k_max[(k_max.attention_input == model_config['attention_input'])\n",
    "                & (k_max.no_softmax == model_config['no_softmax']) \n",
    "                & (k_max.dataset_type == model_config['dataset'])]\n",
    "    \n",
    "    if len(a) == 0:\n",
    "        continue\n",
    "\n",
    "    aother_mean = pd.pivot_table(a, index='p', columns='model_dim', values='val_acc')\n",
    "    aother_max = pd.pivot_table(a_max, index='p', columns='model_dim', values='best_val_acc')\n",
    "    \n",
    "    \n",
    "    smap = ax.pcolor(aother_max.index+1, aother_max.columns+1, aother_max.values.T, cmap='Spectral', vmin=0, vmax=100)\n",
    "    \n",
    "    #smap = ax.pcolor(aother2.index, aother2.columns, aother2.values.T, cmap='binary_r', vmin=0, vmax=1,alpha=0.5,edgecolors='black',linewidth=1)\n",
    "    #smap = ax.scatter(a.p,a.model_dim,  c=a.val_acc,label=name, marker='o', lw=1,vmin=0,vmax=100,cmap='Spectral',s=200)\n",
    "    \n",
    "    ax.axvline(T,c='white')\n",
    "    ax.axhline(T,c='white')\n",
    "    #ax.axhline(np.log(T)+1,c='white')\n",
    "    ax.set_xscale('log')\n",
    "    ax.set_yscale('log')\n",
    "    \n",
    "    if 'linear' in name:\n",
    "        ax2 = ax.twinx()\n",
    "        ax2.set_yscale('log')\n",
    "        ax2.set_ylim(ax.get_ylim())\n",
    "        ax2.set_yticks([T])\n",
    "        ax2.set_yticklabels(['T'])\n",
    "fig.text(0.06, 0.5, 'embedding dimension $d$', va='center', rotation='vertical',fontsize=12)\n",
    "#smap = plt.cm.ScalarMappable(cmap='Spectral', vmin=0, vmax=100)\n",
    "fig.colorbar(smap,ax=axes,location='right',shrink=0.5,label='best out of 5 accuracy [%]')\n",
    "plt.savefig(figure_dir / 'memorizing_counts-phase-diagram-T32-best.png',bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = 32\n",
    "L = 10\n",
    "k = df[(df['T'] == T) & (df.seq_len == L) ]\n",
    "k_std = df_std[(df_std['T'] == T) & (df_std.seq_len == L) ]\n",
    "\n",
    "fig, axes = plt.subplots(2,3,figsize=(12,6),sharex=True,sharey=True)\n",
    "\n",
    "axes_assignment = {\n",
    "        'linear': (1,2),\n",
    "        'linear+sftm': (0,2),\n",
    "        'dot': (1,1),\n",
    "        'dot+sftm': (0,1),\n",
    "        'dotBOS' : (1,0),\n",
    "        'dotBOS+sftm': (0,0),\n",
    "}\n",
    "\n",
    "axes[axes_assignment['linear+sftm']].set_title('[lin]\\nlinear mixing',fontsize=9)\n",
    "axes[axes_assignment['dot+sftm']].set_title('[dot]\\ndot-product attention',fontsize=9)\n",
    "axes[axes_assignment['dotBOS+sftm']].set_title('[bos]\\ndot-product attention & BOS token',fontsize=9)\n",
    "\n",
    "axes[1,0].set_ylabel('no softmax',fontsize=9)\n",
    "axes[0,0].set_ylabel('with softmax',fontsize=9)\n",
    "axes[1,1].set_xlabel('hidden layer size $p$',fontsize=12)\n",
    "\n",
    "\n",
    "for name, model_config in MODELS.items():\n",
    "    ax = axes[axes_assignment[name]]\n",
    "    a = k[(k.attention_input == model_config['attention_input'])\n",
    "            & (k.no_softmax == model_config['no_softmax']) \n",
    "            & (k.dataset_type == model_config['dataset'])]\n",
    "    a_std = k_std[(k_std.attention_input == model_config['attention_input'])\n",
    "            & (k_std.no_softmax == model_config['no_softmax']) \n",
    "            & (k_std.dataset_type == model_config['dataset'])]\n",
    "    if len(a) == 0:\n",
    "        continue\n",
    "\n",
    "    aother = pd.pivot_table(a_std, index='p', columns='model_dim', values='val_acc')\n",
    "    smap = ax.pcolor(aother.index, aother.columns, aother.values.T, cmap='Oranges', vmin=0,vmax=40)\n",
    "    #smap = ax.scatter(a.p,a.model_dim,  c=a.val_acc,label=name, marker='o', lw=1,vmin=0,vmax=100,cmap='Spectral',s=200)\n",
    "    \n",
    "    ax.axvline(T,c='white')\n",
    "    ax.axhline(T,c='white')\n",
    "    ax.set_xscale('log')\n",
    "    ax.set_yscale('log')\n",
    "    \n",
    "    if 'linear' in name:\n",
    "        ax2 = ax.twinx()\n",
    "        ax2.set_yscale('log')\n",
    "        ax2.set_ylim(ax.get_ylim())\n",
    "        ax2.set_yticks([T])\n",
    "        ax2.set_yticklabels(['T'])\n",
    "fig.text(0.06, 0.5, 'embedding dimension $d$', va='center', rotation='vertical',fontsize=12)\n",
    "#smap = plt.cm.ScalarMappable(cmap='Spectral', vmin=0, vmax=100)\n",
    "fig.colorbar(smap,ax=axes,location='right',shrink=0.5,label='std. of accuracy over 5 runs')\n",
    "plt.savefig(figure_dir / 'memorizing_counts-phase-diagram-T32-std.png',bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = 32\n",
    "L = 10\n",
    "k = df_orig[(df_orig['T'] == T) & (df_orig.seq_len == L) ]\n",
    "\n",
    "fig, axes = plt.subplots(1,2,figsize=(7,3),sharex=True,sharey=True)\n",
    "\n",
    "ax = plt.gca()\n",
    "\n",
    "\n",
    "sftm_marker = '*'\n",
    "other_marker = '.'\n",
    "lin_col = 'blue'\n",
    "dot_col = 'red'\n",
    "bos_col = 'orange'\n",
    "for name, model_config in MODELS.items():\n",
    "    a = k[(k.attention_input == model_config['attention_input'])\n",
    "            & (k.no_softmax == model_config['no_softmax']) \n",
    "            & (k.dataset_type == model_config['dataset'])]\n",
    "    a_std = k_std[(k_std.attention_input == model_config['attention_input'])\n",
    "            & (k_std.no_softmax == model_config['no_softmax']) \n",
    "            & (k_std.dataset_type == model_config['dataset'])]\n",
    "    print(name)\n",
    "    if 'linear' in name:\n",
    "        color = lin_col#$l$'\n",
    "    if 'dot' in name:\n",
    "        color = dot_col\n",
    "    if 'BOS' in name:\n",
    "        color = bos_col\n",
    "    \n",
    "    ax = axes[0] if '+' in name else axes[1]\n",
    "    smap = ax.scatter(a.params_count,a.val_acc,color=color,alpha=0.1,s=10, )\n",
    "    b = a.sort_values('params_count')\n",
    "    # get the running maximum\n",
    "    b['val_acc_max'] = b.val_acc.cummax()\n",
    "    ax.plot(b.params_count,b.val_acc_max,color=color,lw=2)\n",
    "    \n",
    "ax.set_xscale('log')\n",
    "ax.plot([],[],color='black',lw=2,label='running best')\n",
    "plt.scatter([],[],marker=\"s\",label='lin',color=lin_col)\n",
    "plt.scatter([],[],marker=\"s\",label='dot',color=dot_col)\n",
    "plt.scatter([],[],marker=\"s\",label='bos',color=bos_col)\n",
    "plt.legend()\n",
    "axes[0].set_xlabel('number of parameters')\n",
    "axes[0].set_title('with softmax')\n",
    "axes[1].set_title('no softmax')\n",
    "axes[1].set_xlabel('number of parameters')\n",
    "axes[0].set_ylabel('accuracy [%]')\n",
    "fig.suptitle('T=32')\n",
    "ax.set_ylim(15,105)\n",
    "plt.savefig(figure_dir / 'memorizing_counts-params-T32.png',bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}