12_custom_models_and_training_with_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 12 – Custom Models and Training with TensorFlow**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code in chapter 12._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table align=\"left\">\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/ageron/handson-ml2/blob/master/12_custom_models_and_training_with_tensorflow.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "try:\n",
    "    # %tensorflow_version only exists in Colab.\n",
    "    %tensorflow_version 2.x\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# TensorFlow ≥2.4 is required in this notebook\n",
    "# Earlier 2.x versions will mostly work the same, but with a few bugs\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.4\"\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensors and operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant([[1., 2., 3.], [4., 5., 6.]]) # matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=42>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(42) # scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = tf.constant([[1., 2., 3.], [4., 5., 6.]])\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 3])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[2., 3.],\n",
       "       [5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[:, 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 1), dtype=float32, numpy=\n",
       "array([[2.],\n",
       "       [5.]], dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[..., 1, tf.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[11., 12., 13.],\n",
       "       [14., 15., 16.]], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[14., 32.],\n",
       "       [32., 77.]], dtype=float32)>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t @ tf.transpose(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using `keras.backend`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[11., 26.],\n",
       "       [14., 35.],\n",
       "       [19., 46.]], dtype=float32)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "K = keras.backend\n",
    "K.square(K.transpose(t)) + 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From/To NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([2., 4., 5.])>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([2., 4., 5.])\n",
    "tf.constant(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([ 4., 16., 25.])>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conflicting Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:AddV2]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a double tensor [Op:AddV2]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40., dtype=tf.float64)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2 = tf.constant(40., dtype=tf.float64)\n",
    "tf.constant(2.0) + tf.cast(t2, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(b\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(\"café\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = tf.constant([ord(c) for c in \"café\"])\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=4>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = tf.strings.unicode_encode(u, \"UTF-8\")\n",
    "tf.strings.length(b, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_decode(b, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### String arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = tf.constant([\"Café\", \"Coffee\", \"caffè\", \"咖啡\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([4, 6, 5, 2], dtype=int32)>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.length(p, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857]]>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = tf.strings.unicode_decode(p, \"UTF8\")\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857]]>\n"
     ]
    }
   ],
   "source": [
    "print(r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ragged tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([ 67 111 102 102 101 101], shape=(6,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "print(r[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232]]>\n"
     ]
    }
   ],
   "source": [
    "print(r[1:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857], [65, 66], [], [67]]>\n"
     ]
    }
   ],
   "source": [
    "r2 = tf.ragged.constant([[65, 66], [], [67]])\n",
    "print(tf.concat([r, r2], axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233, 68, 69, 70], [67, 111, 102, 102, 101, 101, 71], [99, 97, 102, 102, 232], [21654, 21857, 72, 73]]>\n"
     ]
    }
   ],
   "source": [
    "r3 = tf.ragged.constant([[68, 69, 70], [71], [], [72, 73]])\n",
    "print(tf.concat([r, r3], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=string, numpy=array([b'DEF', b'G', b'', b'HI'], dtype=object)>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_encode(r3, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4, 6), dtype=int32, numpy=\n",
       "array([[   67,    97,   102,   233,     0,     0],\n",
       "       [   67,   111,   102,   102,   101,   101],\n",
       "       [   99,    97,   102,   102,   232,     0],\n",
       "       [21654, 21857,     0,     0,     0,     0]], dtype=int32)>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.to_tensor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sparse tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = tf.SparseTensor(indices=[[0, 1], [1, 0], [2, 3]],\n",
    "                    values=[1., 2., 3.],\n",
    "                    dense_shape=[3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 1]\n",
      " [1 0]\n",
      " [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 1., 0., 0.],\n",
       "       [2., 0., 0., 0.],\n",
       "       [0., 0., 0., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "s2 = s * 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unsupported operand type(s) for +: 'SparseTensor' and 'float'\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    s3 = s + 1.\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[ 30.,  40.],\n",
       "       [ 20.,  40.],\n",
       "       [210., 240.]], dtype=float32)>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4 = tf.constant([[10., 20.], [30., 40.], [50., 60.], [70., 80.]])\n",
    "tf.sparse.sparse_dense_matmul(s, s4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 2]\n",
      " [0 1]], shape=(2, 2), dtype=int64), values=tf.Tensor([1. 2.], shape=(2,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "s5 = tf.SparseTensor(indices=[[0, 2], [0, 1]],\n",
    "                     values=[1., 2.],\n",
    "                     dense_shape=[3, 4])\n",
    "print(s5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indices[1] = [0,1] is out of order. Many sparse ops require sorted indices.\n",
      "    Use `tf.sparse.reorder` to create a correctly ordered copy.\n",
      "\n",
      " [Op:SparseToDense]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.sparse.to_dense(s5)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 2., 1., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s6 = tf.sparse.reorder(s5)\n",
    "tf.sparse.to_dense(s6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 6), dtype=int32, numpy=\n",
       "array([[ 2,  3,  4,  5,  6,  7],\n",
       "       [ 0,  7,  9, 10,  0,  0]], dtype=int32)>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set1 = tf.constant([[2, 3, 5, 7], [7, 9, 0, 0]])\n",
    "set2 = tf.constant([[4, 5, 6], [9, 10, 0]])\n",
    "tf.sparse.to_dense(tf.sets.union(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[2, 3, 7],\n",
       "       [7, 0, 0]], dtype=int32)>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.difference(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=int32, numpy=\n",
       "array([[5, 0],\n",
       "       [0, 9]], dtype=int32)>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.intersection(set1, set2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2.,  4.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.assign(2 * v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[0, 1].assign(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  0.],\n",
       "       [ 8., 10.,  1.]], dtype=float32)>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[:, 2].assign([0., 1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'ResourceVariable' object does not support item assignment\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    v[1] = [7., 8., 9.]\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[100.,  42.,   0.],\n",
       "       [  8.,  10., 200.]], dtype=float32)>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.scatter_nd_update(indices=[[0, 0], [1, 2]],\n",
    "                    updates=[100., 200.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[4., 5., 6.],\n",
       "       [1., 2., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_delta = tf.IndexedSlices(values=[[1., 2., 3.], [4., 5., 6.]],\n",
    "                                indices=[1, 0])\n",
    "v.scatter_update(sparse_delta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor Arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = tf.TensorArray(dtype=tf.float32, size=3)\n",
    "array = array.write(0, tf.constant([1., 2.]))\n",
    "array = array.write(1, tf.constant([3., 10.]))\n",
    "array = array.write(2, tf.constant([5., 7.]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=float32, numpy=array([ 3., 10.], dtype=float32)>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.read(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[1., 2.],\n",
       "       [0., 0.],\n",
       "       [5., 7.]], dtype=float32)>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.stack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=float32, numpy=array([2., 3.], dtype=float32)>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean, variance = tf.nn.moments(array.stack(), axes=0)\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=float32, numpy=array([4.6666665, 8.666667 ], dtype=float32)>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom loss function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by loading and preparing the California housing dataset. We first load it, then split it into a training set, a validation set and a test set, and finally we scale it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error = tf.abs(error) < 1\n",
    "    squared_loss = tf.square(error) / 2\n",
    "    linear_loss  = tf.abs(error) - 0.5\n",
    "    return tf.where(is_small_error, squared_loss, linear_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAEDCAYAAAB0/A4MAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA9kklEQVR4nO3dd3gU1dfA8e9NiJBAaNIFBJGAdERFBKQIgorYuwIqgqivFLGg2EUFBREFBBsgSlERUKRIFZSiKPxQFKT3TkhCCin3/eMkEGo2yc7OlvN5nn2SbCY7Z7LJnp25555rrLUopZRSynfC3A5AKaWUCjWafJVSSikf0+SrlFJK+ZgmX6WUUsrHNPkqpZRSPqbJVymllPIxTb5K+RFjTEtjjDXGlPLR/roYYxJ8sS+l1AmafJXKJ2PMGGPMD2e4/7LMRFrFhbCUUn5Mk69SIcAYc57bMSilTtDkq5SPnOmSsjGmSuZ9l52y+ZXGmFXGmGRjzEpjTKNTHusqY8wiY0yiMWanMWakMaZotu8vzLzvXWPMfuCXXMTZ3RizwRhzLPPjI2f4/vrM2PYbY2YbYwpkfq+uMWaeMSbOGBNvjFltjGmVm9+TUqFAk69S/uld4FngMmATMMMYEwWS4IA5wHSgPnAr0AD47JTHuB8wQHOgkyc7NcbcAnwIDAXqAO8DI4wxN2Z+/zJgOPAqUANoA8zK9hBfAbuBK4CGwCtAsofHrFTIKOB2AEoFifZnKFzKz5vb1621swGMMQ8CO4B7gU+Ap4FJ1trBWRsbY3oAfxpjylhr92Xevdla+1Qu99sX+MJa+2Hm1+szz7qfBb4HKgNHgenW2nhgK7A6289fCLxrrf038+sNudy/UiFBz3yV8o6fkbPP7Ld78/F4S7M+sdYmAGuAWpl3NQLuN8YkZN04cVm5WrbHWJmH/V7C6Zeol2Tb909Iwt1sjPnSGNPZGBOdbdshwCfGmPnGmBeMMTXzEINSQU+Tr1LekWit3ZD9hpytZpeR+dFkuy8iD/sKQ86AG2S71QeqA6uybXc0D48NcKalzixA5tnupcCdwDagH/CvMaZC5vdfQRL1VOAq4H/GmIfyGIdSQUuTr1K+sz/zY/ls9zU4y7ZXZn1ijCmMjL/+k3nXH0DtU5N95i0pnzH+AzQ75b5mwNqsL6y1adba+dbafkA9oDDQIdv3/7PWDrPW3gB8CnTNZ0xKBR0d81XKdzYA24FXjDHPAVWA/mfZtn9mlfIu4CXgGFLMBDAQWGaM+QgYBcQDNYEbrbXd8xnjO8DXxpiVSFFXe+A+pKgLY0wH5NL2z8AhoBUQDfxjjIlECsW+BrYAZZHEvTyfMSkVdDT5KuUj1tpUY8zdwAikSGkV8DxwWoMO4DlgMFJR/DfQwVp7NPNx/meMuRp4A1gEhCMV0d95Icapxpj/QwqvhiLju49Za7/P3CQWuBl5QxAFbAS6WmsXZ84lLgGMBcoBBzOPrW9+41Iq2BhrzzS8o5RSSimn6JivUkop5WO5Sr7GmOqZXW3GOxWQUkopFexye+Y7HPjNiUCUUkqpUOFx8s0sFIkF5jkWjVJKKRUCPEq+mQ3bXwNy26pOKaWUUqfwdKrR68Cn1trtxpizbmSM6QZ0AyhUqFCjypUr5z9CP5WRkUFY2Nnfu1gLBw8WpFSpFB9G5R05HVugC+bj2759O9ZaQvl/L9AF4vElJ4dTsGAGxpx79kwgHlturF+//oC1trQn2+aYfI0xDZCVSxrmtK21djQwGqBGjRp23bp1nsQQkBYuXEjLli1z3C4xEaKinI/Hmzw9tkAVzMfXsmVLYmNjWbVqlduhOCaYnz8IvONbtgxq14bo6Jy3DbRjyy1jzFZPt/XkLUhLpBPPNmPMHmTC/G3GmD/yFF0IOXwY6tWDtDS3I1FKKWeMHQu7drkdReDx5LLzaGBitq/7Ism4hxMBBZMSJeCvv6CA9hFTSgWpkSPdjiAw5Xjma61NtNbuyboBCUCytXZ/Tj+rIDUVnn9exoCVUiqY3HQT/P2321EEplyfk2UuGaY8VKQIVKokl54j8rJ4nFJK+an33oMgru1zVPCWnfkJY6BHD9izx+1IlFLKe779FooX12G1vNLk6wNJSXDDDfJRKaWCwfLlOpyWH/qexQciI2H1ajkLVkqpQJeaCoMGuR1FYNMzXx9JT4e77pJ5v0opFaishYYNYedOtyMJbHrm6yMFCkDXrjo+opQKbMbAr79C0aJuRxLY9MzXh9q0gRUrdJxEKRW43npLX8O8QZOvj733HuzXGdJKqQCUng7nnQeFC7sdSeDTi6A+ZIyU5yulVCA6cACe0rXtvELPfH3MWmjbFrZtczsSpZTyXEoKtGgBCQluRxIc9MzXx4yB4cOhYkW3I1FKKc8VLAhr10IQrwjoU/prdEFMDHz9tU47UkoFhmPHoHNnmd+rvEOTr0vWroV9+9yOQimlPHPrrXL2q7xDLzu75NVXZbEFa7XzlVLKv61ZAx07uh1FcNEzXxfdcAOsXOl2FEopdXaHDsmyqBkZbkcSXPTM10WTJsmqIEop5a9KloTZs92OIvjoma+LiheHTz6BdevcjkQppU63fTtcd512tHKCY8k3NlZXjvdEiRJauq+U8k8VKsC772pdiic+/zx32zv2sr9vXyF69pR2ZOrsbrsNypWDuDi3I1FKqROOHIHp06F2bbcj8W8ZGdCvHzz0UO5+ztFzrmHD4KabID7eyb0Evv79YdYst6NQSqkT9u6VKZHq7BITZanYt9+G8PDc/axjybdSpURKloQZM6B5cxk7UGc2dCjceafbUSillEhPh2rV4IUX3I7Ef+3ZA61awTffyPKKP/6Yu593LPlGRqazfLl0c1q9Gho31mk1Z2MMjBsHX37pdiRKKQVz5kCnTm5H4b/++kty2ooVUKWKrG987bW5ewxHLztffDEsXQotW8Lu3XIGPHWqk3sMXJdfDldd5XYUSiklFc4jR7odhX+aNUteq7dtgyuvhGXL8jYu7nidbdYcsS5dIClJWpS9+66Wrp/qkktONC5XSim3LF8uPQiKFnU7Ev8zYoQ0R4qPl6HC+fOhbNm8PZZPJrmcdx589hm8+aYk3aefhu7dtUn3qX75BRYudDsKpVQoi4qSKZDqhPR06N0bHn9cqpv794cJEyAyMu+P6bMOV8ZIOfbFF8tYwscfw6ZNMlitXZ7EHXfIR+33rJRyw4EDUmhVt67bkfiPhAS45x744QeIiJDc1blz/h/X5+0d7rhDzu7KlIF586BJE0nCSvz4I/To4XYUSqlQ9PXX8N57bkfhP3bskFqlH36QqwE//eSdxAsu9XZu3FjGFTp0gL//lq+nTdOCI5An+sor3Y5CKRWKevTQepwsf/wBN94Iu3ZB9eqSgGNivPf4rjU2rFJFxjjbtZNLHa1byzX0UBcdLZc5vv3W7UiUUqFk2DA5CdIhL/k9NG8uiffqq2XWjjcTL7i8sEKxYvJuokcPSEmBe++F117Td17p6XK5QymlfKV9e2jQwO0o3GUtDBkCt9wi3as6dZI5z+ef7/19ud7Sv0ABGD5cxhmMgZdflgNOSXE7MvdUrQo9e0pvVaWUctqKFTKmeeGFbkfintRUORF86ilJwm+8AWPGyBRQJ7iefEGSbq9ecqpfuDCMHw9t2sjl6FC1ebNcig/1qwBKKefNng3//ed2FO45ckRqkEaNkmQ7caK01nTyErxfJN8sN94IixfDBRfAkiVSeBSqa91WrSrjDDr+opRyUkYGvPhi6Ba8btkixz5nDpQuDQsWyGIJTvOr5AvQsKFUQjdsCBs3SgJesMDtqNxx7Bg88oguy6iUcs4118isk1C0bJnMtlm7FmrVktzTpIlv9u13yRfkzHfxYujYEWJjpWF1bhcqDgaFC0sRREaG25EopYLVpEmSeELN5MmyKtG+fdC2rcy+qVrVd/v3y+QLknimTJHB77Q0Wai4X7/QSkTGSC/slSt17Fcp5X2DBskYZygNb1krrY7vuguSk6FbN1n61tedFv02+YIsTvzuu/DRR/L522/LLywx0e3IfGvQIFk7UimlvCUtTRJR4cJuR+I7x47Bgw+eKKYaPFjyS0SE72Px6+SbpXt3mDlTVtn45hu5VBAqycgYuQJQrpzbkSilgsm+ffDsszLdMxQcOiRDmGPHyuIRU6ZAnz7unfUHRPIFuSa/dKl0xlqxQgbJ16xxOyrfufNO+P13t6NQSgWDQ4dkzd60NLcj8Y3//pPi3UWLoHx5+PlnuPlmd2PyKPkaY8YbY3YbY+KMMeuNMV2dDuxMatWS6rQrr5SFjJs2lYWNQ8E778Cll7odhVIqGJQsCatWhcZZ788/S8747z+oX19O3ho1cjsqz8983wKqWGuLAh2BN4wxroRftqwsYHzXXbKg8Q03yALHwa5KFVnxKFTnPSulvGPTJujaNTSKrMaNk4ZNhw5JE40lS6BiRbejEh4lX2vt39barIaPNvNWzbGochAZCV99JQsaZ2TIAse9egX/fNgjR+QNh1JK5VW5cvDww25H4aysxiGdO0vbyJ49YepUKFLE7chOMNbDOSzGmBFAFyAS+BO42lqbcMo23YBuAKVLl240efJkrwZ7JrNnl+Xdd2uQlhZGkyYH6N//H6KinM/CCQkJFHHhmUxPN6SkhDl6jG4dm68E8/H16tWL9PR0PvjgA7dDcUwwP3/g7PEdPhzB7t2R1KoV58jj58QXz92xY2G8/XZNFiwoQ1iY5Ykn/uOWW3Y5us8srVq1Wmmtvcyjja21Ht+AcKAZ0B+IONe2MTEx1lcWLbK2ZElrwdr69a3dvt35fS5YsMD5nZzBG29YO3iws/tw69h8JZiPr0WLFrZ+/fpuh+GoYH7+rHX2+JYuldcQtzj93O3da22TJpILoqOtnTnT0d2dBvjdephPc1XtbK1Nt9YuASoCPXL1lsBBV18thVjVq8Pq1XDFFdKYIhg984yUxyulVG6kpUnh0QsvuB2JM9aulVkwS5dC5crSsap9e7ejOru8TjUqgItjvmdSvbok4BYtYPduSchTp7odlfdFRMgf1cCBbkeilAok778vjYqC0U8/SU/mLVvg8sulR3Pdum5HdW45Jl9jTBljzN3GmCLGmHBjTDvgHmC+8+HlTsmSsjJF587SBevWW6WDSbC1ZrzoImjZ0u0olFKBpGdPWa822Hz8scxZjouD226DhQsDoymRJ2e+FrnEvAM4DLwL9LLWTnMysLw67zxZhGHAAEm6ffvCo49KxVuwKF9e3tUtXux2JEqpQDB2LPzvf1CsmNuReE96Ojz9tPRmTk+H556TxRKiotyOzDM5TrG21u4HWvggFq8xBp5/Hi6+WM6CR4+WuW1ff+375tlOOXgQPvsMmjd3OxKllL8rWTK4Eu/Ro3D//TK0WKCA9GcOtOlTAdNeMi/uvFPWAi5TBubOlQWTN292OyrvqFRJzvCDfW6zUip/1q2TZkTV/KpKJ+927ZLanqlT5WRq9uzAS7wQ5MkXpLpv+XKoXRv++edENVwwSEqCOnVCb5UnpZTnnnoKNm50OwrvWL1aXsNXrpTal6VLoXVrt6PKm6BPviCtGX/5RVa02L9fVkWaONHtqPIvMlLGfQNljEMp5Xs//CCzQQLdjBnSz3/HDvm4fDnUrOl2VHkXEskXZLxjxgwpvkpJgXvugTfeCPxK6FKl5DgOHXI7EqWUP0lMhGbNguPK2LBh0LGjjPXee68MI5Yq5XZU+RMyyRdkYH7ECHjvPSnKyur9mZKS88/6s8qVQ2dpMKWUZ6KipNg0kK+MpaXBE0/INKmMDHjlFRg/HgoVcjuy/Aup5AuSdHv1ksH6woXhiy9kreCDB92OLO86dZI/zCNH3I5EKeUPjh6FTz+VZVgDVVycnO0OHy5TSMePh5dfDp7VmEIu+Wbp2FHGSytUkI9XXhnYy/W9+Sb8+qvbUSil/MGhQ3DggNtR5N22bXLJfOZMubw8bx7cd5/bUXlXyCZfgIYNZWHlhg1hwwZpT7ZwodtR5c2wYdLlRSkV2pKT4fzz4dln3Y4kb377Tfrzr1kDNWpI2+BmzdyOyvtCOvkCXHAB/PyznAkfPiwV0Z9/7nZUeTNuHAwa5HYUSik3zZkj46SB6NtvZQ7v3r0yK2Xp0uCZn3yqkE++IAssT5kiqwWlpsJDD0mHrIwMtyPLnTZtJHalVOjq2BFGjXI7ityxVk4cbr9d+hc8/DDMmgUlSrgdmXM0+WYKD5dFGEaOlM/fegvuukv+EAJFhQpSuf31125HopRyw7BhMH26rH4WKI4dg0ceOXGZfOBAWSzhvPPcjctpOfZ2DjWPPiqdU+64A775Rgb+p0+HsmXdjswzGRmyrJZSKvS0axdY03AOH5az3fnzJe7x42VlolCgZ75ncO21Ujl84YVSkNW4Mfz1l9tReaZSJVnpY+9etyNRSvnS7NlyknDhhW5H4pmNG6XIdf58iXvRotBJvKDJ96xq15b2ZY0bw9atsijD7NluR+WZhAS45hqpelRKhYY5c2RubCD45ZcT0zvr1pWTnCuucDsq39Lkew5ly8qqSHfeCfHxsjLIyJFuR5WzIkVk7c5AuvyklMq7o0elZqVyZbcjydlXX8liCAcOQPv2sGRJYMTtbZp8cxAZCRMmwAsvyPJ9jz0GvXv7/1J+YWHQpUtgNw5RSuUsPh4aNPD/K13WwtixF3LffVJk9fjj8P33ULSo25G5Q5OvB8LCZPGCMWOkinDoUHjppTokJLgd2bk98YSs6KSUCl7R0bLUnj9f6UpJgQcegDFjqhIWBu+/Dx9+KP32Q5Um31zo3Bl++knmnv36aymaN5flrfzVZZfJupf//ON2JEopJ6xfD88959+LJxw4ID0IvvwSChVKZ9o0ePJJt6NynybfXGrRQtqdVayYyKpVUpD1xx9uR3V2mzbBrl1uR6GUckKpUjI7w1/9+6+8Ri5ZAhUrwgcf/EmHDm5H5R80+eZBTAx8+OEfXH21JLbmzWHaNLejOrP775fiBl3xSKngsmGDrMbWurXbkZzZ/PkylWjTJrj0Upk9cvHFfj5W50OafPOoWLE05syRS9GJiXDLLTBkiBQV+JupU+Gpp9yOQinlTatW+e9CMJ99Jg0/YmPhppukf36FCm5H5V9CeLg7/woWlEUYqleH/v0lwa1fDx984F/t3Tp2lJtSKjgcOyadofxNRob0xR84UL7u2xfeflta9qqT6ZlvPhkj05AmTZJkPGqUzAf2p8u84eFS9HDvvYG3WIRS6nT33ON/Z72JidITYeBAec0ZNQreeUcT79lo8vWSO++Uf4bSpaUi+qqrYPNmt6M6oUwZ6NpV3iwopQLb2LFSa+Iv9uyBli1lScCiRWHmTOjWze2o/JsmXy+68kopKqhVC9aulSq/pUvdjkoYI4UZkycH1kpNSqkT0tKge3f53F/OKNeskde6336TvgJLl0Lbtm5H5f80+XpZ1aqyKEPbtrB/vywIPWmS21Gd8NdfuuiCUoGsbVsoXNjtKMSsWdC0qaz+lv3kQ+VMk68DihWDGTPkHWpKCtx9t3TI8odK6Ndfl6rD+Hi3I1FK5cbRo/LG/vbb/WP4aMQIqW+Jj5e1z+fPl+Et5RlNvg6JiJBFGAYPln+UF1+UXsspKW5HJgl4/Hi3o1BK5camTdIL2W3p6dCrl/RmzsiQmR5ffSV98JXndKqRg4yBPn2gWjWpNB43Tha6nzIFzj/fvbhefjm0e6oqFWiSk6FOHakedlNCglRa//CDnGB88gl06uRuTIFKz3x94KabYPFiudz7888yNrJ+vXvxFCggLTEff9y9GJRSnnv3XVnQxU07dkiF9Q8/QMmSMHeuJt780OTrI1nt1Ro0kLZwTZrAokXuxVOzpkw9Ukr5v379TlQ5u2HlSlnsftUqaSq0bBlcfbV78QQDTb4+VLGinAHfeCMcOiRVi2PHuhNLVBTUri2XjfyhEEwpdWZPPglbt7q3ctG0aZJod++Wj0uXSgJW+aPJ18eKFIHvvoPevSE1VYqw+vd3p/NUeLhc/k5M9P2+lVKeuf56eePua9ZKwegtt8hrRNaSqm7WqwQTTb4uCA+XRRhGjJDPBwyQIgZfN78ID4dBg2S/x475dt9KqXNLS5OmOO3awXnn+XbfqanQo4f0ZrZWpkp+/rnv4whmmnxd1KOHzAeOjpZ/slat3GmA0asX/PKL7/erlDq7/ftlbNXXjhyR+bujRkm/+kmTpH+9P8wtDiaafF3Wrp1MnL/wQinIatwY/v7btzGMGyeJXynlHxISoEQJuULmy6S3ebP0pf/pJ+lTv3Ch9K1X3qfJ1w/UqXMi8W7dKn/8s2f7bv9hYXIG3qeP7/aplDq7yZNlPr4vLV0qr0Fr10qLyOXLZVqkckaOydcYU9AY86kxZqsxJt4Y86cx5jpfBBdKypaFBQvgjjsgLk4u+3z0ke/236yZjO8opdz30EMyzuorkybJ1a/9+2UWxq+/Sp965RxPznwLANuBFkAx4EVgsjGmioNxhaTISJg4URajTk+XMeE+feRzpxUrJmPPb76pU4+UctOgQTVYtUo6SDnNWin4vPtuaX3bvbtcBStWzPl9h7ock6+19qi19hVr7RZrbYa19gdgM9DI+fBCT1iY/DN8/rn88733Htx6q4wBOS0qSgos0tK0skIpt9x22w6frAyUknJiqqMxMq1o5EjfJH2VhzFfY0xZIAbwcVlQaOnSBebMkaKL6dOlrduOHc7uMzwcnnoKdu+O1Lm/SvlYSookwCpVjjo+pefgQbj2Wim2jIqS3gN9+mhFsy/lqr2+MSYC+BIYa6399wzf7wZ0AyhdujQLFy70Rox+KSEhwSfH9/77kfTrV5dVq6Jo2DCFN99cQ/Xqzp4GT5hQlYSEldSqFZzrDvrquXNDbGws6enpQXt8ELzPX1xcATZuLEeNGs4e344d8pqyY0cUpUqlMGDAGooVS8AXv9Jgfe7yxFrr0Q05S54I/AhE5LR9TEyMDWYLFizw2b7277e2eXNrwdqoKGunTXN2f1nHlpLi7H7c4svnztdatGhh69ev73YYjgrG52/PHmv37pXPnTy+hQutLVlSXkvq17d2+3bHdnVGwfjcZQf8bj3MqR5ddjbGGOBToCxwm7U21aH3AuoMSpWSeXcPPCBt3m6+WcaCnSyMmjoVHnvMucdXSp0wbx58/LGz+xg3TiqZDx2CDh1gyRJ32lYq4ell55HAJUAba62PmyAqkEKosWMhJgZefFHGZ9avhw8+cGZt3uuvlzEhpZSz0tJkvW+nZGTInOGsqUu9eskSheHhzu1T5cyTeb4XAt2BBsAeY0xC5u0+p4NTJzNGKhMnTJBk/NFHMh/4yBHv7yur4OO++3zfc1qpUNKuHaxZ48xjJyVJYn/jDZlJMXy4XDXTxOu+HM+ZrLVbAa2B8yN33y3tKG+6SSqimzaVBa6rVPHufqKi5FK3NlNXyjkTJ8rQkrft2yevEcuWnegf37699/ej8kbbSwaoJk2k/dsll0gv6MaNnWnC3r699HfdtMn7j61UKNu+HR5/XBKvt6f4rF174jWhcmVZOEUTr3/R5BvAqlaVNnBt28q73Fat5N2tt23Z4s5qS0oFs5IlZa1cbyfen36SN+dbtsDll8ub9Lp1vbsPlX+afANc8eLSDq5bN0hOhrvu8n6LyIcflnfRmzd77zGVCmVLlsC2bdCmjXcfd/RouO466Q9/++1y1apcOe/uQ3mHJt8gEBEhxVeDB8u76BdegAcfhGPHvLePVat04QWlvGXbNti923uPl54u/5/du8vnzz0niyVERXlvH8q7HJikotxgjEw/uugiqVAeO1YuO337LZx/fv4f/9JL4Ztv5B9bKyWVyrvNm707tejoUfmfnzZNph2OGiWrIin/pme+Qebmm2HxYqhQARYtkrGf//7z3uO3bi3zi5VSuXfsmCxOHxvrncfbtQuuvloSb/HiMvtBE29g0OQbhC69VIos6teXxHvllZKI88sYKeiKicn/YykVajIy5Mx0xQpJlPm1ahVccQX88QdUqwZLl0rRpQoMmnyDVMWKUtTRoYO0k2vbVtrL5VfZsjBrlkzWV0p5bvJkWTXMG9XNP/wAzZrBzp0yz3/ZMqhZM/+Pq3xHk28QK1JEejT36gWpqdC5s3TIysjI3+PWrCn/+Eopz91xhxRD5oe18P770jwja6x33jxnmnQoZ2nyDXLh4dJObvhw+XzAACn2yE/LyCpVpLnHJ584u7iDUsHAWmmmsXlz/pJkWhr83//Jm+mMDHj1VfjiC2k1qwKPJt8Q8dhjcqkqOlqmILRuLY058iosDDZskFWWlFJnZ4w006hcOe+PERcHN94ob6LPOw++/BJeesn7DTqU72jyDSHt20ubucqVZYyocWNpTZkXBQrA229DQgLs2ePdOJUKFgcPyrS/Nm3y3iN961YZ1501S86c5893dhUk5RuafENM3bpSCX3FFTIP+KqrpB1dXo0dm7+fVyqYxcbKWWte/fabvEn+6y+ptVi2TBKxCnyafENQuXKwYIG0n4uLk3Z0o0bl7bGeeUZWPjp61LsxKhXo1qyR+fb/9395+/lvv4UWLaSveuvW0se9WjXvxqjco8k3REVFydhvv37SterRR2UaRHp67h9r3z6ZS5yW5v04lQpUH38Mf/6Z+5+zFgYOlDfHSUnSW33WLChRwvsxKvdoe8kQFhYmizBUry4LMwwZAhs3QvfuuXtPVqaMXA4roH9NSgFyuXnYsNz/3LFj8O67NfjxR/l64EB4+mktrApGeuarePBBaUtXooS0qevZsyE7d+buMQoXlurLKVOciVGpQLFunTS3ye00vMOHpSjyxx/LExkpl52feUYTb7DS5KsAaUu3dKmMKf33XzSNG+f+klnnztCunTPxKRUIrIUaNaTxRW6S5saN0od9wQIoWTKFRYvg1ludi1O5T5OvOq5GDbl8XK9eLDt3QvPm8P33nv98tWoy9eiZZ7T5hgpNjz0Gs2fnrvHFkiVS0bxuncxGGDHiDy6/3LkYlX/Q5KtOUqoUvPPOau6/XyqYb7oJhg71PJmWLAkNGjgZoVL+68UXZZUhT335JVxzjcwHvu46ScRly6Y4F6DyG5p81WnOO88ybhy89pok3d69pT2eJ9XMERHSAGDuXOmApVQo2LEDevSA8uUhMjLn7a2FV16B+++XIqsnnoDp06FoUcdDVX5Ck686I2PkXfxXX8kltJEjpYjkyBHPfn7HDti/39kYlfIX558vLSQ9GedNTpak++qrMuNg2DD44AOdLRBqNPmqc7rnHmlnV7q0jGU1bSqdsXLy4IMyjvXbb46HqJSrvvoKtm+Ha6/Nedv9+6XV5Fdfyapj06fnvQmHCmyuvdeKi4tj3759pKamuhVCvhQrVox//vnH7TAcceqxlSkTweLFZbjllqL8/bck1enT5eO57NsHb7wh04/Cwx0OWimXJCfLcEtO/v0XbrgBNm2S9bZ/+AHq13c+PuWfXEm+cXFx7N27lwsuuIDIyEhMAE5ki4+PJzo62u0wHJH92Ky1JCUlsXPnTubOhc6dizJ3LrRsCePGyRqlZ1OunMwbTkiQMa4g/XWpEJWcLAuVPPRQztvOnw+33SbNNxo1klkE5cs7HqLyY65cdt63bx8XXHABUVFRAZl4Q4kxhqioKC644AISE/fx44/wyCPywnPnnfDWWzlXQg8cCJMn+yZepXxl61Y5e83Jp5/K/PfYWLj5Zli0SBOvcin5pqamEulJSaDyG5GRkaSmphIRIYswvPuuFJc8/7y88z927Ow/+8or0p/2XNsoFUjWr5e2rO+9d/ZtMjLgueega1eZKfD009K1qnBh38Wp/JdrBVd6xhtYsj9fxsgiDFOmyAINY8ZIscmhQ2f+2fBwufTcsKGufqSCwwsvyDJ/Z5OYKFeGBg6Uv//Ro2HQIKluVgq02lnlw803w88/yyW0RYukPd5//5152yJFpIFA4cLa/UoFrrQ0WYZz8mSoV+/M2+zZIzUR334LxYrJikSPPOLTMFUA0OSr8qVRI1ixQqo216+XpQUXLz7ztiVKyBSL117zbYxKecuPP0KfPmefz7tmzYkpdlWryhq8bdr4NkYVGDT5qnyrWFES7g03yKXna66BL74487bt28vawUoFmrQ06NgRRow48/dnzpR58Nu2yVWgZcugVi3fxqgChybfXGrZsiVPPPGE64+Rk8OHD1O2bFk2btyY47a33347Q4YMydf+oqOzliOE1FTo1EmWGDz1EnPJkrL+74MPSrWoUoEgLQ0uvxwOHIDzzjv9+8OHSwe4+Hi4+26ZWlSmjO/jVIFDk2+QevPNN7n++uupVq1ajtu+/PLLvPHGGxzxtHfkWYSHyyIMH34ohSWvvy59npOTT97OGEm+FSrka3dK+YS10vpx9mxZeCS79HTo1Ut6M2dkSEvWL7+EQoVcCVUFEE2+QeRY5lyexMREPvnkEx5++GGPfq5u3bpcdNFFjB8/3itxPP64zH+MjoaJE6F1a+l2ld3VV8uZ75tvemWXSjlmwACp6D/1TDY+Xlb9ev996XCVtRiJVjQrT+ifSR5kZGTw6quvUqpUKcqUKUPfvn3JyMgAznxJuUuXLnTo0OGk+9LS0ujZsyclSpSgRIkSPP3008cfA6Sz1KBBg6hWrRqRkZHUrVv3tOTYsmVLevToQd++fSldujRNmzYF4McffyQsLOz41wCDBg3CGHPa7aWXXgKgY8eOTJgwwWu/o+uuk+4/lSvD0qVShLJ27cnblC4tcyWV8mePPipjvdnt2CHrXc+YIUMpc+fCAw+4E58KTH6RfI1x55ZXX375JeHh4fz66698+OGHDB06lEmTJuX6MTIyMli6dCmjRo1i9OjRDB069Pj3+/fvz6effsrw4cNZu3Yt/fr1o3v37syYMeOkxxk/fjzWWhYvXsy4ceMAWLx4MY0aNTppbm6PHj3YvXv38dtTTz1FuXLl6NSpEwBXXHEFK1asICkpKY+/ldPVrQvLl8tY2ZYtUoTy008nvl+smLSnnDIFVq3y2m6V8opNm+C++2TFopIlT9y/ciVccQWsXg0xMVJYlZs1fJUCFxdWCGS1atWif//+REdHExMTw8cff8y8efO45557PH6M8uXLM2zYMIwx1KxZk/Xr1zNkyBD69OnD0aNHGTJkCHPmzKF58+YAVK1alRUrVjB8+HBuuOGG449TtWpVBg8efNJjb926lfKn9K+Ljo4+3q954MCBTJgwgYULF3LxxRcDUKFCBVJTU9m1axdlvFgpUq4cLFwoBVjffitnxCNGQLduJ7YJD9e5v8r/VK4sBYTZ36hPnSoJOTHxxFze7IlZKU/5xZmvte7c8qreKbPrK1SowL5TBzVzcOWVV550ZtqkSRN27txJXFwca9euJTk5mfbt21OkSJHjt5EjR55WvdyoUaPTHjspKYlCZ6n4eOuttxg2bBgLFiygRo0ax+/PavfpzTPfLFFR0pTgueekQKV7d+jbVz4HGTerVw8+/liqSpVyk7Uyl3frVjnDzbpv8GC49VZJvJ07SwGWJl6VVx6d+RpjngC6AHWBCdbaLg7G5PciTlk/zBhzfLw2LCwMe0pmz+2yiVmP9f3331O5cuVz7rvwGRrFlipVisOHD592/4ABA/joo49YtGjR8TPeLIcye0OWLl06V7F6KixMFmGoXl2S7+DBsHEjjB9/ouvVli3SfrJYMUdCUMojxkDbtnDBBfJ1aqpUM48eLV+/+aa8kdQOuSo/PD3z3QW8AXzmYCxBoXTp0uzevfuk+1avXn3adsuXLz8pSS9btowKFSpQtGhRatWqRcGCBdm6dSsXX3zxSbcLL7wwxxgaNmzI2lOqm15//XVGjRp10qXm7P766y8qVKhA2bJlPT3UPHnoIZgzB4oXl0t4V18Nu3bJVI4BA+TMd948R0NQ6qymTpUahOuuk+lCsbFw/fWSeAsVkis4/fpp4lX551HytdZOsdZOBQ46G07ga926NTNnzmT69OmsW7eOPn36sH379tO227VrF7169WLdunV88803vPPOO/Tu3RuQ8dm+ffvSt29fPvvsMzZs2MCqVav46KOPGJ319vsc2rVrxz///MPBg/J0DRgwgPfff5+JEydSuHBh9uzZw549e0jONgF38eLFtG/f3ku/hXNr1UqKVKpVgz/+kEt7WQVXO3dKD2il3HDRRVCliny+ebN0rJo7V6YZLVx47vWrlcoNrxZcGWO6Ad1AzgAXLlx4xu2KFStGfHy8N3ftM+np6Rw7doz09PTjx5CamkpaWhrx8fHccccd/P777zz44IMAdO3alQ4dOnDw4MHj26enp3PnnXeSlJRE48aNMcbwwAMP0LVr1+PbPPPMMxQrVoxBgwbRo0cPoqOjqVevHj179jzpcY4dO3ba77JKlSo0atSIMWPG8MgjjzBo0CDi4uJOmnoEMH36dFq2bElycjLfffcdU6ZMIT4+/qRjyy45Ofmsz2leDB4cwYsv1mbNmuI0aZLOSy+tpUmTg7RoAZ98UoSiRVMpUybFa/vLkpCQ4NXj8CexsbGkp6cH7fGBM89fbGwE06ZVoFOnrRgDw4cXpX//OsTGnkeVKkd56601JCUl44tfazD/fQbzseWatdbjG3LpeYwn28bExNizWbt27Vm/Fyji4uLcDuGcZs6caWNiYmxaWlqO23744Ye2bdu2x78+27E58bwlJ1t7//1SAhcWZu3QodZmZFg7bJi1M2d6fXfWWmsXLFjgzAP7gRYtWtj69eu7HYajnHj+Dh2yduxY+XziRGsLFpS/yWuvtTY21uu7O6dg/vsM5mOz1lrgd+thPvWLamflfe3bt+fxxx9nx44dOW4bERHBBx984IOoTlewoHQGevVVac+X1aqvRw9ZhGHevBNV0Up5m7VSCGitNMl44w3pzZySIs01ZszQAkDlDJ3nG8SefPJJj7brln3SrQuMkUUYqleHLl1kHvCmTTBhglRD16x5ovJUKW+yVtqgGiN/e+PGyedDhpw+x1cpb/J0qlGBzG3DgXBjTCEgzVqrszKV19xzjzQ2uPlmWYC8eXPpEV22rBS7tGzpcoAqqHz5JVxyifzd3Xwz/PyzzEmfMOH0dpJKeZunl537A0nAc8D9mZ/3dyooFbqaNpWWlDVrwl9/SU/omTOlsb12wVLeVKSILPhx5ZWSeCtUkHWpNfEqX/B0qtEr1lpzyu0Vh2NTIeqii2Qxhmuugb174c47Za3U/fvht9/cjk4Fuv/9Ty4vFy8urSI3bICGDWHFCrj0UrejU6FCC66UXypeXM54u3aV9YDvuANeeAEWLHA7MhXoChWSxRHatoVDh+DGG+XMV+sKlC9p8lV+KyJCOgsNGiSFL598Av/+K004PCjiVuoke/bA88/D2LEwbJi0jezdG777Ti5BK+VLmnyVXzMGnn5aVo+JjITPP4cHH4Tff3c7MhVoChSQKydvvikraY0YIVXN4eFuR6ZCkSZfFRBuuUUuDZYrJ2N0zz4rL5xHjrgdmfJ3KSmypGX79tLWNDpa5u/26OF2ZCqUafJVAeOyy6Qopl49WL8e+veHn35yOyrl7/77T6aurVwJF14Iv/4K7dq5HZUKdZp8VUCpVEnGfK+/HpKS4N57pRragWWIVRC47jqZSrR/v0xbW74c6tRxOyqlNPnmWseOHSlRogQPPPCA26GErOhomDYNnnxSima+/lraAuo8YJXFWhg1SpavPHpUquUXLJCGLUr5A02+udS7d2/GjRuX65/bvn07LVu2pFatWtSvX58pU6Y4EF3oKFAA3n8fPvgAwsKkiKZmTXmhVaEtPV3m6z76qPQLf/55mDhRCvaU8heafHOpVatWREdH5/rnChQowNChQ1m7di0//fQTPXv2JDEx0YEIQ8sTT8D330PhwjIOfO21colRhaYjR+DWW2V96AIFpDp+wAB5g6aUP9E/SR8pX748DRo0AKBMmTKUKFGCAwcOuBtUkLj+eimiqVRJPlatCv/843ZUytd27pRezdOnQ4kSUozXpYvbUSl1Zpp8XfD777+TmppKpUqV3A4laNSrJ8U0l10ml56bNIHZs92OSvnK0qXy3O/eDdWqyZQiXYhD+TNNvj528OBBOnXqxKefforR9cq8qnx5WLQIbrtNLj9edx18/LHbUSmnff+9JNo9e2QlrGXLICbG7aiUOjdNvl40aNAgjDGn3V566SUAUlJSuOWWW+jXrx9XXXWVy9EGp6gomDwZnnlGKl67dYPHHpPCGxVcrJUq944d4dgxuP9+udRcqpTbkSmVM02+udSmTRvuuOMO5syZQ8WKFVm6dOnx7/Xo0YPdu3cfvz311FOUK1eOTp06Ya2lS5cutG7dWqcpOSwsDAYOlF7QYWEwcqScDWsldPBIS5NiuxdflK9ff11WKipY0N24lPJUAbcDCDRz584FID4+/rSq5+jo6OP3DRw4kAkTJrBw4UIuvvhilixZwqRJk6hXrx5Tp04F4IsvvqBu3bo+jT+UPPywFF/deitMnSpjgvPmuR2Vyq+jR8Np2RJ++UWS7eefwz33uB2VUrmjydcBb731Fh9++CELFiwgJnPwqVmzZmTotU+fa91axgBbt5YVkRo3hpdfLqzFOAFq61Z44omGbNkCRYvKspM6gqMCkd9cdn7lFbmBFEusXy+9WBs1kvueegoGD5bPK1SAXbtg4cITFY3dusnycyAdkOLjpRDjxhvlvnvvha++ks/zWueUfRy3aNGip43tAgwYMIARI0awaNGi44lXuatmTVlAvWlTWYrwsccuZcYMt6NSubViBTRoAFu2FOGSS+DPPzXxqgBmrXXkFhMTY89m7dq1Z/2eP9u2bZtt0aKFveSSS2ydOnXst99+e9L3X3vtNVupUiW7YcMGlyL0jri4uDPeH6jPW5akJGsfvnabrcg2a4y1Q4a4HZH3tWjRwtavX9/tMLxuzBhrCxa0tiLb7DUxf9nDh92OyDkLFixwOwTHBPOxWWst8Lv1MEf6zZlvIMjepWratGkndakaMGAA77//PhMnTqRw4cLs2bOHPXv2kJyc7HLUKkuhQvDxrEq06ZKGtdCnjxTtpKW5HZk6m7Q0eZ66dJGlAa/vVonnhh+geHG3I1MqfzT55kL2LlWlS5c+3qXKWsugQYM4ePAgTZs2pXz58sdvv/zyi7tBq5OYyZPoW+lzxo2DiAgYPlxWvdm92+3I1KkOHIBmzeC996RV5PDhMKr1JMr/rFVzKvBpwVUe/fHHH8e7VBljOKKrugeGkSO5IDaW2qteo1o1mSO6ciU0bAiTJkGLFm4HqAD++EOmh2UVVs2YIYmYlvL88dprLkeoVP7omW8eHDx4kO7du2uXqgB31VXw99/QqhXs3SsfBw3SpQndZK2sVnX55ZJ4L79cnqNmzdyOTCnv0uSbS1ldqvr06aNdqoJA2bKy5mvv3vLC/+yz0pZy3z63Iws9Bw9CmzbQq5d0JHv4Yfj5Z6hY0e3IlPI+Tb65YLN1qbpHZ/UHjQIFYMgQmDZNLnHOng116qDTkXxo8WKZRjR/viwP+e230qGsUCG3I1PKGZp8c+GXX35h0qRJTJ06laZNm9KgQQPWrFnjdljKSzp2hDVrpDn//v3QoYOcfemyy85JSpI5/FdfLXOwmzSBv/6SrmRKBTMtuMqF7F2qztReUgWAb77h719+oelZvl25sjRveecdeP55+OwzufT5+ec67uhtv/4K990nY7vGSBJ+802pQj+rHJ4/pQKFnvmq0FKqFKnFip1zk7AwGfv94w+oXRs2bJCz4UcfhdhY34QZzBIToW9f6Ti2ZQvUqCFrMb/zTg6JFzx6/pQKBJp8VWgZM4Zys2Z5tGn9+vD779Cvn4wLjxoFF10k45FaEZ171sq4evXq0irWGHmTs2qVVDV7JBfPn1L+TJOvCi25fPEuVEguha5aBfXqweHDcPvt0lP8n38cizLobNggc6hvvln6sl98sSx48fbbuSyq0uSrgoQmX6U8ULu2NPIfOVKqcX/+GerWlfaUBw64HZ3/OnIEnnkGatWSiubISBg2TN64XHGF29Ep5R5Nvkp5KCxMxn03bYLu3WUu6vDhUK2aXEZNSnI7Qv+RlCS/k4sukrHc1FTo3Bk2b4b/+z+5jK9UKHMt+VodNAso+nydUKYMfPQRrF4thVhxcVJAVLEiDB0a2kk4NRU+/VTekPTtC4cOSWHVr7/CmDHS1EQp5VLyjYiIICmUX6ECUFJSEhE5lqKGlrp1YdEiacZRvbokmt69oVw5adpx9KjbEfpOQoK88bjgAujaVRaqiImRxe4XL5b5u0qpE1xJvmXKlGHnzp0kJibqGZWfs9aSmJjIzp07KVOmjNvh5N+PP/K/t9/22sMZA9dfD+vWwfTpMrYZFydzVsuVg549ZTpNsNq7F/r3lzPa3r2lOUmlSjBhgozrtm8vvyOv8fLzp5RbXBl5KVq0KAC7du0iNTXVjRDyLTk5mUJB2vvu1GOLiIigbNmyx5+3gBYVRYYDz5sxcOON0hVrxgwYMECqeYcNgw8+gNat4bnn5GNYgFdaZGTA3LlybLNmQXq63H/ZZfDii/I7cOwYHXr+lPI118oeihYtGtAv5gsXLqRhw4Zuh+GIYD42Roygwvr1MlfIAcZI8unQAVaskMQ7YQLMmye30qWhUydZHL5OHUdCcMyGDTBxoiTd/fvlPmPgppvg6adlbNdxDj9/SvmK1hyq0DJ5MmV81Kbqiivgiy+k2nf0aClE2rZNqoAHD5a5rp07y2Xrhg29fHnWS9auhW++kTcQ//574v4KFaBHD3jwQRnn9RkfPn9KOcmji0PGmJLGmO+MMUeNMVuNMfc6HZhSwaJcOXjpJZlms3ixTFMqUkTOJF98ERo1gvLlZRGHr7+WJhRu2bNHEu3DD8ubg9q14eWXJfEWKiS9mGfPhu3bZazXp4lXqSDi6ZnvcOAYUBZoAMwwxqy21v7tVGBKBZuwMFmcoVkzWTB+9mz44Qc5s9y7VxZx+Owz2bZMGbmM26KFdNaqWVOSuDfPjg8elOlSf/4pHbwWLICdO0/eJjpaxrLvuw+uuQYKFvTe/pUKZTkmX2NMYeA2oI61NgFYYoyZDjwAPOdwfEoFpYIFZQnDjh2lZ/T//ifV0nPnSiLctw+++05uWQoXhqpVZay4eHGZV1y2rIwjR0ZK44ojR+Do0QIsWQLx8SduR45Ic5AdO2DrVti4Ue4/VWSkvDlo2RLatpXL4doQQynvMzlN9THGNAR+tdZGZruvL9DCWnvj2X4uKirKXhHE/eNiY2MpXry422E4IpiPjVWrSEtLo8Bll7kdyVlZKyv/xMVJ0kxIgJQUSEvz5KdXZX5skOOWYWEQFSVJvWhRuRQeHe2fY8/HBcDzl1/B/P8XzMcGsGjRopXWWo/+OD1Jvs2Br6215bLd9whwn7W25SnbdgO6ZX5ZB/grF3EHmlJAsHb1DeZjAz2+QKfHF7iC+dgAalhrPVro3ZMLSgnAqXOCigKnXbSy1o4GRgMYY3739B1AIArm4wvmYwM9vkCnxxe4gvnYQI7P0209qXZeDxQwxlTPdl99QIutlFJKqTzIMflaa48CU4DXjDGFjTFNgZuAL5wOTimllApGnjaBewyIBPYBE4AeHkwzGp2fwAJAMB9fMB8b6PEFOj2+wBXMxwa5OL4cC66UUkop5V0B3uJdKaWUCjyafJVSSikf80nyNcZUN8YkG2PG+2J/vmKMGW+M2W2MiTPGrDfGdHU7Jm8xxhQ0xnya2cs73hjzpzHmOrfj8iZjzBPGmN+NMSnGmDFux5NfwdyDPdieq1MF+/9bML9WZpebXOerxnHDgd98tC9fegt42FqbYoypCSw0xvxprV3pdmBeUADYDrQAtgHXA5ONMXWttVvcDMyLdgFvAO2QgsJAF8w92IPtuTpVsP+/BfNrZXYe5zrHz3yNMXcDscA8p/fla9bav621KVlfZt6quRiS11hrj1prX7HWbrHWZlhrfwA2A43cjs1brLVTrLVTgYNux5Jf2Xqwv2itTbDWLgGyerAHvGB6rs4k2P/fgvm1Mktuc52jydcYUxR4DXjKyf24yRgzwhiTCPwL7AZ+dDkkRxhjygIxaHMVfxUDpFtr12e7bzVQ26V4VD4E4/9bML9W5iXXOX3m+zrwqbV2u8P7cY219jEgGmiONCNJOfdPBB5jTATwJTDWWvtvTtsrVxQBjpxy3xHkb1MFkGD9fwvy18pc57o8J19jzEJjjD3LbYkxpgHQBngvr/twU07Hl31ba2165mW+ikAPdyLOHU+PzxgThnQzOwY84VrAuZSb5y9IeNyDXfmvQP1/81QgvlbmJK+5Ls8FV6euaHSGgHoBVYBtRtYoKwKEG2NqWWsvzet+fSWn4zuLAgTIOIYnx2fkifsUKeC53lqb6nRc3pLH5y+QHe/Bbq39L/M+7cEeQAL5/y0PAua10gMtyUOuc/Ky82jkl9sg8/YRMAOpVgx4xpgyxpi7jTFFjDHhxph2wD3AfLdj86KRwCXAjdbaJLeD8TZjTAFjTCEgHPlnKWSMCcil44O9B3swPVfnEJT/byHwWpmnXOdY8rXWJlpr92TdkMtiydba/U7t08csctlkB3AYeBfoZa2d5mpUXmKMuRDojvwx7THGJGTe7nM3Mq/qDyQBzwH3Z37e39WI8icvPdgDRbA9VycJ8v+3oH6tzGuu097OSimllI9pe0mllFLKxzT5KqWUUj6myVcppZTyMU2+SimllI9p8lVKKaV8TJOvUkop5WOafJVSSikf0+SrlFJK+ZgmX6WUUsrHNPkqFQSMMc+cZQWn19yOTSl1Om0vqVQQMMZEA4Wz3dUXuA9obq3d4E5USqmz0eSrVJAxxjwLPAm0ttauczsepdTpgm1JLqVCmjGmH7IIeytr7Xq341FKnZkmX6WChDHmBeBRoIVealbKv2nyVSoIGGNeBB4BWlprN7odj1Lq3DT5KhXgMs94ewIdgaPGmHKZ34q11ia7F5lS6my04EqpAGaMMUAsUPQM325jrZ3n24iUUp7Q5KuUUkr5mDbZUEoppXxMk69SSinlY5p8lVJKKR/T5KuUUkr5mCZfpZRSysc0+SqllFI+pslXKaWU8jFNvkoppZSPafJVSimlfOz/AdCyz26O7UkeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x252 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 3.5))\n",
    "z = np.linspace(-4, 4, 200)\n",
    "plt.plot(z, huber_fn(0, z), \"b-\", linewidth=2, label=\"huber($z$)\")\n",
    "plt.plot(z, z**2 / 2, \"b:\", linewidth=1, label=r\"$\\frac{1}{2}z^2$\")\n",
    "plt.plot([-1, -1], [0, huber_fn(0., -1.)], \"r--\")\n",
    "plt.plot([1, 1], [0, huber_fn(0., 1.)], \"r--\")\n",
    "plt.gca().axhline(y=0, color='k')\n",
    "plt.gca().axvline(x=0, color='k')\n",
    "plt.axis([-4, 4, 0, 4])\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"$z$\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Huber loss\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = X_train.shape[1:]\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=huber_fn, optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.0443 - mae: 1.4660 - val_loss: 0.2862 - val_mae: 0.5866\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 737us/step - loss: 0.2379 - mae: 0.5407 - val_loss: 0.2382 - val_mae: 0.5281\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc000dcc850>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving/Loading Models with Custom Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_loss.h5\",\n",
    "                                custom_objects={\"huber_fn\": huber_fn})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 970us/step - loss: 0.2054 - mae: 0.4982 - val_loss: 0.2209 - val_mae: 0.5050\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 769us/step - loss: 0.1999 - mae: 0.4900 - val_loss: 0.2127 - val_mae: 0.4986\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfd8c09090>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_huber(threshold=1.0):\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = threshold * tf.abs(error) - threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    return huber_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.2318 - mae: 0.4979 - val_loss: 0.2540 - val_mae: 0.4907\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 749us/step - loss: 0.2309 - mae: 0.4960 - val_loss: 0.2372 - val_mae: 0.4879\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfd8c969d0>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_threshold_2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2.h5\",\n",
    "                                custom_objects={\"huber_fn\": create_huber(2.0)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.2147 - mae: 0.4800 - val_loss: 0.2133 - val_mae: 0.4654\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 763us/step - loss: 0.2119 - mae: 0.4762 - val_loss: 0.1992 - val_mae: 0.4643\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbff7a0a590>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberLoss(keras.losses.Loss):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=HuberLoss(2.), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.3123 - mae: 1.3345 - val_loss: 0.3378 - val_mae: 0.5485\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 760us/step - loss: 0.2659 - mae: 0.5270 - val_loss: 0.2660 - val_mae: 0.5089\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfe06dad50>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_class.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_loss_class.h5\",\n",
    "                                custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 966us/step - loss: 0.2286 - mae: 0.4970 - val_loss: 0.2120 - val_mae: 0.4723\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 757us/step - loss: 0.2216 - mae: 0.4904 - val_loss: 0.2045 - val_mae: 0.4725\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfb0630d50>"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.loss.threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Custom Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(z): # return value is just tf.nn.softplus(z)\n",
    "    return tf.math.log(tf.exp(z) + 1.0)\n",
    "\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weights))\n",
    "\n",
    "def my_positive_weights(weights): # return value is just tf.nn.relu(weights)\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = keras.layers.Dense(1, activation=my_softplus,\n",
    "                           kernel_initializer=my_glorot_initializer,\n",
    "                           kernel_regularizer=my_l1_regularizer,\n",
    "                           kernel_constraint=my_positive_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=my_l1_regularizer,\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 2.3829 - mae: 1.1635 - val_loss: 1.4154 - val_mae: 0.5607\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 757us/step - loss: 0.6299 - mae: 0.5410 - val_loss: 1.4399 - val_mae: 0.5137\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfd8e49690>"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"my_l1_regularizer\": my_l1_regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyL1Regularizer(keras.regularizers.Regularizer):\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "    def get_config(self):\n",
    "        return {\"factor\": self.factor}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=MyL1Regularizer(0.01),\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 2.3829 - mae: 1.1635 - val_loss: 1.4154 - val_mae: 0.5607\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 757us/step - loss: 0.6299 - mae: 0.5410 - val_loss: 1.4399 - val_mae: 0.5137\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfd90141d0>"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"MyL1Regularizer\": MyL1Regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 572us/step - loss: 3.5903 - huber_fn: 1.5558\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 552us/step - loss: 0.8054 - huber_fn: 0.3095\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbff7cbe850>"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: if you use the same function as the loss and a metric, you may be surprised to see different results. This is generally just due to floating point precision errors: even though the mathematical equations are equivalent, the operations are not run in the same order, which can lead to small differences. Moreover, when using sample weights, there's more than just precision errors:\n",
    "* the loss since the start of the epoch is the mean of all batch losses seen so far. Each batch loss is the sum of the weighted instance losses divided by the _batch size_ (not the sum of weights, so the batch loss is _not_ the weighted mean of the losses).\n",
    "* the metric since the start of the epoch is equal to the sum of weighted instance losses divided by sum of all weights seen so far. In other words, it is the weighted mean of all the instance losses. Not the same thing.\n",
    "\n",
    "If you do the math, you will find that loss = metric * mean of sample weights (plus some floating point precision error)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 598us/step - loss: 0.1245 - huber_fn: 0.2515\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 558us/step - loss: 0.1216 - huber_fn: 0.2473\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.11749906837940216, 0.11906625573138947)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"huber_fn\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Streaming metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = keras.metrics.Precision()\n",
    "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision([0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 1, 0, 0, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a streaming metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Metric):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs) # handles base args (e.g., dtype)\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        self.total.assign_add(tf.reduce_sum(metric))\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32))\n",
    "    def result(self):\n",
    "        return self.total / self.count\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=14.0>"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = HuberMetric(2.)\n",
    "\n",
    "# total = 2 * |10 - 2| - 2²/2 = 14\n",
    "# count = 1\n",
    "# result = 14 / 1 = 14\n",
    "m(tf.constant([[2.]]), tf.constant([[10.]])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# total = total + (|1 - 0|² / 2) + (2 * |9.25 - 5| - 2² / 2) = 14 + 7 = 21\n",
    "# count = count + 2 = 3\n",
    "# result = total / count = 21 / 3 = 7\n",
    "m(tf.constant([[0.], [5.]]), tf.constant([[1.], [9.25]]))\n",
    "\n",
    "m.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=21.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=3.0>]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=0.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=0.0>]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.reset_states()\n",
    "m.variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that the `HuberMetric` class works well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 592us/step - loss: 1.5182 - huber_metric: 1.5182\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 538us/step - loss: 0.2909 - huber_metric: 0.2909\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc000fdfbd0>"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_metric.h5\",\n",
    "                                custom_objects={\"huber_fn\": create_huber(2.0),\n",
    "                                                \"HuberMetric\": HuberMetric})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 545us/step - loss: 0.2350 - huber_metric: 0.2350\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 524us/step - loss: 0.2278 - huber_metric: 0.2278\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc0011c1a50>"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: In TF 2.2, tf.keras adds an extra first metric in `model.metrics` at position 0 (see [TF issue #38150](https://github.com/tensorflow/tensorflow/issues/38150)). This forces us to use `model.metrics[-1]` rather than `model.metrics[0]` to access the `HuberMetric`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like it works fine! More simply, we could have created the class like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Mean):\n",
    "    def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        super().__init__(name=name, dtype=dtype)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        super(HuberMetric, self).update_state(metric, sample_weight)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This class handles shapes better, and it also supports sample weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=keras.losses.Huber(2.0), optimizer=\"nadam\", weighted_metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 627us/step - loss: 0.7800 - HuberMetric: 1.5672\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 607us/step - loss: 0.1462 - HuberMetric: 0.2939\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32),\n",
    "                    epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.44554394483566284, 0.44554404180100277)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"HuberMetric\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric_v2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_metric_v2.h5\",\n",
    "                                custom_objects={\"HuberMetric\": HuberMetric})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 578us/step - loss: 0.2377 - HuberMetric: 0.2377\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 559us/step - loss: 0.2279 - HuberMetric: 0.2279\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfb052b150>"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "exponential_layer = keras.layers.Lambda(lambda x: tf.exp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float32, numpy=array([0.36787945, 1.        , 2.7182817 ], dtype=float32)>"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exponential_layer([-1., 0., 1.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding an exponential layer at the output of a regression model can be useful if the values to predict are positive and with very different scales (e.g., 0.001, 10., 10000):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 0s 828us/step - loss: 2.1275 - val_loss: 0.4457\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 645us/step - loss: 0.4750 - val_loss: 0.3798\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 656us/step - loss: 0.4230 - val_loss: 0.3548\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 657us/step - loss: 0.3905 - val_loss: 0.3464\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 664us/step - loss: 0.3784 - val_loss: 0.3449\n",
      "162/162 [==============================] - 0s 420us/step - loss: 0.3586\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.3586340546607971"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "    exponential_layer\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"sgd\")\n",
    "model.fit(X_train_scaled, y_train, epochs=5,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.kernel = self.add_weight(\n",
    "            name=\"kernel\", shape=[batch_input_shape[-1], self.units],\n",
    "            initializer=\"glorot_normal\")\n",
    "        self.bias = self.add_weight(\n",
    "            name=\"bias\", shape=[self.units], initializer=\"zeros\")\n",
    "        super().build(batch_input_shape) # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        return self.activation(X @ self.kernel + self.bias)\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return tf.TensorShape(batch_input_shape.as_list()[:-1] + [self.units])\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"units\": self.units,\n",
    "                \"activation\": keras.activations.serialize(self.activation)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    MyDense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    MyDense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 898us/step - loss: 4.1268 - val_loss: 0.9472\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 707us/step - loss: 0.7086 - val_loss: 0.6219\n",
      "162/162 [==============================] - 0s 419us/step - loss: 0.5474\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5473727583885193"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_layer.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_layer.h5\",\n",
    "                                custom_objects={\"MyDense\": MyDense})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMultiLayer(keras.layers.Layer):\n",
    "    def call(self, X):\n",
    "        X1, X2 = X\n",
    "        return X1 + X2, X1 * X2\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        batch_input_shape1, batch_input_shape2 = batch_input_shape\n",
    "        return [batch_input_shape1, batch_input_shape2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs1 = keras.layers.Input(shape=[2])\n",
    "inputs2 = keras.layers.Input(shape=[2])\n",
    "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a layer with a different behavior during training and testing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AddGaussianNoise(keras.layers.Layer):\n",
    "    def __init__(self, stddev, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.stddev = stddev\n",
    "\n",
    "    def call(self, X, training=None):\n",
    "        if training:\n",
    "            noise = tf.random.normal(tf.shape(X), stddev=self.stddev)\n",
    "            return X + noise\n",
    "        else:\n",
    "            return X\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 899us/step - loss: 0.5496 - val_loss: 0.5329\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 750us/step - loss: 0.4430 - val_loss: 0.4786\n",
      "162/162 [==============================] - 0s 420us/step - loss: 0.3990\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.3990039527416229"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_scaled = X_test_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualBlock(keras.layers.Layer):\n",
    "    def __init__(self, n_layers, n_neurons, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(n_neurons, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "                       for _ in range(n_layers)]\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        return inputs + Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = keras.layers.Dense(30, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "        self.block1 = ResidualBlock(2, 30)\n",
    "        self.block2 = ResidualBlock(2, 30)\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = self.hidden1(inputs)\n",
    "        for _ in range(1 + 3):\n",
    "            Z = self.block1(Z)\n",
    "        Z = self.block2(Z)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 837us/step - loss: 22.7478\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 739us/step - loss: 1.2735\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 737us/step - loss: 0.9792\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 740us/step - loss: 0.5905\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 739us/step - loss: 0.5544\n",
      "162/162 [==============================] - 0s 526us/step - loss: 0.6513\n"
     ]
    }
   ],
   "source": [
    "model = ResidualRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as dense_1_layer_call_and_return_conditional_losses, dense_1_layer_call_fn, dense_2_layer_call_and_return_conditional_losses, dense_2_layer_call_fn, dense_3_layer_call_and_return_conditional_losses while saving (showing 5 of 20). These functions will not be directly callable after loading.\n",
      "WARNING:absl:Found untraced functions such as dense_1_layer_call_and_return_conditional_losses, dense_1_layer_call_fn, dense_2_layer_call_and_return_conditional_losses, dense_2_layer_call_fn, dense_3_layer_call_and_return_conditional_losses while saving (showing 5 of 20). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_custom_model.ckpt/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_custom_model.ckpt/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_custom_model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_custom_model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 851us/step - loss: 0.9476\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 736us/step - loss: 0.6998\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 737us/step - loss: 0.4668\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 758us/step - loss: 0.4818\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 756us/step - loss: 0.4591\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could have defined the model using the sequential API instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "block1 = ResidualBlock(2, 30)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "    block1, block1, block1, block1,\n",
    "    ResidualBlock(2, 30),\n",
    "    keras.layers.Dense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 709us/step - loss: 1.5508\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 645us/step - loss: 0.5562\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 625us/step - loss: 0.6406\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 636us/step - loss: 0.3759\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 623us/step - loss: 0.3875\n",
      "162/162 [==============================] - 0s 463us/step - loss: 0.4852\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Losses and Metrics Based on Model Internals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: due to an issue introduced in TF 2.2 ([#46858](https://github.com/tensorflow/tensorflow/issues/46858)), it is currently not possible to use `add_loss()` along with the `build()` method. So the following code differs from the book: I create the `reconstruct` layer in the constructor instead of the `build()` method. Unfortunately, this means that the number of units in this layer must be hard-coded (alternatively, it could be passed as an argument to the constructor)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReconstructingRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(30, activation=\"selu\",\n",
    "                                          kernel_initializer=\"lecun_normal\")\n",
    "                       for _ in range(5)]\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "        self.reconstruct = keras.layers.Dense(8) # workaround for TF issue #46858\n",
    "        self.reconstruction_mean = keras.metrics.Mean(name=\"reconstruction_error\")\n",
    "\n",
    "    #Commented out due to TF issue #46858, see the note above\n",
    "    #def build(self, batch_input_shape):\n",
    "    #    n_inputs = batch_input_shape[-1]\n",
    "    #    self.reconstruct = keras.layers.Dense(n_inputs)\n",
    "    #    super().build(batch_input_shape)\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        self.add_loss(0.05 * recon_loss)\n",
    "        if training:\n",
    "            result = self.reconstruction_mean(recon_loss)\n",
    "            self.add_metric(result)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 761us/step - loss: 1.6313 - reconstruction_error: 1.0474\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 667us/step - loss: 0.4536 - reconstruction_error: 0.4022\n"
     ]
    }
   ],
   "source": [
    "model = ReconstructingRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "y_pred = model.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing Gradients with Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + 2 * w1 * w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.000003007075065"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1, w2 = 5, 3\n",
    "eps = 1e-6\n",
    "(f(w1 + eps, w2) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.000000003174137"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f(w1, w2 + eps) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1, w2 = tf.Variable(5.), tf.Variable(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A non-persistent GradientTape can only be used tocompute one set of gradients (or jacobians)\n"
     ]
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "try:\n",
    "    dz_dw2 = tape.gradient(z, w2)\n",
    "except RuntimeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "dz_dw2 = tape.gradient(z, w2) # works now!\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>)"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dz_dw1, dz_dw2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(c1)\n",
    "    tape.watch(c2)\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=136.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=30.0>]"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tape.gradient([z1, z2, z3], [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tf.reduce_sum(tf.stack([tape.gradient(z, [w1, w2]) for z in (z1, z2, z3)]), axis=0)\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as hessian_tape:\n",
    "    with tf.GradientTape() as jacobian_tape:\n",
    "        z = f(w1, w2)\n",
    "    jacobians = jacobian_tape.gradient(z, [w1, w2])\n",
    "hessians = [hessian_tape.gradient(jacobian, [w1, w2])\n",
    "            for jacobian in jacobians]\n",
    "del hessian_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jacobians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[<tf.Tensor: shape=(), dtype=float32, numpy=6.0>,\n",
       "  <tf.Tensor: shape=(), dtype=float32, numpy=2.0>],\n",
       " [<tf.Tensor: shape=(), dtype=float32, numpy=2.0>, None]]"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hessians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=30.0>, None]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + tf.stop_gradient(2 * w1 * w2)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=nan>]"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable(100.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=30.0>"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.math.log(tf.exp(tf.constant(30., dtype=tf.float32)) + 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>]"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([100.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.custom_gradient\n",
    "def my_better_softplus(z):\n",
    "    exp = tf.exp(z)\n",
    "    def my_softplus_gradients(grad):\n",
    "        return grad / (1 + 1 / exp)\n",
    "    return tf.math.log(exp + 1), my_softplus_gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_better_softplus(z):\n",
    "    return tf.where(z > 30., z, tf.math.log(tf.exp(z) + 1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1000.], dtype=float32)>,\n",
       " [<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>])"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([1000.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_better_softplus(x)\n",
    "\n",
    "z, tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing Gradients Using Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "l2_reg = keras.regularizers.l2(0.05)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\",\n",
    "                       kernel_regularizer=l2_reg),\n",
    "    keras.layers.Dense(1, kernel_regularizer=l2_reg)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_batch(X, y, batch_size=32):\n",
    "    idx = np.random.randint(len(X), size=batch_size)\n",
    "    return X[idx], y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{}/{} - \".format(iteration, total) + metrics,\n",
    "          end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A fancier version with a progress bar:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "def progress_bar(iteration, total, size=30):\n",
    "    running = iteration < total\n",
    "    c = \">\" if running else \"=\"\n",
    "    p = (size - 1) * iteration // total\n",
    "    fmt = \"{{:-{}d}}/{{}} [{{}}]\".format(len(str(total)))\n",
    "    params = [iteration, total, \"=\" * p + c + \".\" * (size - p - 1)]\n",
    "    return fmt.format(*params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' 3500/10000 [=>....]'"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "progress_bar(3500, 10000, size=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None, size=30):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{} - {}\".format(progress_bar(iteration, total), metrics), end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 [==============================] - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.mean_squared_error\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.MeanAbsoluteError()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 [==============================] - mean: 1.3955 - mean_absolute_error: 0.5722\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - mean: 0.6774 - mean_absolute_error: 0.5280\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - mean: 0.6351 - mean_absolute_error: 0.5177\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - mean: 0.6384 - mean_absolute_error: 0.5181\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - mean: 0.6440 - mean_absolute_error: 0.5222\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, n_epochs + 1):\n",
    "    print(\"Epoch {}/{}\".format(epoch, n_epochs))\n",
    "    for step in range(1, n_steps + 1):\n",
    "        X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "        with tf.GradientTape() as tape:\n",
    "            y_pred = model(X_batch)\n",
    "            main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "            loss = tf.add_n([main_loss] + model.losses)\n",
    "        gradients = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "        for variable in model.variables:\n",
    "            if variable.constraint is not None:\n",
    "                variable.assign(variable.constraint(variable))\n",
    "        mean_loss(loss)\n",
    "        for metric in metrics:\n",
    "            metric(y_batch, y_pred)\n",
    "        print_status_bar(step * batch_size, len(y_train), mean_loss, metrics)\n",
    "    print_status_bar(len(y_train), len(y_train), mean_loss, metrics)\n",
    "    for metric in [mean_loss] + metrics:\n",
    "        metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "911ccc0821324e99b156302727dbf9c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21c8135ff59540c085b55653e6d69243",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06190b83afc348e999aeb3a3c523aae8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44eb6b7d11b74a4aa9af17892e2a8ce9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c62578a1aa24b7d96201cc285025e83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46a0328d0c5b4112995297e67bc290b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/362 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "try:\n",
    "    from tqdm.notebook import trange\n",
    "    from collections import OrderedDict\n",
    "    with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "        for epoch in epochs:\n",
    "            with trange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "                for step in steps:\n",
    "                    X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "                    with tf.GradientTape() as tape:\n",
    "                        y_pred = model(X_batch)\n",
    "                        main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                        loss = tf.add_n([main_loss] + model.losses)\n",
    "                    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                    for variable in model.variables:\n",
    "                        if variable.constraint is not None:\n",
    "                            variable.assign(variable.constraint(variable))                    \n",
    "                    status = OrderedDict()\n",
    "                    mean_loss(loss)\n",
    "                    status[\"loss\"] = mean_loss.result().numpy()\n",
    "                    for metric in metrics:\n",
    "                        metric(y_batch, y_pred)\n",
    "                        status[metric.name] = metric.result().numpy()\n",
    "                    steps.set_postfix(status)\n",
    "            for metric in [mean_loss] + metrics:\n",
    "                metric.reset_states()\n",
    "except ImportError as ex:\n",
    "    print(\"To run this cell, please install tqdm, ipywidgets and restart Jupyter\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cube(x):\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.eager.def_function.Function at 0x7fbfe0c54d50>"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube = tf.function(cube)\n",
    "tf_cube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=8>"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF Functions and Concrete Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x7fbfd9444310>"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function = tf_cube.get_concrete_function(tf.constant(2.0))\n",
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function is tf_cube.get_concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploring Function Definitions and Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x7fbfd9444310>"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'pow/y' type=Const>,\n",
       " <tf.Operation 'pow' type=Pow>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ops = concrete_function.graph.get_operations()\n",
    "ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'x:0' shape=() dtype=float32>,\n",
       " <tf.Tensor 'pow/y:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op = ops[2]\n",
    "list(pow_op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'pow:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op.outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'x' type=Placeholder>"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_operation_by_name('x')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Identity:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_tensor_by_name('Identity:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"__inference_cube_1073862\"\n",
       "input_arg {\n",
       "  name: \"x\"\n",
       "  type: DT_FLOAT\n",
       "}\n",
       "output_arg {\n",
       "  name: \"identity\"\n",
       "  type: DT_FLOAT\n",
       "}"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.function_def.signature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How TF Functions Trace Python Functions to Extract Their Computation Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def tf_cube(x):\n",
    "    print(\"print:\", x)\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: Tensor(\"x:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: 2\n",
      "print: 3\n",
      "print: Tensor(\"x:0\", shape=(1, 2), dtype=float32)\n",
      "print: Tensor(\"x:0\", shape=(2, 2), dtype=float32)\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function tf_cube at 0x7fbfc0363440> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 5 calls to <function tf_cube at 0x7fbfc0363440> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: Tensor(\"x:0\", shape=(3, 2), dtype=float32)\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function tf_cube at 0x7fbfc0363440> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:6 out of the last 6 calls to <function tf_cube at 0x7fbfc0363440> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(2)\n",
    "result = tf_cube(3)\n",
    "result = tf_cube(tf.constant([[1., 2.]])) # New shape: trace!\n",
    "result = tf_cube(tf.constant([[3., 4.], [5., 6.]])) # New shape: trace!\n",
    "result = tf_cube(tf.constant([[7., 8.], [9., 10.], [11., 12.]])) # no trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also possible to specify a particular input signature:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function(input_signature=[tf.TensorSpec([None, 28, 28], tf.float32)])\n",
    "def shrink(images):\n",
    "    print(\"Tracing\", images)\n",
    "    return images[:, ::2, ::2] # drop half the rows and columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing Tensor(\"images:0\", shape=(None, 28, 28), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "img_batch_1 = tf.random.uniform(shape=[100, 28, 28])\n",
    "img_batch_2 = tf.random.uniform(shape=[50, 28, 28])\n",
    "preprocessed_images = shrink(img_batch_1) # Traces the function.\n",
    "preprocessed_images = shrink(img_batch_2) # Reuses the same concrete function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python inputs incompatible with input_signature:\n",
      "  inputs: (\n",
      "    tf.Tensor(\n",
      "[[[0.7413678  0.62854624]\n",
      "  [0.01738465 0.3431449 ]]\n",
      "\n",
      " [[0.51063764 0.3777541 ]\n",
      "  [0.07321596 0.02137029]]], shape=(2, 2, 2), dtype=float32))\n",
      "  input_signature: (\n",
      "    TensorSpec(shape=(None, 28, 28), dtype=tf.float32, name=None))\n"
     ]
    }
   ],
   "source": [
    "img_batch_3 = tf.random.uniform(shape=[2, 2, 2])\n",
    "try:\n",
    "    preprocessed_images = shrink(img_batch_3)  # rejects unexpected types or shapes\n",
    "except ValueError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Autograph To Capture Control Flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"static\" `for` loop using `range()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in range(10):\n",
    "        x += 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'add/y' type=Const>,\n",
       " <tf.Operation 'add' type=AddV2>,\n",
       " <tf.Operation 'add_1/y' type=Const>,\n",
       " <tf.Operation 'add_1' type=AddV2>,\n",
       " <tf.Operation 'add_2/y' type=Const>,\n",
       " <tf.Operation 'add_2' type=AddV2>,\n",
       " <tf.Operation 'add_3/y' type=Const>,\n",
       " <tf.Operation 'add_3' type=AddV2>,\n",
       " <tf.Operation 'add_4/y' type=Const>,\n",
       " <tf.Operation 'add_4' type=AddV2>,\n",
       " <tf.Operation 'add_5/y' type=Const>,\n",
       " <tf.Operation 'add_5' type=AddV2>,\n",
       " <tf.Operation 'add_6/y' type=Const>,\n",
       " <tf.Operation 'add_6' type=AddV2>,\n",
       " <tf.Operation 'add_7/y' type=Const>,\n",
       " <tf.Operation 'add_7' type=AddV2>,\n",
       " <tf.Operation 'add_8/y' type=Const>,\n",
       " <tf.Operation 'add_8' type=AddV2>,\n",
       " <tf.Operation 'add_9/y' type=Const>,\n",
       " <tf.Operation 'add_9' type=AddV2>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" loop using `tf.while_loop()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    condition = lambda i, x: tf.less(i, 10)\n",
    "    body = lambda i, x: (tf.add(i, 1), tf.add(x, 1))\n",
    "    final_i, final_x = tf.while_loop(condition, body, [tf.constant(0), x])\n",
    "    return final_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" `for` loop using `tf.range()` (captured by autograph):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x = x + 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'range/start' type=Const>,\n",
       " <tf.Operation 'range/limit' type=Const>,\n",
       " <tf.Operation 'range/delta' type=Const>,\n",
       " <tf.Operation 'range' type=Range>,\n",
       " <tf.Operation 'sub' type=Sub>,\n",
       " <tf.Operation 'floordiv' type=FloorDiv>,\n",
       " <tf.Operation 'mod' type=FloorMod>,\n",
       " <tf.Operation 'zeros_like' type=Const>,\n",
       " <tf.Operation 'NotEqual' type=NotEqual>,\n",
       " <tf.Operation 'Cast' type=Cast>,\n",
       " <tf.Operation 'add' type=AddV2>,\n",
       " <tf.Operation 'zeros_like_1' type=Const>,\n",
       " <tf.Operation 'Maximum' type=Maximum>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(0)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Handling Variables and Other Resources in TF Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(counter, c=1):\n",
    "    return counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "increment(counter)\n",
    "increment(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"counter\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function(counter).function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(c=1):\n",
    "    return counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "increment()\n",
    "increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"assignaddvariableop_resource\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function().function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Counter:\n",
    "    def __init__(self):\n",
    "        self.counter = tf.Variable(0)\n",
    "\n",
    "    @tf.function\n",
    "    def increment(self, c=1):\n",
    "        return self.counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = Counter()\n",
    "c.increment()\n",
    "c.increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def tf__add(x):\n",
      "    with ag__.FunctionScope('add_10', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
      "        do_return = False\n",
      "        retval_ = ag__.UndefinedReturnValue()\n",
      "\n",
      "        def get_state():\n",
      "            return (x,)\n",
      "\n",
      "        def set_state(vars_):\n",
      "            nonlocal x\n",
      "            (x,) = vars_\n",
      "\n",
      "        def loop_body(itr):\n",
      "            nonlocal x\n",
      "            i = itr\n",
      "            x = ag__.ld(x)\n",
      "            x += 1\n",
      "        i = ag__.Undefined('i')\n",
      "        ag__.for_stmt(ag__.converted_call(ag__.ld(tf).range, (10,), None, fscope), None, loop_body, get_state, set_state, ('x',), {'iterate_names': 'i'})\n",
      "        try:\n",
      "            do_return = True\n",
      "            retval_ = ag__.ld(x)\n",
      "        except:\n",
      "            do_return = False\n",
      "            raise\n",
      "        return fscope.ret(retval_, do_return)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x += 1\n",
    "    return x\n",
    "\n",
    "print(tf.autograph.to_code(add_10.python_function))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_tf_code(func):\n",
    "    from IPython.display import display, Markdown\n",
    "    if hasattr(func, \"python_function\"):\n",
    "        func = func.python_function\n",
    "    code = tf.autograph.to_code(func)\n",
    "    display(Markdown('```python\\n{}\\n```'.format(code)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "def tf__add(x):\n",
       "    with ag__.FunctionScope('add_10', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
       "        do_return = False\n",
       "        retval_ = ag__.UndefinedReturnValue()\n",
       "\n",
       "        def get_state():\n",
       "            return (x,)\n",
       "\n",
       "        def set_state(vars_):\n",
       "            nonlocal x\n",
       "            (x,) = vars_\n",
       "\n",
       "        def loop_body(itr):\n",
       "            nonlocal x\n",
       "            i = itr\n",
       "            x = ag__.ld(x)\n",
       "            x += 1\n",
       "        i = ag__.Undefined('i')\n",
       "        ag__.for_stmt(ag__.converted_call(ag__.ld(tf).range, (10,), None, fscope), None, loop_body, get_state, set_state, ('x',), {'iterate_names': 'i'})\n",
       "        try:\n",
       "            do_return = True\n",
       "            retval_ = ag__.ld(x)\n",
       "        except:\n",
       "            do_return = False\n",
       "            raise\n",
       "        return fscope.ret(retval_, do_return)\n",
       "\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_tf_code(add_10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using TF Functions with tf.keras (or Not)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, tf.keras will automatically convert your custom code into TF Functions, no need to use\n",
    "`tf.function()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom loss function\n",
    "def my_mse(y_true, y_pred):\n",
    "    print(\"Tracing loss my_mse()\")\n",
    "    return tf.reduce_mean(tf.square(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom metric function\n",
    "def my_mae(y_true, y_pred):\n",
    "    print(\"Tracing metric my_mae()\")\n",
    "    return tf.reduce_mean(tf.abs(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom layer\n",
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.kernel = self.add_weight(name='kernel', \n",
    "                                      shape=(input_shape[1], self.units),\n",
    "                                      initializer='uniform',\n",
    "                                      trainable=True)\n",
    "        self.biases = self.add_weight(name='bias', \n",
    "                                      shape=(self.units,),\n",
    "                                      initializer='zeros',\n",
    "                                      trainable=True)\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self, X):\n",
    "        print(\"Tracing MyDense.call()\")\n",
    "        return self.activation(X @ self.kernel + self.biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom model\n",
    "class MyModel(keras.models.Model):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = MyDense(30, activation=\"relu\")\n",
    "        self.hidden2 = MyDense(30, activation=\"relu\")\n",
    "        self.output_ = MyDense(1)\n",
    "\n",
    "    def call(self, input):\n",
    "        print(\"Tracing MyModel.call()\")\n",
    "        hidden1 = self.hidden1(input)\n",
    "        hidden2 = self.hidden2(hidden1)\n",
    "        concat = keras.layers.concatenate([input, hidden2])\n",
    "        output = self.output_(concat)\n",
    "        return output\n",
    "\n",
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "340/363 [===========================>..] - ETA: 0s - loss: 2.8762 - my_mae: 1.2771Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 2.7755 - my_mae: 1.2455 - val_loss: 0.5569 - val_my_mae: 0.4819\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 802us/step - loss: 0.4697 - my_mae: 0.4911 - val_loss: 0.4664 - val_my_mae: 0.4576\n",
      "162/162 [==============================] - 0s 469us/step - loss: 0.4164 - my_mae: 0.4639\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.4163525104522705, 0.4639028012752533]"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can turn this off by creating the model with `dynamic=True` (or calling `super().__init__(dynamic=True, **kwargs)` in the model's constructor):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel(dynamic=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not the custom code will be called at each iteration. Let's fit, validate and evaluate with tiny datasets to avoid getting too much output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.507260322570801, 2.0566811561584473]"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively, you can compile a model with `run_eagerly=True`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae], run_eagerly=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.507260322570801, 2.0566811561584473]"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining custom optimizers is not very common, but in case you are one of the happy few who gets to write one, here is an example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMomentumOptimizer(keras.optimizers.Optimizer):\n",
    "    def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n",
    "        \"\"\"Call super().__init__() and use _set_hyper() to store hyperparameters\"\"\"\n",
    "        super().__init__(name, **kwargs)\n",
    "        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate)) # handle lr=learning_rate\n",
    "        self._set_hyper(\"decay\", self._initial_decay) # \n",
    "        self._set_hyper(\"momentum\", momentum)\n",
    "    \n",
    "    def _create_slots(self, var_list):\n",
    "        \"\"\"For each model variable, create the optimizer variable associated with it.\n",
    "        TensorFlow calls these optimizer variables \"slots\".\n",
    "        For momentum optimization, we need one momentum slot per model variable.\n",
    "        \"\"\"\n",
    "        for var in var_list:\n",
    "            self.add_slot(var, \"momentum\")\n",
    "\n",
    "    @tf.function\n",
    "    def _resource_apply_dense(self, grad, var):\n",
    "        \"\"\"Update the slots and perform one optimization step for one model variable\n",
    "        \"\"\"\n",
    "        var_dtype = var.dtype.base_dtype\n",
    "        lr_t = self._decayed_lr(var_dtype) # handle learning rate decay\n",
    "        momentum_var = self.get_slot(var, \"momentum\")\n",
    "        momentum_hyper = self._get_hyper(\"momentum\", var_dtype)\n",
    "        momentum_var.assign(momentum_var * momentum_hyper - (1. - momentum_hyper)* grad)\n",
    "        var.assign_add(momentum_var * lr_t)\n",
    "\n",
    "    def _resource_apply_sparse(self, grad, var):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {\n",
    "            **base_config,\n",
    "            \"learning_rate\": self._serialize_hyperparameter(\"learning_rate\"),\n",
    "            \"decay\": self._serialize_hyperparameter(\"decay\"),\n",
    "            \"momentum\": self._serialize_hyperparameter(\"momentum\"),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 0s 444us/step - loss: 4.9648\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 444us/step - loss: 1.7888\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 437us/step - loss: 1.0021\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 451us/step - loss: 0.7869\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 446us/step - loss: 0.7122\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbfe0a94d50>"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.Sequential([keras.layers.Dense(1, input_shape=[8])])\n",
    "model.compile(loss=\"mse\", optimizer=MyMomentumOptimizer())\n",
    "model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 11.\n",
    "See Appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12. Implement a custom layer that performs _Layer Normalization_\n",
    "_We will use this type of layer in Chapter 15 when using Recurrent Neural Networks._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a.\n",
    "_Exercise: The `build()` method should define two trainable weights *α* and *β*, both of shape `input_shape[-1:]` and data type `tf.float32`. *α* should be initialized with 1s, and *β* with 0s._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Solution: see below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: The `call()` method should compute the mean_ μ _and standard deviation_ σ _of each instance's features. For this, you can use `tf.nn.moments(inputs, axes=-1, keepdims=True)`, which returns the mean μ and the variance σ<sup>2</sup> of all instances (compute the square root of the variance to get the standard deviation). Then the function should compute and return *α*⊗(*X* - μ)/(σ + ε) + *β*, where ⊗ represents itemwise multiplication (`*`) and ε is a smoothing term (small constant to avoid division by zero, e.g., 0.001)._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LayerNormalization(keras.layers.Layer):\n",
    "    def __init__(self, eps=0.001, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.eps = eps\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.alpha = self.add_weight(\n",
    "            name=\"alpha\", shape=batch_input_shape[-1:],\n",
    "            initializer=\"ones\")\n",
    "        self.beta = self.add_weight(\n",
    "            name=\"beta\", shape=batch_input_shape[-1:],\n",
    "            initializer=\"zeros\")\n",
    "        super().build(batch_input_shape) # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        mean, variance = tf.nn.moments(X, axes=-1, keepdims=True)\n",
    "        return self.alpha * (X - mean) / (tf.sqrt(variance + self.eps)) + self.beta\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"eps\": self.eps}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that making _ε_ a hyperparameter (`eps`) was not compulsory. Also note that it's preferable to compute `tf.sqrt(variance + self.eps)` rather than `tf.sqrt(variance) + self.eps`. Indeed, the derivative of sqrt(z) is undefined when z=0, so training will bomb whenever the variance vector has at least one component equal to 0. Adding _ε_ within the square root guarantees that this will never happen."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c.\n",
    "_Exercise: Ensure that your custom layer produces the same (or very nearly the same) output as the `keras.layers.LayerNormalization` layer._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create one instance of each class, apply them to some data (e.g., the training set), and ensure that the difference is negligeable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=5.6045884e-08>"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = X_train.astype(np.float32)\n",
    "\n",
    "custom_layer_norm = LayerNormalization()\n",
    "keras_layer_norm = keras.layers.LayerNormalization()\n",
    "\n",
    "tf.reduce_mean(keras.losses.mean_absolute_error(\n",
    "    keras_layer_norm(X), custom_layer_norm(X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Yep, that's close enough. To be extra sure, let's make alpha and beta completely random and compare again:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=2.2921004e-08>"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_alpha = np.random.rand(X.shape[-1])\n",
    "random_beta = np.random.rand(X.shape[-1])\n",
    "\n",
    "custom_layer_norm.set_weights([random_alpha, random_beta])\n",
    "keras_layer_norm.set_weights([random_alpha, random_beta])\n",
    "\n",
    "tf.reduce_mean(keras.losses.mean_absolute_error(\n",
    "    keras_layer_norm(X), custom_layer_norm(X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Still a negligeable difference! Our custom layer works fine."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13. Train a model using a custom training loop to tackle the Fashion MNIST dataset\n",
    "_The Fashion MNIST dataset was introduced in Chapter 10._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a.\n",
    "_Exercise: Display the epoch, iteration, mean training loss, and mean accuracy over each epoch (updated at each iteration), as well as the validation loss and accuracy at the end of each epoch._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
    "X_train_full = X_train_full.astype(np.float32) / 255.\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
    "X_test = X_test.astype(np.float32) / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(100, activation=\"relu\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\"),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.SparseCategoricalAccuracy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "901e5649b50840538874aed5bab0d4ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f29690a5ade4bd8a6d164d106ba2d31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2ea6579132c48c087c7f59e6309387a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94fa1527062c4cf7a277a548bbddc855",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbcfc9a0c4b64151a18cf27080872dd3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc3831d9b98e488c837ba9644bf4b94a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        with trange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "            for step in steps:\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                with tf.GradientTape() as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))                    \n",
    "                status = OrderedDict()\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                steps.set_postfix(status)\n",
    "            y_pred = model(X_valid)\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(keras.metrics.sparse_categorical_accuracy(\n",
    "                tf.constant(y_valid, dtype=np.float32), y_pred))\n",
    "            steps.set_postfix(status)\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: Try using a different optimizer with a different learning rate for the upper layers and the lower layers._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_layers = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(100, activation=\"relu\"),\n",
    "])\n",
    "upper_layers = keras.models.Sequential([\n",
    "    keras.layers.Dense(10, activation=\"softmax\"),\n",
    "])\n",
    "model = keras.models.Sequential([\n",
    "    lower_layers, upper_layers\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_optimizer = keras.optimizers.SGD(lr=1e-4)\n",
    "upper_optimizer = keras.optimizers.Nadam(lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "loss_fn = keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.SparseCategoricalAccuracy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e81cf85cf7548748ec760afcbd71aa2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "475109b927d044a7bba030f234c67838",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af0a22afae4f47359f8fdfbac96e38bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5519ec96e42f4281987a84a5434a0734",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed04f31b3a7d4b3b9a0cd63b644e2ed5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfc4b1b4d40f4003ab8a3140a68ec883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        with trange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "            for step in steps:\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                with tf.GradientTape(persistent=True) as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                for layers, optimizer in ((lower_layers, lower_optimizer),\n",
    "                                          (upper_layers, upper_optimizer)):\n",
    "                    gradients = tape.gradient(loss, layers.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, layers.trainable_variables))\n",
    "                del tape\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))                    \n",
    "                status = OrderedDict()\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                steps.set_postfix(status)\n",
    "            y_pred = model(X_valid)\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(keras.metrics.sparse_categorical_accuracy(\n",
    "                tf.constant(y_valid, dtype=np.float32), y_pred))\n",
    "            steps.set_postfix(status)\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}