3_weights_and_biases.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "106ddfec",
   "metadata": {},
   "source": [
    "## Weights and Biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5f381f33",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf \n",
    "\n",
    "from toy_model import create_model\n",
    "from tensorflow.keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c48c5b0",
   "metadata": {},
   "source": [
    "We can also explore weights and biases from the model's layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b84090fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "x_train_net = tf.expand_dims(x_train, axis=3)\n",
    "y_train_net = tf.one_hot(y_train, n_classes)\n",
    "\n",
    "x_train_net.shape, y_train_net.shape\n",
    "n_classes = tf.unique(y_test).y.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d28c6fd",
   "metadata": {},
   "source": [
    "- `histogram_freq`: Frequency (in epochs) at which to compute weight histograms for the layers of the model.\n",
    "- `write_images`: whether to write model weights to visualize as image in TensorBoard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "46cbd663",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam()\n",
    "loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
    "clf.compile(loss=loss, optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "\n",
    "tb_cback = TensorBoard(log_dir='logs',\n",
    "                       write_graph=True, \n",
    "                       histogram_freq=1, \n",
    "                       write_images=False,\n",
    "                       update_freq='epoch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dca377fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "188/188 [==============================] - 8s 40ms/step - loss: 0.0765 - accuracy: 0.9782 - val_loss: 0.0692 - val_accuracy: 0.9799\n",
      "CPU times: user 48.5 s, sys: 837 ms, total: 49.3 s\n",
      "Wall time: 7.88 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "history = clf.fit(x_train_net, \n",
    "                  y_train_net, \n",
    "                  batch_size=256, \n",
    "                  epochs=1, \n",
    "                  validation_split=0.2,\n",
    "                  callbacks=[tb_cback])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b1e5f82",
   "metadata": {},
   "source": [
    "## Saving weights and biases on custom TF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "191ec486",
   "metadata": {},
   "outputs": [],
   "source": [
    "from toy_model import train_step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ab820746",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds  = tf.data.Dataset.from_tensor_slices((x_train_net, y_train_net))\n",
    "train_ds  = train_ds.batch(256)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "\n",
    "writer = tf.summary.create_file_writer('hist')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "054ffebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, (x, y) in enumerate(train_ds):\n",
    "    t_loss, t_acc = train_step(clf, x, y, optimizer)\n",
    "    with writer.as_default():\n",
    "        for w in clf.weights:\n",
    "            tf.summary.histogram(w.name, w.value(), step=index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c49c5701",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}