tf_code_investigation_optimizer_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tf_code_investigation_optimizer_v2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Ta5PsEv5uwMJ",
        "outputId": "ba65fb42-59cd-4397-9391-3671d68cfdff"
      },
      "source": [
        "import tensorflow as tf\n",
        "tf.__version__"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2.5.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CxKumSCa2BfU"
      },
      "source": [
        "# setup logging\n",
        "**Only run it once!!!**\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UOya2Hjfyfke"
      },
      "source": [
        "to check the code at a specific line"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k_nw3Oiwt-er",
        "outputId": "d5bdae85-8032-4dfe-98c0-558f58ea3959"
      },
      "source": [
        "!sed '636!d' /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "        self._create_all_weights(var_list)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p3TEVcGOysHY"
      },
      "source": [
        "to insert logging at a specific line"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8U_xwyL0yex8"
      },
      "source": [
        "!sed -i \"636 i \\ \\ \\ \\ \\ \\ \\ \\ tf.print('apply_gradients::with ops.init_scope: ', var_list[0].numpy(), var_list[1].numpy())\" /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QRkXf4-MyqYF"
      },
      "source": [
        "to check the code at a specific line"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZfICiT1gzTkP",
        "outputId": "64981a35-ad99-4eb5-adca-0d8327215d23"
      },
      "source": [
        "!sed '636!d' /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "        tf.print('apply_gradients::with ops.init_scope: ', var_list[0].numpy(), var_list[1].numpy())\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qeYqMayhzUGw"
      },
      "source": [
        "!sed -i \"630 i \\ \\ \\ \\ tf.print('apply_gradients is called')\" /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2kGxktyZzepi",
        "outputId": "6d520ba5-2b2a-480c-8be4-1b1482403b85"
      },
      "source": [
        "!sed '630!d' /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "    tf.print('apply_gradients is called')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JbACvhsh1x_2"
      },
      "source": [
        "!sed -i \"20 i import tensorflow as tf\" /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4U9sosvpzgJh",
        "outputId": "844c262c-a72b-49ba-9ea4-a87905a7464f"
      },
      "source": [
        "!sed '20!d' /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "import tensorflow as tf\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1hhQM4lU2JRU"
      },
      "source": [
        "# model training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DDMvZpVI07U6",
        "outputId": "a15b32a0-20b0-4305-d9cb-3914965d9f70"
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "x_train = np.array([[0, 0], [1, 1], [2, 2]])\n",
        "y_train = [0, 1, 1]\n",
        "x_train = np.asarray(x_train, np.float32)\n",
        "y_train = np.asarray(y_train, np.float32)\n",
        "\n",
        "initializer = tf.keras.initializers.Constant(.1)\n",
        "initializer(shape=(2,))\n",
        "\n",
        "optimizer = tf.keras.optimizers.SGD(\n",
        "    learning_rate=0.1, name='SGD'\n",
        ")\n",
        "\n",
        "model = tf.keras.models.Sequential([\n",
        "  tf.keras.layers.Dense(1, activation=tf.nn.sigmoid, kernel_initializer=initializer),\n",
        "])\n",
        "\n",
        "def custom_loss(y, a):\n",
        "    loss = -(y*tf.math.log(a) + (1-y)*tf.math.log(1-a))\n",
        "    return loss\n",
        "\n",
        "model.compile(loss=custom_loss,\n",
        "              optimizer=optimizer,\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model.fit(x_train, y_train, epochs=1, verbose=0)\n",
        "\n",
        "print('done')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "apply_gradients::with ops.init_scope:  array([[0.1],\n",
            "       [0.1]], dtype=float32) array([0.], dtype=float32)\n",
            "apply_gradients::with ops.init_scope:  array([[0.1],\n",
            "       [0.1]], dtype=float32) array([0.], dtype=float32)\n",
            "apply_gradients is called\n",
            "done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "doMyhgG01QiT"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}