src/weight_decay.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Different weight regularizations in a sandbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.simple import SimpleCNN\n",
    "import numpy as np\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "from util.regularization import WeightRegularization\n",
    "from torch_optimizer import SGDW\n",
    "from optimizer.sgdw import SGDW as mySGDW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset():\n",
    "    _imgs = torch.ones(size=(1,1,1,1))\n",
    "    _true = torch.ones(_imgs.shape[0], dtype=torch.long)\n",
    "    model = SimpleCNN(img_size=_imgs.shape[-2:], in_channels=_imgs.shape[1], num_classes=2, num_layers=1, num_channels=1, kernel_size=1)\n",
    "    model.fc.bias = torch.nn.Parameter(torch.zeros((1)))\n",
    "    model.fc.weight = torch.nn.Parameter(torch.full(size=model.fc.weight.shape, fill_value=-3/np.prod(model.fc.weight.shape), requires_grad=True))\n",
    "    model.conv1.bias = torch.nn.Parameter(torch.zeros((model.num_channels)))\n",
    "    model.conv1.weight = torch.nn.Parameter(torch.full(size=model.conv1.weight.shape, fill_value=1.0/np.prod(model.conv1.weight.shape), requires_grad=True))\n",
    "    return model, _imgs, _true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[[[1.]]]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-1.5000],\n",
      "        [-1.5000]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "loss = torch.nn.CrossEntropyLoss()\n",
    "model, _imgs, _true = reset()\n",
    "\n",
    "print(model.conv1.weight)\n",
    "print(model.fc.weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L2\n",
    "### Weight decay in pytorch SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[0.]]]])\n",
      "tensor([[0.],\n",
      "        [0.]])\n",
      "Parameter containing:\n",
      "tensor([[[[0.9900]]]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-1.4850],\n",
      "        [-1.4850]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "model, _imgs, _true = reset()\n",
    "out = model(_imgs)\n",
    "l = loss(out, _true)*0\n",
    "l.backward()\n",
    "print(model.conv1.weight.grad)\n",
    "print(model.fc.weight.grad)\n",
    "optimizer = SGD(model.parameters(), lr=0.1, weight_decay=0.1)\n",
    "optimizer.step()\n",
    "print(model.conv1.weight)\n",
    "print(model.fc.weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decoupled Weight decay / L2 norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[0.]]]])\n",
      "tensor([[0.],\n",
      "        [0.]])\n",
      "Parameter containing:\n",
      "tensor([[[[0.9000]]]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-1.3500],\n",
      "        [-1.3500]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "model, _imgs, _true = reset()\n",
    "out = model(_imgs)\n",
    "l = loss(out, _true)*0\n",
    "l.backward()\n",
    "print(model.conv1.weight.grad)\n",
    "print(model.fc.weight.grad)\n",
    "optimizer = mySGDW(model.parameters(), lr=0.1, weight_decay=0.1)\n",
    "optimizer.step()\n",
    "print(model.conv1.weight)\n",
    "print(model.fc.weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### External weight regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.5500, grad_fn=<AddBackward0>)\n",
      "tensor([[[[0.2000]]]])\n",
      "tensor([[-0.3000],\n",
      "        [-0.3000]])\n",
      "Parameter containing:\n",
      "tensor([[[[0.9800]]]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-1.4700],\n",
      "        [-1.4700]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "model, _imgs, _true = reset()\n",
    "regularizer = WeightRegularization(model, weight_decay=0.1)\n",
    "out = model(_imgs)\n",
    "l = loss(out, _true)*0 + regularizer.penalty()\n",
    "print(l)\n",
    "l.backward()\n",
    "print(model.conv1.weight.grad)\n",
    "print(model.fc.weight.grad)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
    "optimizer.step()\n",
    "print(model.conv1.weight)\n",
    "print(model.fc.weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L1\n",
    "### \"SGDW\" -> implementation is effectively L1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[0.]]]])\n",
      "tensor([[0.],\n",
      "        [0.]])\n",
      "Parameter containing:\n",
      "tensor([[[[0.9000]]]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-1.3500],\n",
      "        [-1.3500]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "model, _imgs, _true = reset()\n",
    "out = model(_imgs)\n",
    "l = loss(out, _true) * 0\n",
    "l.backward()\n",
    "print(model.conv1.weight.grad)\n",
    "print(model.fc.weight.grad)\n",
    "optimizer = SGDW(model.parameters(), lr=0.1, weight_decay=0.1)\n",
    "optimizer.step()\n",
    "print(model.conv1.weight)\n",
    "print(model.fc.weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### proper L1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[0.]]]])\n",
      "tensor([[0.],\n",
      "        [0.]])\n",
      "Parameter containing:\n",
      "tensor([[[[0.9000]]]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-1.4000],\n",
      "        [-1.4000]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "model, _imgs, _true = reset()\n",
    "out = model(_imgs)\n",
    "l = loss(out, _true) * 0\n",
    "l.backward()\n",
    "print(model.conv1.weight.grad)\n",
    "print(model.fc.weight.grad)\n",
    "optimizer = mySGDW(model.parameters(), lr=0.1, weight_decay=0.1, weight_norm=1)\n",
    "optimizer.step()\n",
    "print(model.conv1.weight)\n",
    "print(model.fc.weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### external l1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.4000, grad_fn=<AddBackward0>)\n",
      "tensor([[[[0.1000]]]])\n",
      "tensor([[-0.1000],\n",
      "        [-0.1000]])\n",
      "Parameter containing:\n",
      "tensor([[[[0.9900]]]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([[-1.4900],\n",
      "        [-1.4900]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "model, _imgs, _true = reset()\n",
    "regularizer = WeightRegularization(model, weight_decay=0.1, p = 1)\n",
    "out = model(_imgs)\n",
    "l = loss(out, _true)*0 + regularizer.penalty()\n",
    "print(l)\n",
    "l.backward()\n",
    "print(model.conv1.weight.grad)\n",
    "print(model.fc.weight.grad)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
    "optimizer.step()\n",
    "print(model.conv1.weight)\n",
    "print(model.fc.weight)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "receptive-fields",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}