GPT-1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "d_model = 512  # Dimension of the model\n",
    "d_ff = 2048  # Dimension of feedforward network\n",
    "seq_length = 20  # Sequence length\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize parameters\n",
    "def initialize_parameters():\n",
    "    params = {}\n",
    "    params['Wq'] = np.random.randn(d_model, d_model) * 0.1\n",
    "    params['Wk'] = np.random.randn(d_model, d_model) * 0.1\n",
    "    params['Wv'] = np.random.randn(d_model, d_model) * 0.1\n",
    "    params['Wo'] = np.random.randn(d_model, d_model) * 0.1\n",
    "    params['W1'] = np.random.randn(d_model, d_ff) * 0.1\n",
    "    params['W2'] = np.random.randn(d_ff, d_model) * 0.1\n",
    "    params['b1'] = np.zeros((1, d_ff))\n",
    "    params['b2'] = np.zeros((1, d_model))\n",
    "    return params\n",
    "\n",
    "params = initialize_parameters()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Self-Attention\n",
    "def self_attention(Q, K, V):\n",
    "    scores = np.matmul(Q, K.transpose(0, 2, 1)) / np.sqrt(Q.shape[-1])\n",
    "    # Apply the softmax trick for numerical stability\n",
    "    scores -= np.max(scores, axis=-1, keepdims=True)\n",
    "    weights = np.exp(scores) / np.sum(np.exp(scores), axis=-1, keepdims=True)\n",
    "    output = np.matmul(weights, V)\n",
    "    return output, weights\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Multi-Head Attention\n",
    "def multi_head_attention(X, params):\n",
    "    # Linear projections\n",
    "    Q = np.dot(X, params['Wq'])\n",
    "    K = np.dot(X, params['Wk'])\n",
    "    V = np.dot(X, params['Wv'])\n",
    "\n",
    "    # Self-attention mechanism\n",
    "    attn_output, _ = self_attention(Q, K, V)\n",
    "    attn_output = np.dot(attn_output, params['Wo'])  # Linear transformation\n",
    "    return attn_output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Feedforward Network\n",
    "def feedforward_network(X, params):\n",
    "    hidden = np.dot(X, params['W1']) + params['b1']\n",
    "    hidden = np.maximum(0, hidden)  # ReLU activation\n",
    "    output = np.dot(hidden, params['W2']) + params['b2']\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transformer Block\n",
    "def transformer_block(X, params):\n",
    "    # Apply multi-head attention\n",
    "    attn_output = multi_head_attention(X, params)\n",
    "    attn_output = X + attn_output  # Add & Norm\n",
    "\n",
    "    # Apply feedforward network\n",
    "    ff_output = feedforward_network(attn_output, params)\n",
    "    output = attn_output + ff_output  # Add & Norm\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dummy data for illustration\n",
    "X = np.random.randn(10, seq_length, d_model)  # (batch_size, seq_length, d_model)\n",
    "y = np.random.randint(0, 2, (10, seq_length, d_model))  # Target labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward pass through multiple transformer blocks\n",
    "for _ in range(12):  # Number of transformer blocks (like GPT-1)\n",
    "    X = transformer_block(X, params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 5.0785315130133246e+38\n"
     ]
    }
   ],
   "source": [
    "# Compute loss (simple mean squared error for illustration)\n",
    "loss = np.mean((X - y) ** 2)\n",
    "\n",
    "print(\"Loss:\", loss)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}