[theory]-explicit_constructions-d<T-mutual-coherence.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Q9BEuNPB1PuQ"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np_normal\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.decomposition import PCA\n",
        "from memory_exp import *\n",
        "from manual_models import *"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "bfg-4OhJ1PuT"
      },
      "outputs": [],
      "source": [
        "def coherence(phi):\n",
        "    m, n = phi.shape\n",
        "    a = phi\n",
        "    a = phi /  np.linalg.norm(phi, axis=0)\n",
        "    a_ = np.matmul(a.T.conj(), a)\n",
        "    a = np.abs(a_ - np.eye(a_.shape[0]))\n",
        "    return np.max(a, axis=None)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def welch(T,d):\n",
        "    assert T > d\n",
        "    return np.sqrt((T-d)/(d*(T-1)))\n",
        "\n",
        "data = [\n",
        "    (7, 3, [1, 2, 4]),\n",
        "    (7, 4, [0, 3, 5, 6]),\n",
        "    (13, 4, [0, 1, 3, 9]),\n",
        "    (11, 5, [1, 3, 4, 5, 9]),\n",
        "    (21, 5, [3, 6, 7, 12, 14]),\n",
        "    (11, 6, [0, 2, 6, 7, 8, 10]),\n",
        "    (31, 6, [1, 5, 11, 24, 25, 27]),\n",
        "    (15, 7, [0, 1, 2, 4, 5, 8, 10]),\n",
        "    (15, 8, [3, 6, 7, 9, 11, 12, 13, 14]),\n",
        "    (57, 8, [1, 6, 7, 9, 19, 38, 42, 49]),\n",
        "    (13, 9, [2, 4, 5, 6, 7, 8, 10, 11, 12]),\n",
        "    (37, 9, [1, 7, 9, 10, 12, 16, 26, 33, 34]),\n",
        "    (73, 9, [1, 2, 4, 8, 16, 32, 37, 55, 64]),\n",
        "    (40, 13, [0, 1, 3, 5, 9, 15, 22, 25, 26, 27, 34, 35, 38])\n",
        "]\n",
        "for T,d,_ in data:\n",
        "    print(T*2,d*2, welch(T=T,d=d), 1/welch(T=T,d=d)**2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1AjFBNh0fIEY",
        "outputId": "1dd128a8-5592-464b-891f-e5de40b3c412"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "14 6 0.4714045207910317 4.5\n",
            "14 8 0.3535533905932738 7.999999999999998\n",
            "26 8 0.4330127018922193 5.333333333333334\n",
            "22 10 0.34641016151377546 8.333333333333334\n",
            "42 10 0.4 6.249999999999999\n",
            "22 12 0.28867513459481287 12.0\n",
            "62 12 0.37267799624996495 7.199999999999999\n",
            "30 14 0.2857142857142857 12.250000000000002\n",
            "30 16 0.25 16.0\n",
            "114 16 0.33071891388307384 9.142857142857142\n",
            "26 18 0.19245008972987526 26.999999999999996\n",
            "74 18 0.2939723678960656 11.571428571428573\n",
            "146 18 0.31426968052735443 10.125000000000002\n",
            "80 26 0.23076923076923078 18.777777777777775\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "def fft_matrix(N):\n",
        "    # Initialize an N x N matrix\n",
        "    F_N = np.zeros((N, N), dtype=np.complex128)\n",
        "\n",
        "    # Fill the matrix with the FFT coefficients\n",
        "    for j in range(N):\n",
        "        for k in range(N):\n",
        "            F_N[j, k] = np.exp(2j * np.pi * j * k / N)\n",
        "\n",
        "    # Return the FFT matrix\n",
        "    return F_N\n",
        "\n",
        "# Example usage:\n",
        "N, k, cols = 15, 7, [0, 1, 2, 4, 5, 8, 10]\n",
        "embd = fft_matrix(N)#.T#.real\n",
        "embd = embd[:,cols] * 1/np.sqrt(k)\n",
        "#plt.imshow(embd)\n",
        "embd = embd #/ np.linalg.norm(embd,keepdims=True,axis=-1)\n",
        "print(embd.shape)\n",
        "overlap_matrix = (embd @ embd.conj().T)#.real.T\n",
        "\n",
        "def complex_to_real_matrix(A):\n",
        "    # Separate the real and imaginary parts\n",
        "    Re_A = np.real(A)\n",
        "    Im_A = np.imag(A)\n",
        "\n",
        "    # Construct the new real matrix\n",
        "    top = np.hstack((Re_A, -Im_A))\n",
        "    bottom = np.hstack((Im_A, Re_A))\n",
        "    B = np.vstack((top, bottom))\n",
        "\n",
        "    return B\n",
        "\n",
        "embd = complex_to_real_matrix(embd)\n",
        "print(embd.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5sGJC2IjRCqQ",
        "outputId": "9a78f5bf-3089-4382-dfa9-a9ad20c1a8e6"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(15, 7)\n",
            "(30, 14)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "embd = embd[:,-T:]\n",
        "current_phi = embd"
      ],
      "metadata": {
        "id": "MsP2uydFjFpk"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "T = 30\n",
        "L = 10\n",
        "d = 14\n",
        "\n",
        "config = {\n",
        "    'T': T,\n",
        "    'model_dim': d,\n",
        "    'p': T,\n",
        "    'seq_len':L,\n",
        "    'attention_input': 'only_sem',\n",
        "    'no_softmax': True,\n",
        "    'dataset_type': 'backward'\n",
        "}\n"
      ],
      "metadata": {
        "id": "z03-HpbDh1fM"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "has_BOS =  'BOS' in config['dataset_type']\n",
        "L = config['seq_len']\n",
        "d = config['model_dim']\n",
        "model = TransformerSeq2Seq(T=config['T']+int(has_BOS),\n",
        "                            model_dim=config['model_dim'],\n",
        "                            p=config['p'],n_classes=config['seq_len']+1,\n",
        "                            L=config['seq_len'],\n",
        "                            attention_input=config['attention_input'],\n",
        "                            use_softmax=not config['no_softmax']).to(device)"
      ],
      "metadata": {
        "id": "La7FAPdEh9fk"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "word_embeddings_basis = current_phi# generate_orthogonal_vectors(config['model_dim']) #current_phi.T#\n",
        "\n",
        "word_embeddings = word_embeddings_basis\n",
        "model.semantic_emb.weight.data = torch.tensor(word_embeddings).float().to(device)\n",
        "\n",
        "model.token_mixer.Q.data = 1/L * torch.eye(config['model_dim']).to(device) * np.sqrt(config['model_dim'])\n",
        "model.token_mixer.K.data = torch.eye(config['model_dim']).to(device)\n",
        "\n",
        "softmax_table = []\n",
        "for i in range(1,config['seq_len']+1):\n",
        "    x = [0] * i + (L-i) * [1]\n",
        "    x = torch.tensor(x ).to(device).reshape(1,-1)\n",
        "    # load attn matrix\n",
        "    r = model(x)\n",
        "    attn_probs = model.attn_probs.cpu().detach().numpy()\n",
        "    softmax_table.append(attn_probs[0,0,0]*i)\n",
        "\n",
        "W_1 = torch.tensor(word_embeddings).float().to(device)\n",
        "b_1 = -1.0\n",
        "model.fc1.weight.data = torch.tensor(W_1).float().to(device)\n",
        "model.fc1.bias.data = torch.tensor(b_1).float().to(device)\n",
        "\n",
        "softmax_table = np.array(softmax_table)\n",
        "descision_boundaries = softmax_table[:-1] + (softmax_table[1:] - softmax_table[:-1]) -0.01# + (softmax_table[1:] - softmax_table[:-1]) * 0.001\n",
        "print(descision_boundaries)\n",
        "ws, bs = implement_W2(descision_boundaries)\n",
        "# weight 0 is always smallest\n",
        "ws = np.array([-100] + list(ws))\n",
        "bs = np.array([-100] + list(bs))\n",
        "W_2, b_2 = ws, bs\n",
        "W_2 = W_2[:,None].repeat(config['T'],1)\n",
        "model.fc2.weight.data = torch.tensor(W_2).float().to(device)\n",
        "model.fc2.bias.data = torch.tensor(b_2).float().to(device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZDnvZHgDDuuH",
        "outputId": "c61749c9-48b9-4ad3-d6cb-7b2ece6b3fd2"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.19       0.29       0.39000001 0.49000001 0.59000001 0.69000001\n",
            " 0.79000001 0.89000001 0.99000001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-13-1b3bd76d99c8>:20: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  model.fc1.weight.data = torch.tensor(W_1).float().to(device)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def test(model, T , L, has_BOS = True, verbose=False):\n",
        "    works = True\n",
        "    res = []\n",
        "    failiures = 0\n",
        "    for i in range(1000):\n",
        "        for k in range(1,L+1):\n",
        "\n",
        "            a = np.random.randint(1,T)\n",
        "            b = np.random.randint(1,T-1)\n",
        "            if a == b:\n",
        "                b = T-1\n",
        "            if has_BOS:\n",
        "                x = [T] + [a] * k + (L-k) * [b]\n",
        "            else:\n",
        "                x = [a] * k + (L-k) * [b]\n",
        "            x = torch.tensor(x ).to(device).reshape(1,-1)\n",
        "            r = model(x)\n",
        "            #print(model.attn_scores)\n",
        "\n",
        "            res.append((k,model.b[0,0,a].item()))\n",
        "            if has_BOS:\n",
        "                #print(r[0,1])\n",
        "                x_hat = r[0,1].argmax().item()\n",
        "            else:\n",
        "                x_hat = r[0,0].argmax().item()\n",
        "            if x_hat != k:\n",
        "                if verbose:\n",
        "                    #print(model.b)\n",
        "                    print(f\"I got {x_hat}, but should have been {k}.\")\n",
        "                    failiures += 1\n",
        "                works = False\n",
        "\n",
        "    print('errors ', failiures/(1000*L))\n",
        "    if works:\n",
        "        print(\"Test passed, model works!\")\n",
        "    else:\n",
        "        print(\":((( Test failed, model does not work!\")\n",
        "\n",
        "    return res"
      ],
      "metadata": {
        "id": "wp4yinG_m9FY"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "res = test(model, T, L, has_BOS='BOS' in config['dataset_type'], verbose=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u3BGFWBuh6o8",
        "outputId": "111ff01d-02a1-4aa1-f5b4-c390b25eabcc"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "errors  0.0\n",
            "Test passed, model works!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "coherence(current_phi.T)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1RKfK1_fjX_1",
        "outputId": "aeaf12ab-f3b5-468a-e123-d9e1831d22c7"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.28571428571428653"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "aP451cuZl0Y8"
      },
      "outputs": [],
      "source": [
        "import jax.numpy as jnp\n",
        "import jax\n",
        "from jax import grad, jit\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np_normal\n",
        "%matplotlib inline\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "itTcMc9xl0ZA"
      },
      "outputs": [],
      "source": [
        "@jit\n",
        "def coherence(phi, _=None):\n",
        "    m, n = phi.shape\n",
        "    a = phi\n",
        "    a = phi /  jnp.linalg.norm(phi, axis=0)\n",
        "    a_ = jnp.matmul(a.T.conj(), a)\n",
        "    a = jnp.abs(a_ - jnp.eye(a_.shape[0]))\n",
        "    return jnp.max(a, axis=None)\n",
        "\n",
        "@jit\n",
        "def soft_coherence(phi,inv_temp=15):\n",
        "    m, n = phi.shape\n",
        "    a = phi\n",
        "    a = phi /  jnp.linalg.norm(phi, axis=0)\n",
        "    a_ = jnp.matmul(a.T.conj(), a)\n",
        "    a = jnp.abs(a_ - jnp.eye(a_.shape[0]))\n",
        "    return jnp.mean(jnp.exp(a * inv_temp))\n",
        "\n",
        "grad_coherence = jit(grad(coherence))\n",
        "grad_soft_coherence = jit(grad(soft_coherence))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "5QMQcVnS1PuU"
      },
      "outputs": [],
      "source": [
        "def optimize(phi, steps, grad_fn):\n",
        "    coherences = []\n",
        "    soft_coherences = []\n",
        "    kappa = 0.1\n",
        "    for i in range(steps):\n",
        "        kappa += 1/steps\n",
        "        coh = coherence(phi)\n",
        "        soft_coh = soft_coherence(phi,kappa)\n",
        "        g = grad_soft_coherence(phi + np_normal.random.normal(0, 0.00001, size=phi.shape))\n",
        "        phi = phi - g\n",
        "        if i%100==0:\n",
        "            #print(i, coh, soft_coh)\n",
        "            pass\n",
        "    coherences.append(coherence(phi))\n",
        "    return phi, coherences, soft_coherences\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "d=12\n",
        "T=32"
      ],
      "metadata": {
        "id": "69kuQaK9kR_7"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "djoBv4Lz1PuW",
        "outputId": "afc8abac-1b3d-4fca-f2d1-1eb1210b5a3f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.40883902\n",
            "0.33624908\n",
            "0.38297036\n",
            "0.3092438\n",
            "0.3361281\n",
            "0.34776464\n",
            "0.36798733\n",
            "0.31491444\n",
            "0.39204502\n",
            "0.3586696\n",
            "0.4403425\n",
            "0.38895074\n",
            "0.36322242\n",
            "0.31081682\n",
            "0.32563943\n",
            "0.36015484\n",
            "0.3447719\n",
            "0.36414498\n",
            "0.32182646\n",
            "0.38121673\n",
            "0.3715696\n",
            "0.37212643\n",
            "0.35196003\n",
            "0.42913303\n",
            "0.36856994\n",
            "0.38486266\n",
            "0.3981856\n",
            "0.3334132\n",
            "0.3053283\n",
            "0.43559226\n",
            "0.49907112\n",
            "0.37238127\n",
            "0.3621313\n",
            "0.32590637\n",
            "0.37581578\n",
            "0.45462912\n",
            "0.34916463\n",
            "0.3946003\n",
            "0.316701\n",
            "0.42596495\n",
            "0.39702222\n",
            "0.49741247\n",
            "0.36408168\n",
            "0.4338788\n",
            "0.31539893\n",
            "0.39504975\n",
            "0.37330562\n",
            "0.3921702\n",
            "0.31907994\n",
            "0.4909868\n",
            "0.31412703\n",
            "0.36028075\n",
            "0.37955633\n",
            "0.31986406\n",
            "0.32136115\n",
            "0.42047858\n",
            "0.30536643\n",
            "0.4238991\n",
            "0.31958866\n",
            "0.3288729\n",
            "0.35326904\n",
            "0.3917241\n",
            "0.3548895\n",
            "0.32975096\n",
            "0.35697678\n",
            "0.38738036\n",
            "0.50631773\n",
            "0.3789734\n",
            "0.3304639\n",
            "0.4100708\n",
            "0.32235295\n",
            "0.3181391\n",
            "0.40569332\n",
            "0.35960707\n",
            "0.36470845\n",
            "0.324301\n",
            "0.36265236\n",
            "0.46055776\n",
            "0.39843872\n",
            "0.38193977\n",
            "0.33856222\n",
            "0.36727798\n",
            "0.3682106\n",
            "0.38466957\n",
            "0.39068574\n",
            "0.3686409\n",
            "0.2994248\n",
            "0.3298288\n",
            "0.33990058\n",
            "0.32348332\n",
            "0.47247487\n",
            "0.30197197\n",
            "0.34793928\n",
            "0.36847618\n",
            "0.45998037\n",
            "0.3712228\n",
            "0.51768094\n",
            "0.3670657\n",
            "0.35264534\n",
            "0.36912957\n"
          ]
        }
      ],
      "source": [
        "current_coherence = np.inf\n",
        "current_phi = None\n",
        "for i in range(100):\n",
        "  phi = np_normal.random.randn(d, T)\n",
        "  phi_new , coherences, _ = optimize(phi, 5000, grad_coherence); # _soft\n",
        "  print(coherences[-1])\n",
        "  if coherences[-1] < current_coherence:\n",
        "    current_phi = phi_new\n",
        "    current_coherence = coherences[-1]\n",
        "\n",
        "current_phi = current_phi /  jnp.linalg.norm(current_phi, axis=0)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_coherence"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qe6fMhnPl2sq",
        "outputId": "4d009690-ad23-4753-976a-0df04f6d455d"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Array(0.2994248, dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_phi = np.array(current_phi)"
      ],
      "metadata": {
        "id": "PBQKZdBomV9n"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "T = 32\n",
        "L = 10 #(works until L=12)\n",
        "d = 12 # ok with mc = 0.33\n",
        "\n",
        "config = {\n",
        "    'T': T,\n",
        "    'model_dim': d,\n",
        "    'p': T,\n",
        "    'seq_len':L,\n",
        "    'attention_input': 'only_sem',\n",
        "    'no_softmax': True,\n",
        "    'dataset_type': 'backward'\n",
        "}\n"
      ],
      "metadata": {
        "id": "NZSnILpVmZ-S"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "has_BOS =  'BOS' in config['dataset_type']\n",
        "L = config['seq_len']\n",
        "d = config['model_dim']\n",
        "model = TransformerSeq2Seq(T=config['T']+int(has_BOS),\n",
        "                            model_dim=config['model_dim'],\n",
        "                            p=config['p'],n_classes=config['seq_len']+1,\n",
        "                            L=config['seq_len'],\n",
        "                            attention_input=config['attention_input'],\n",
        "                            use_softmax=not config['no_softmax']).to(device)"
      ],
      "metadata": {
        "id": "3RWoYwBjmZ-U"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "word_embeddings_basis = current_phi.T# generate_orthogonal_vectors(config['model_dim']) #current_phi.T#\n",
        "\n",
        "word_embeddings = word_embeddings_basis\n",
        "model.semantic_emb.weight.data = torch.tensor(word_embeddings).float().to(device)\n",
        "\n",
        "model.token_mixer.Q.data = 1/L * torch.eye(config['model_dim']).to(device) * np.sqrt(config['model_dim'])\n",
        "model.token_mixer.K.data = torch.eye(config['model_dim']).to(device)\n",
        "\n",
        "softmax_table = []\n",
        "for i in range(1,config['seq_len']+1):\n",
        "    x = [0] * i + (L-i) * [1]\n",
        "    x = torch.tensor(x ).to(device).reshape(1,-1)\n",
        "    # load attn matrix\n",
        "    r = model(x)\n",
        "    attn_probs = model.attn_probs.cpu().detach().numpy()\n",
        "    softmax_table.append(attn_probs[0,0,0]*i)\n",
        "\n",
        "W_1 = torch.tensor(word_embeddings).float().to(device)\n",
        "b_1 = -1.0\n",
        "model.fc1.weight.data = torch.tensor(W_1).float().to(device)\n",
        "model.fc1.bias.data = torch.tensor(b_1).float().to(device)\n",
        "\n",
        "softmax_table = np.array(softmax_table)\n",
        "descision_boundaries = softmax_table[:-1] + (softmax_table[1:] - softmax_table[:-1]) -0.01# + (softmax_table[1:] - softmax_table[:-1]) * 0.001\n",
        "print(descision_boundaries)\n",
        "ws, bs = implement_W2(descision_boundaries)\n",
        "# weight 0 is always smallest\n",
        "ws = np.array([-100] + list(ws))\n",
        "bs = np.array([-100] + list(bs))\n",
        "W_2, b_2 = ws, bs\n",
        "W_2 = W_2[:,None].repeat(config['T'],1)\n",
        "model.fc2.weight.data = torch.tensor(W_2).float().to(device)\n",
        "model.fc2.bias.data = torch.tensor(b_2).float().to(device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1e83235-5d5c-4500-bc81-54b0b14875cc",
        "id": "2K_DavnAmZ-U"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.18999999 0.28999998 0.38999998 0.48999997 0.58999996 0.68999996\n",
            " 0.78999995 0.88999995 0.98999994]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-27-0acd37579640>:20: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  model.fc1.weight.data = torch.tensor(W_1).float().to(device)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "res = test(model, T, L, has_BOS='BOS' in config['dataset_type'], verbose=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9621b3f4-40a3-4e78-b8b7-b59de7943a0d",
        "id": "ue_IYA_ymZ-V"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "errors  0.0\n",
            "Test passed, model works!\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}