Untitled1-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/NIGG/anaconda/lib/python3.5/site-packages/matplotlib/__init__.py:1035: UserWarning: Duplicate key in file \"/Users/NIGG/.matplotlib/matplotlibrc\", line #515\n",
      "  (fname, cnt))\n",
      "/Users/NIGG/anaconda/lib/python3.5/site-packages/matplotlib/__init__.py:1035: UserWarning: Duplicate key in file \"/Users/NIGG/.matplotlib/matplotlibrc\", line #516\n",
      "  (fname, cnt))\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/NIGG/anaconda/lib/python3.5/site-packages/pandas/io/data.py:33: FutureWarning: \n",
      "The pandas.io.data module is moved to a separate package (pandas-datareader) and will be removed from pandas in a future version.\n",
      "After installing the pandas-datareader package (https://github.com/pydata/pandas-datareader), you can change the import ``from pandas.io import data, wb`` to ``from pandas_datareader import data, wb``.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "#　統計用ツール\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.tsa.api as tsa\n",
    "from patsy import dmatrices\n",
    "\n",
    "#描画\n",
    "import matplotlib.pyplot as plt\n",
    "from pandas.tools.plotting import autocorrelation_plot\n",
    "\n",
    "#株価\n",
    "import pandas as pd\n",
    "import pandas.io.data as web\n",
    "\n",
    "#深層学習\n",
    "import chainer\n",
    "from chainer import cuda, Function, gradient_check, Variable, optimizers, serializers, utils\n",
    "from chainer import Link, Chain, ChainList\n",
    "import chainer.functions as F\n",
    "import chainer.links as L\n",
    "\n",
    "#k平均法\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas.io.data as web\n",
    "import quandl\n",
    "\n",
    "\n",
    "def setdata(s, e, t='Adj Close'):\n",
    "    start = datetime.datetime(s[0], s[1], s[2])\n",
    "    end = datetime.datetime(e[0], e[1], e[2])\n",
    "    N225 = web.DataReader('^N225', 'yahoo', start, end)[t]\n",
    "    DJI = web.DataReader('^DJI', 'yahoo', start, end)[t]\n",
    "    IXIC = web.DataReader('^IXIC', 'yahoo', start, end)[t]\n",
    "    jpy = web.DataReader('DEXJPUS', 'fred')\n",
    "    jpy = jpy.ix[start:end]\n",
    "    usdjpy = quandl.get(\"BOE/XUDLJYD\", start_date=start, end_date=end).assign(\n",
    "                    USDJPY_roc = lambda df: df[\"Value\"].pct_change()\n",
    "                ).rename(columns={\n",
    "                    \"Value\" : \"USDJPY\"\n",
    "                })\n",
    "    eurjpy = quandl.get(\"BOE/XUDLBK63\", start_date=start, end_date=end).assign(\n",
    "                    EURJPY_roc = lambda df: df[\"Value\"].pct_change()\n",
    "                ).rename(columns={\n",
    "                    \"Value\" : \"EURJPY\"\n",
    "                })\n",
    "    gbpjpy = quandl.get(\"BOE/XUDLJYS\", start_date=start, end_date=end).assign(\n",
    "                    EURJPY_roc = lambda df: df[\"Value\"].pct_change()\n",
    "                ).rename(columns={\n",
    "                    \"Value\" : \"GBPJPY\"\n",
    "                })\n",
    "    usdjpy = usdjpy[\"USDJPY\"]\n",
    "    eurjpy = eurjpy[\"EURJPY\"]\n",
    "    gbpjpy = gbpjpy[\"GBPJPY\"]\n",
    "    \n",
    "    dataframe = pd.concat([N225, DJI, IXIC, usdjpy, eurjpy, gbpjpy], axis=1).dropna()\n",
    "    \n",
    "    datas = np.array(dataframe)\n",
    "    target = datas.T[0]\n",
    "\n",
    "    return datas, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s = (1990, 1, 1)\n",
    "e = (2015, 9, 30)\n",
    "t = 'Adj Close'\n",
    "start = datetime.datetime(s[0], s[1], s[2])\n",
    "end = datetime.datetime(e[0], e[1], e[2])\n",
    "SaP = web.DataReader('GSPC', 'yahoo', start, end)[t]\n",
    "N225 = web.DataReader('^N225', 'yahoo', start, end)[t]\n",
    "DJI = web.DataReader('^DJI', 'yahoo', start, end)[t]\n",
    "IXIC = web.DataReader('^IXIC', 'yahoo', start, end)[t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "SaP = np.array([SaP[1:].values - SaP[:-1].values]).T*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "kms = KMeans(n_clusters=3).fit_predict(SaP)\n",
    "\n",
    "mean1 = SaP[kms == 0].mean()\n",
    "var1 = SaP[kms == 0].var()\n",
    "mean2 = SaP[kms == 1].mean()\n",
    "var2 = SaP[kms == 1].var()\n",
    "mean3 = SaP[kms == 2].mean()\n",
    "var3 = SaP[kms == 2].var()\n",
    "\n",
    "for t in range(100):\n",
    "    f = SaP[t:t+20]\n",
    "\n",
    "    v = np.array([])\n",
    "    for i in range(20):\n",
    "        v1 = np.round(np.exp(-1*(f[i] - mean1)**2/var1), 5)\n",
    "        v2 = np.round(np.exp(-1*(f[i] - mean2)**2/var2), 5)\n",
    "        v3 = np.round(np.exp(-1*(f[i] - mean3)**2/var3), 5)\n",
    "        v = np.append(v, v1)\n",
    "        v = np.append(v, v2)\n",
    "        v = np.append(v, v3)\n",
    "\n",
    "    if t != 0:\n",
    "        fs = np.c_[fs, v]\n",
    "    else:\n",
    "        fs = np.array([v]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class AE(Chain):\n",
    "    def __init__(self):\n",
    "        super(AE, self).__init__(\n",
    "            l1 = L.Linear(60, 20),\n",
    "            l2 = L.Linear(20, 60),\n",
    "        )\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        bv, fv = self.fwd(x)\n",
    "        loss = F.mean_squared_error(bv, x)\n",
    "        return loss\n",
    "    \n",
    "    def fwd(self, x):\n",
    "        fv = F.sigmoid(self.l1(x))\n",
    "        bv = self.l2(fv)\n",
    "        return bv, fv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(1000):\n",
    "    ft = np.array([fs[:,0]], dtype='float32')\n",
    "\n",
    "    model = AE()\n",
    "    optimizer = optimizers.SGD()\n",
    "    optimizer.setup(model)\n",
    "\n",
    "    n = 60\n",
    "    bs = 5\n",
    "    \n",
    "    for j in range(10):\n",
    "        x = Variable(ft)\n",
    "        model.zerograds()\n",
    "        loss = model(x)\n",
    "        loss.backward()\n",
    "        optimizer.weight_decay(0.0005)\n",
    "        optimizer.update()\n",
    "    \n",
    "    if i != 0:\n",
    "        Ft = model.fwd(x)[1].data\n",
    "        Fs = np.append(Fs, Ft, axis=0)\n",
    "    else:\n",
    "        Fs = model.fwd(x)[1].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CAR(Chain):\n",
    "    def __init__(self, unit1, unit2, unit3, col_num):\n",
    "        super(CAR, self).__init__(\n",
    "            l1 = L.Linear(col_num, unit1),\n",
    "            l2 = L.Linear(self.unit1, self.unit1),\n",
    "            l3 = L.Linear(self.unit1, self.unit2),\n",
    "            l4 = L.Linear(self.unit2, self.unit3),\n",
    "            l5 = L.Linear(self.unit3, self.unit3),\n",
    "            l6 = L.Linear(self.unit3, 1),\n",
    "        )\n",
    "    \n",
    "    def __call__(self, x, y):\n",
    "        fv = self.fwd(x, y)\n",
    "        loss = F.mean_squared_error(fv, y)\n",
    "        return loss\n",
    "    \n",
    "    def fwd(self, x, y):\n",
    "        h1 = F.sigmoid(self.l1(x))\n",
    "        h2 = F.sigmoid(self.l2(h1))\n",
    "        h3 = F.sigmoid(self.l3(h2))\n",
    "        h4 = F.sigmoid(self.l4(h3))\n",
    "        h5 = F.sigmoid(self.l5(h4))\n",
    "        h6 = self.l6(h5)\n",
    "        return h6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class RNN(chainer.Chain):\n",
    "    def __init__(self):\n",
    "        super(RNN, self).__init__(\n",
    "            w = L.Linear(20, 1),\n",
    "            b = L.Linear(1, 1),\n",
    "            u = L.Linear(1, 1),\n",
    "        )\n",
    "        \n",
    "    def __call__(self, Fs):\n",
    "        dUt = 0\n",
    "        d_1 = Variable(np.zeros((1, 1), dtype=np.float32))\n",
    "        old_grad = np.array([np.zeros(21)], dtype='float32')\n",
    "        one = Variable(np.array([[1]], dtype=np.float32))\n",
    "        zero = Variable(np.array([[0]], dtype=np.float32))\n",
    "        U = Variable(np.array([[0]], dtype=np.float32))\n",
    "        for i in range(len(Fs[:, 0])):\n",
    "            x_k = Variable(np.array([Farray[i]], dtype=np.float32))\n",
    "            d = F.tanh(self.w(x_k) + self.b(one) + self.u(d_1))\n",
    "            z = Variable(np.array([[Farray[i][-1]]], dtype=np.float32))\n",
    "            R = F.forget(f, d, d_1, z)\n",
    "            R.data = [[0]] if R.data[0][0] == 'nan' else R.data\n",
    "            self.g = R\n",
    "            U = U.__radd__(R)\n",
    "            d_1 = d\n",
    "        \n",
    "        loss = F.mean_squared_error(U.__pow__(1/2), zero)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def f(d, d_1, z):\n",
    "    c = 100\n",
    "    \n",
    "    R = d_1.__rmatmul__(z) - c*(d - d_1).__abs__()\n",
    "    \n",
    "    return R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-73-9456a96f971d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-71-ad6f3709ef83>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, Fs)\u001b[0m\n\u001b[1;32m     18\u001b[0m             \u001b[0md\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_k\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md_1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m             \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mFarray\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m             \u001b[0mR\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m             \u001b[0mR\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mR\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'nan'\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mR\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mR\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/functions/util/forget.py\u001b[0m in \u001b[0;36mforget\u001b[0;34m(func, *xs)\u001b[0m\n\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m     \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mForget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *inputs)\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0;31m# Forward prop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0min_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0min_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitervalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/functions/util/forget.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mvariable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvolatile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mout\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/functions/util/forget.py\u001b[0m in \u001b[0;36m_call_func\u001b[0;34m(self, xs)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-72-f1418786a6ff>\u001b[0m in \u001b[0;36mf\u001b[0;34m(d, d_1, z)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mR\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0md_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__rmatmul__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0md_1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__abs__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mR\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/functions/math/basic_math.py\u001b[0m in \u001b[0;36mmul\u001b[0;34m(self, rhs)\u001b[0m\n\u001b[1;32m    275\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mMul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrhs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m     \u001b[0m_check_constant_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrhs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mMulConstant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrhs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *inputs)\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0;31m# Forward prop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0min_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0min_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitervalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/functions/math/basic_math.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    258\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_preprocess_const\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 260\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforce_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    262\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/chainer/utils/__init__.py\u001b[0m in \u001b[0;36mforce_array\u001b[0;34m(x, dtype)\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;31m# We need to convert such a value to a 0-dimension array because `Function`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;31m# object needs to return an `numpy.ndarray`.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misscalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/NIGG/anaconda/lib/python3.5/site-packages/numpy/core/numeric.py\u001b[0m in \u001b[0;36misscalar\u001b[0;34m(num)\u001b[0m\n\u001b[1;32m   1968\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1969\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1970\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0misscalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1971\u001b[0m     \"\"\"\n\u001b[1;32m   1972\u001b[0m     \u001b[0mReturns\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mtype\u001b[0m \u001b[0mof\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mnum\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0ma\u001b[0m \u001b[0mscalar\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = RNN()\n",
    "optimizer = optimizers.SGD()\n",
    "optimizer.setup(model)\n",
    "\n",
    "Farray = Fs\n",
    "\n",
    "c = 1\n",
    "c0 = 10\n",
    "dU = 0\n",
    "\n",
    "rho = 1\n",
    "while True:\n",
    "    rho_c = min(rho, rho**(c0/c))\n",
    "    \n",
    "    x = Variable(Fs)\n",
    "    loss = model(x)\n",
    "    loss.backward()\n",
    "    optimizer.update()\n",
    "    \n",
    "    c += 1\n",
    "    if c == 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nan = model.g.data[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ nan]], dtype=float32)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.g.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.float32"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss.data = np.array(10.0, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss.__pow__(1/2).data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "while True:\n",
    "    rho_c = min(rho, rho**(c0/c))\n",
    "    \n",
    "    x = Variable(Fs)\n",
    "    dU = model(x, rho_c)\n",
    "    \n",
    "    c += 1\n",
    "    if c == 1000:\n",
    "        break\n",
    "    \n",
    "    if sum(abs(dU[0]) > 0.1) == 0:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(100):\n",
    "    x = Variable(Fs)\n",
    "    loss = model(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s = Variable(np.array([[1]], dtype=np.float32))\n",
    "t = Variable(np.array([[1]], dtype=np.float32).T)\n",
    "j = Variable(np.array([[0.46143287]], dtype=np.float32).T)\n",
    "R = F.forget(f, s, t, j)\n",
    "R.backward()\n",
    "\n",
    "s.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "j.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "h = t.__rmatmul__(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "h.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.array([np.zeros(21)], dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.append(h.W.grad, h.b.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = Variable(np.array([-1, 0, 1], dtype=np.float32))\n",
    "z = F.sin(x)\n",
    "z.grad = np.ones(3, dtype=np.float32)\n",
    "z.backward()\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(100):\n",
    "    Farray = Fs\n",
    "    x = Variable(Fs)\n",
    "    model.zerograds()\n",
    "    loss = model(x)\n",
    "    loss.backward()\n",
    "    optimizer.update()\n",
    "    print('epoch:', i)\n",
    "    print('train mean loss={}'.format(loss_val))\n",
    "    print(' - - - - - - - - - ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = [0]\n",
    "\n",
    "one = Variable(np.ones((1, 1), dtype=np.float32))\n",
    "\n",
    "for i in range(1000):\n",
    "    X = Variable(np.array([Farray[i]], dtype=np.float32))\n",
    "    delta = np.tanh(model.W.data*X.data + model.u.data*d[-1] + model.b.data)\n",
    "    d.append(delta[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}